PMM-12548 Response length json tag.

Bumps [github.com/rs/zerolog](https://github.com/rs/zerolog) from 1.33.0 to 1.34.0.
- [Commits](https://github.com/rs/zerolog/compare/v1.33.0...v1.34.0)

---
updated-dependencies:
- dependency-name: github.com/rs/zerolog
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/toolkit.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Build the Docker image
       run: echo "FROM oraclelinux:9-slim" > Dockerfile; echo "RUN microdnf -y update" >> Dockerfile; echo "COPY bin/* /usr/bin/" >> Dockerfile; docker build . --file Dockerfile --tag percona-toolkit:${{ github.sha }}
     - name: Run Trivy vulnerability scanner
-      uses: aquasecurity/trivy-action@0.29.0
+      uses: aquasecurity/trivy-action@0.30.0
       with:
           image-ref: 'percona-toolkit:${{ github.sha }}'
           format: 'table'
@@ -36,7 +36,7 @@ jobs:
           vuln-type: 'os,library'
           severity: 'CRITICAL,HIGH'
     - name: Upload a Build Artifact
-      uses: actions/upload-artifact@v4.4.3
+      uses: actions/upload-artifact@v4.6.2
       with:
         name: binaries
         path: bin/*

bin/pt-archiver

@@ -2993,7 +2993,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";
@@ -6966,7 +6969,7 @@ sub main {
           warn "Invalid output format:". $o->get('format');
           warn "Using default 'dump' format";
       } elsif ($o->get('output-format') || '' eq 'csv') {
-              $fields_separated_by = ", ";
+              $fields_separated_by = ",";
               $optionally_enclosed_by = '"';
       }
       my $need_hdr = $o->get('header') && !-f $archive_file;
@@ -7508,7 +7511,7 @@ sub escape {
       s/([\t\n\\])/\\$1/g if defined $_;  # Escape tabs etc
       my $s = defined $_ ? $_ : '\N';             # NULL = \N
       # var & ~var will return 0 only for numbers
-      if ($s !~ /^[0-9,.E]+$/  && $optionally_enclosed_by eq '"') {
+      if ($s !~ /^[0-9,.E]+$/  && $optionally_enclosed_by eq '"' && $s ne '\N') {
           $s =~ s/([^\\])"/$1\\"/g;
           $s = $optionally_enclosed_by."$s".$optionally_enclosed_by;
       }

bin/pt-deadlock-logger

@@ -2088,7 +2088,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-duplicate-key-checker

@@ -133,7 +133,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-find

@@ -1690,7 +1690,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-fk-error-logger

@@ -1242,7 +1242,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-heartbeat

@@ -3564,7 +3564,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-index-usage

@@ -589,7 +589,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-kill

@@ -4905,7 +4905,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-online-schema-change

@@ -2866,7 +2866,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";
@@ -5862,8 +5865,6 @@ sub _nibble_params {
       );
       PTDEBUG && _d('Ascend params:', Dumper($asc));
 
-      my $force_concat_enums;
-
 
       my $from     = "$tbl->{name} FORCE INDEX(`$index`)";
       my $order_by = join(', ', map {$q->quote($_)} @{$index_cols});
@@ -8942,12 +8943,20 @@ sub main {
       $sth->finish();
       PTDEBUG && _d('Last chunk:', Dumper($last_chunk));
 
-      if ( !$last_chunk || !$last_chunk->{new_table_name} ) {
+      if ( !$last_chunk ) {
          $oktorun = undef;
          _die("Option --resume refers non-existing job ID: ${old_job_id}. Exiting."
             , UNSUPPORTED_OPERATION);
       }
 
+      if ( !$last_chunk->{new_table_name}
+         || !$last_chunk->{lower_boundary}
+         || !$last_chunk->{upper_boundary} ) {
+         $oktorun = undef;
+         _die("Option --resume refers job ${old_job_id} with empty boundaries. Exiting."
+            , UNSUPPORTED_OPERATION);
+      }
+
       if ( $last_chunk->{db} ne $db
          || $last_chunk->{tbl} ne $tbl
          || $last_chunk->{altr} ne $o->get('alter') ){
@@ -9396,12 +9405,12 @@ sub main {
       };
       if ( $EVAL_ERROR ) {
          chomp $EVAL_ERROR;
-         _die("Error checking --max-load or --critial-load: $EVAL_ERROR.  "
+         _die("Error checking --max-load or --critical-load: $EVAL_ERROR.  "
             . "Check that the variables specified for --max-load and "
             . "--critical-load are spelled correctly and exist in "
             . "SHOW GLOBAL STATUS.  Current values for these options are:\n"
             . "  --max-load " . (join(',', @{$o->get('max-load')})) . "\n"
-            . "  --critial-load "  . (join(',', @{$o->get('critical-load')}))
+            . "  --critical-load "  . (join(',', @{$o->get('critical-load')}))
             , INVALID_PARAMETERS);
       }
 
@@ -9606,11 +9615,16 @@ sub main {
          #                   ''
          #   doesn't match '(?-xism:Failed to find a unique new table name)'
 
-         # (*) Frank:  commented them out because it caused infinite loop
-         # and the mentioned test error doesn't arise
-
          my $original_error = $EVAL_ERROR;
-         my $original_error_code = $?;
+         my $original_error_code;
+         if ( $? ) {
+            $original_error_code = $?;
+         } 
+         else {
+            $original_error_code = $!;
+         }
+
+         $SIG{__DIE__} = 'DEFAULT';
 
          foreach my $task ( reverse @cleanup_tasks ) {
             eval {
@@ -9796,9 +9810,9 @@ sub main {
 
    if ( $o->get('history') ) {
       my $sth = $cxn->dbh()->prepare(
-         "UPDATE ${hist_table} SET new_table_name = ?"
+         "UPDATE ${hist_table} SET new_table_name = ? WHERE job_id = ?"
       );
-      $sth->execute($new_tbl->{tbl});
+      $sth->execute($new_tbl->{tbl}, $job_id);
    }
 
    # If the new table still exists, drop it unless the tool was interrupted.
@@ -9911,7 +9925,7 @@ sub main {
       );
    }
 
-   if ( my $alter = $o->get('alter') ) {
+   if ( (my $alter = $o->get('alter')) && !$o->get('resume') ) {
       print "Altering new table...\n";
       my $sql = "ALTER TABLE $new_tbl->{name} $alter";
       print $sql, "\n" if $o->get('print');
@@ -9920,10 +9934,12 @@ sub main {
          $cxn->dbh()->do($sql);
       };
       if ( $EVAL_ERROR ) {
-        if ( $plugin && $plugin->can('before_die') ) {
-          $plugin->before_die(exit_status => $EVAL_ERROR);
-        }
+         if ( $plugin && $plugin->can('before_die') ) {
+            $plugin->before_die(exit_status => $EVAL_ERROR);
+         }
          # this is trapped by a signal handler. Don't replace it with _die
+         # we need to override $SIG{__DIE__} to return correct error code
+         $SIG{__DIE__} = sub { print(STDERR "$_[0]"); exit ERROR_ALTERING_TABLE;  };
          die "Error altering new table $new_tbl->{name}: $EVAL_ERROR\n";
       }
       print "Altered $new_tbl->{name} OK.\n";

bin/pt-query-digest

@@ -1257,7 +1257,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-replica-restart

@@ -135,7 +135,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-table-checksum

@@ -4134,7 +4134,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";
@@ -6683,8 +6686,6 @@ sub _nibble_params {
       );
       PTDEBUG && _d('Ascend params:', Dumper($asc));
 
-      my $force_concat_enums;
-
 
       my $from     = "$tbl->{name} FORCE INDEX(`$index`)";
       my $order_by = join(', ', map {$q->quote($_)} @{$index_cols});
@@ -11074,7 +11075,7 @@ sub main {
                     . "(db, tbl, chunk, chunk_index,"
                     . " lower_boundary, upper_boundary, this_cnt, this_crc) "
                     . "SELECT"
-                    . ($cluster->is_cluster_node($source_cxn) ? ' /*!99997*/' : '')
+                    . ($cluster->is_cluster_node($source_cxn) ? ' /*!99997 */' : '')
                     . " ?, ?, ?, ?, ?, ?,";
    my $past_cols    = " COUNT(*), '0'";
 

bin/pt-table-sync

@@ -1909,7 +1909,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";
@@ -3612,6 +3615,14 @@ sub make_UPDATE {
       @cols = $self->sort_cols($row);
    }
    my $types = $self->{tbl_struct}->{type_for};
+
+   foreach my $col ( @cols ) {
+      my $is_json = ($types->{$col} || '') =~ m/json/i;
+      if ( $is_json && defined $row->{$col} ) {
+         utf8::decode($row->{$col});
+      }
+   }
+
    return "UPDATE $self->{dst_db_tbl} SET "
       . join(', ', map {
             my $is_hex = ($types->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i;
@@ -3660,6 +3671,13 @@ sub make_row {
    my $q     = $self->{Quoter};
    my $type_for = $self->{tbl_struct}->{type_for};
 
+   foreach my $col ( @cols ) {
+      my $is_json = ($type_for->{$col} || '') =~ m/json/i;
+      if ( $is_json && defined $row->{$col} ) {
+         utf8::decode($row->{$col});
+      }
+   }
+
    return "$verb INTO $self->{dst_db_tbl}("
       . join(', ', map { $q->quote($_) } @cols)
       . ') VALUES ('
@@ -3708,6 +3726,7 @@ sub sort_cols {
    my @cols;
    if ( $self->{tbl_struct} ) {
       my $pos = $self->{tbl_struct}->{col_posn};
+      my $is_generated = $self->{tbl_struct}->{is_generated};
       my @not_in_tbl;
       @cols = sort {
             $pos->{$a} <=> $pos->{$b}
@@ -3721,6 +3740,9 @@ sub sort_cols {
                1;
             }
          }
+         grep {
+            !$is_generated->{$_}
+         }
          sort keys %$row;
       push @cols, @not_in_tbl if @not_in_tbl;
    }

bin/pt-table-usage

@@ -6680,7 +6680,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

bin/pt-upgrade

@@ -1254,7 +1254,10 @@ sub quote_val {
    return $val if $val =~ m/^0x[0-9a-fA-F]+$/  # quote hex data
                   && !$args{is_char};          # unless is_char is true
 
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    $val =~ s/(['\\])/\\$1/g;
    return "'$val'";

config/rpm/percona-toolkit.spec

@@ -11,7 +11,6 @@ Vendor:    Percona
 URL:       http://www.percona.com/software/percona-toolkit/
 Source:    percona-toolkit-%{version}.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
-BuildArch: @@ARCHITECTURE@@
 
 BuildRequires: perl(ExtUtils::MakeMaker) make
 Requires:  perl(DBI) >= 1.13, perl(DBD::mysql) >= 1.0, perl(Time::HiRes), perl(IO::Socket::SSL), perl(Digest::MD5), perl(Term::ReadKey)

config/scripts/pt_builder.sh

@@ -333,11 +333,6 @@ build_srpm(){
     cd ${WORKDIR}/rpmbuild/SPECS
     echo '%undefine _missing_build_ids_terminate_build' | cat - percona-toolkit.spec > pt.spec && mv pt.spec percona-toolkit.spec
     echo '%define debug_package %{nil}' | cat - percona-toolkit.spec > pt.spec && mv pt.spec percona-toolkit.spec
-    if [ x"$ARCH" = "xaarch64" ]; then
-	sed -i "s/@@ARCHITECTURE@@/aarch64/" percona-toolkit.spec
-    else
-	sed -i "s/@@ARCHITECTURE@@/x86_64/" percona-toolkit.spec
-    fi
 
     cd ${WORKDIR}/${PRODUCT_FULL}
     rm -rf bin/govendor
@@ -398,7 +393,7 @@ build_rpm(){
     ARCH=$(echo $(uname -m) | sed -e 's:i686:i386:g')
     echo "RHEL=${RHEL}" >> percona-toolkit.properties
     echo "ARCH=${ARCH}" >> percona-toolkit.properties
-    rpmbuild --target=${ARCH} --define "version $VERSION" --define "VERSION $VERSION" --define "dist .el${RHEL}" --define "release $RPM_RELEASE.el${RHEL}" --define "_topdir ${WORKDIR}/rpmbuild" --rebuild rpmbuild/SRPMS/${SRC_RPM}
+    rpmbuild --define "version $VERSION" --define "VERSION $VERSION" --define "dist .el${RHEL}" --define "release $RPM_RELEASE.el${RHEL}" --define "_topdir ${WORKDIR}/rpmbuild" --rebuild rpmbuild/SRPMS/${SRC_RPM}
 
     return_code=$?
     if [ $return_code != 0 ]; then

config/sphinx-build/_static/_templates/theme/localtoc.html

@@ -22,3 +22,4 @@
    {% endif %}
   </ul>
 </nav>
+

config/sphinx-build/_static/_templates/theme/relbar2.html

@@ -1,9 +1,18 @@
 <div class="md-relbar2__inner md-grid">
 <div class="md-content">
   <article class="md-content__inner md-typeset" role="main">
-    <h4>Get Expert Help </h4>
-       <p>If you need assistance, visit the <a class="reference external" href="https://forums.percona.com/c/polyglot-projects/percona-toolkit/13?utm_campaign=Doc%20pages" target="_blank">community forum</a> for comprehensive and free database knowledge.</p> 
-       <p>Contact our <a class="reference external" href="https://www.percona.com/about/contact">Percona Database Experts</a> for professional support and services.</p>
+    <svg width="78" height="69" viewBox="0 0 78 69" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path d="M56.7281 30.7666C62.6528 26.8938 64.5914 18.9942 61.0059 12.7854C59.2094 9.67113 56.3053 7.44082 52.8311 6.50951C49.6122 5.64535 46.2502 5.99872 43.2912 7.49366L39.251 0.5L30.8648 15.0245L11.4811 48.5937H67.021L56.7281 30.7666ZM51.881 10.0674C54.4064 10.7401 56.5079 12.3616 57.8168 14.6194C60.3895 19.0701 59.053 24.7153 54.8808 27.5665L45.1362 10.6905C47.2355 9.68104 49.6034 9.46087 51.881 10.0674ZM39.251 7.87125L60.6339 44.907H48.1228L32.9927 18.7102L39.2499 7.87235L39.251 7.87125ZM17.8682 44.907L30.8637 22.4035L43.8592 44.907H17.8682Z" fill="url(#paint0_linear_2899_1968)"/>
+      <path d="M4.981 64.943H3.157V68.207H.756V57.323H5.217C7.822 57.323 9.397 58.861 9.397 61.086V61.116C9.397 63.635 7.433 64.94 4.984 64.94L4.981 64.943V64.943ZM6.961 61.134C6.961 60.061 6.213 59.485 5.011 59.485H3.154V62.812H5.056C6.258 62.812 6.958 62.096 6.958 61.163V61.134H6.961ZM10.738 68.208V57.323H18.973V59.455H13.124V61.664H18.27V63.796H13.124V66.082H19.051V68.214H10.738V68.208 68.208ZM27.557 68.208 25.218 64.726H23.332V68.208H20.931V57.323H25.921C28.496 57.323 30.039 58.677 30.039 60.915V60.945C30.039 62.702 29.088 63.807 27.7 64.32L30.367 68.207H27.556L27.557 68.208ZM27.605 61.041C27.605 60.016 26.887 59.485 25.719 59.485H23.333V62.61H25.767C26.936 62.61 27.605 61.987 27.605 61.071V61.042 61.041ZM36.922 68.499C33.668 68.499 31.249 65.994 31.249 62.825V62.795C31.249 59.659 33.619 57.091 37.019 57.091 39.105 57.091 40.356 57.783 41.383 58.792L39.834 60.571C38.98 59.798 38.113 59.327 37.004 59.327 35.141 59.327 33.795 60.871 33.795 62.762V62.793C33.795 64.684 35.107 66.257 37.004 66.257 38.268 66.257 39.043 65.753 39.913 64.964L41.462 66.524C40.322 67.738 39.059 68.493 36.925 68.493L36.922 68.499ZM47.885 68.499C44.47 68.499 42.021 65.962 42.021 62.825V62.795C42.021 59.659 44.503 57.091 47.915 57.091 51.327 57.091 53.779 59.628 53.779 62.765V62.795C53.779 65.931 51.297 68.499 47.885 68.499ZM51.237 62.795C51.237 60.904 49.846 59.331 47.885 59.331 45.925 59.331 44.567 60.874 44.567 62.766V62.796C44.567 64.688 45.959 66.261 47.919 66.261 49.879 66.261 51.237 64.717 51.237 62.826V62.795 62.795ZM67.001 68.217 72.374 57.091 77.746 68.218H75.052L72.374 62.681 69.705 68.218H67.001V68.217ZM66.007 57.327V68.5L57.813 61.884V68.199H55.264V57.091L63.458 63.681V57.327H66.007Z" fill="#2C323E"/>
+      <defs>
+          <linearGradient id="paint0_linear_2899_1968" x1="18.1513" y1="44.7152" x2="61.4356" y2="20.9786" gradientUnits="userSpaceOnUse">
+              <stop stop-color="#FC3519"/>
+              <stop offset="1" stop-color="#F0D136"/>
+          </linearGradient>
+      </defs>
+  </svg>
+    <p>For help, click the link below to get free database assistance or contact our experts for personalized support.</p>
+       <p> <a class="reference external" href="https://docs.percona.com/percona-toolkit/get-help.html" target="_blank">Get help from Percona</a> </p> 
          <hr>
          {# Add the last updated timestamp from git commits using gitstamp extension #}
          {%- if gitstamp %} <small> Last update: {{gitstamp}} </small>

config/sphinx-build/_static/css/material.css

@@ -31,3 +31,4 @@ body,input{
     font-feature-settings:"kern","liga";
     font-family:"Chivo", "Colfax", "Franziska", Helvetica, Arial, sans-serif;
 }
+

config/sphinx-build/_static/percona-logo-1.svg

@@ -0,0 +1,9 @@
+<svg width="520" height="451" viewBox="0 0 520 451" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M423.631 283.376C479.103 247.116 497.253 173.155 463.684 115.024C446.863 85.8662 419.673 64.9845 387.145 56.265C357.008 48.1741 325.531 51.4826 297.826 65.4793L260 0L181.483 135.988L0 450.285H520L423.631 283.376ZM378.25 89.5766C401.894 95.8741 421.57 111.056 433.825 132.195C457.912 173.866 445.399 226.719 406.336 253.414L315.1 95.4103C334.755 85.9589 356.925 83.8976 378.25 89.5766ZM260 69.0145L460.2 415.767H343.063L201.406 170.496L259.99 69.0248L260 69.0145ZM59.8002 415.767L181.472 205.075L303.144 415.767H59.8002Z" fill="url(#paint0_linear_552_15)"/>
+<defs>
+<linearGradient id="paint0_linear_552_15" x1="58.6494" y1="460.228" x2="444.575" y2="74.3019" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FC3519"/>
+<stop offset="1" stop-color="#F0D136"/>
+</linearGradient>
+</defs>
+</svg>

config/sphinx-build/_static/percona-logo.svg

@@ -1,9 +1,10 @@
-<svg width="520" height="451" viewBox="0 0 520 451" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M423.631 283.376C479.103 247.116 497.253 173.155 463.684 115.024C446.863 85.8662 419.673 64.9845 387.145 56.265C357.008 48.1741 325.531 51.4826 297.826 65.4793L260 0L181.483 135.988L0 450.285H520L423.631 283.376ZM378.25 89.5766C401.894 95.8741 421.57 111.056 433.825 132.195C457.912 173.866 445.399 226.719 406.336 253.414L315.1 95.4103C334.755 85.9589 356.925 83.8976 378.25 89.5766ZM260 69.0145L460.2 415.767H343.063L201.406 170.496L259.99 69.0248L260 69.0145ZM59.8002 415.767L181.472 205.075L303.144 415.767H59.8002Z" fill="url(#paint0_linear_552_15)"/>
-<defs>
-<linearGradient id="paint0_linear_552_15" x1="58.6494" y1="460.228" x2="444.575" y2="74.3019" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FC3519"/>
-<stop offset="1" stop-color="#F0D136"/>
-</linearGradient>
-</defs>
-</svg>
+<svg width="78" height="69" viewBox="0 0 78 69" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M56.7281 30.7666C62.6528 26.8938 64.5914 18.9942 61.0059 12.7854C59.2094 9.67113 56.3053 7.44082 52.8311 6.50951C49.6122 5.64535 46.2502 5.99872 43.2912 7.49366L39.251 0.5L30.8648 15.0245L11.4811 48.5937H67.021L56.7281 30.7666ZM51.881 10.0674C54.4064 10.7401 56.5079 12.3616 57.8168 14.6194C60.3895 19.0701 59.053 24.7153 54.8808 27.5665L45.1362 10.6905C47.2355 9.68104 49.6034 9.46087 51.881 10.0674ZM39.251 7.87125L60.6339 44.907H48.1228L32.9927 18.7102L39.2499 7.87235L39.251 7.87125ZM17.8682 44.907L30.8637 22.4035L43.8592 44.907H17.8682Z" fill="url(#paint0_linear_2899_1968)"/>
+    <path d="M4.981 64.943H3.157V68.207H.756V57.323H5.217C7.822 57.323 9.397 58.861 9.397 61.086V61.116C9.397 63.635 7.433 64.94 4.984 64.94L4.981 64.943V64.943ZM6.961 61.134C6.961 60.061 6.213 59.485 5.011 59.485H3.154V62.812H5.056C6.258 62.812 6.958 62.096 6.958 61.163V61.134H6.961ZM10.738 68.208V57.323H18.973V59.455H13.124V61.664H18.27V63.796H13.124V66.082H19.051V68.214H10.738V68.208 68.208ZM27.557 68.208 25.218 64.726H23.332V68.208H20.931V57.323H25.921C28.496 57.323 30.039 58.677 30.039 60.915V60.945C30.039 62.702 29.088 63.807 27.7 64.32L30.367 68.207H27.556L27.557 68.208ZM27.605 61.041C27.605 60.016 26.887 59.485 25.719 59.485H23.333V62.61H25.767C26.936 62.61 27.605 61.987 27.605 61.071V61.042 61.041ZM36.922 68.499C33.668 68.499 31.249 65.994 31.249 62.825V62.795C31.249 59.659 33.619 57.091 37.019 57.091 39.105 57.091 40.356 57.783 41.383 58.792L39.834 60.571C38.98 59.798 38.113 59.327 37.004 59.327 35.141 59.327 33.795 60.871 33.795 62.762V62.793C33.795 64.684 35.107 66.257 37.004 66.257 38.268 66.257 39.043 65.753 39.913 64.964L41.462 66.524C40.322 67.738 39.059 68.493 36.925 68.493L36.922 68.499ZM47.885 68.499C44.47 68.499 42.021 65.962 42.021 62.825V62.795C42.021 59.659 44.503 57.091 47.915 57.091 51.327 57.091 53.779 59.628 53.779 62.765V62.795C53.779 65.931 51.297 68.499 47.885 68.499ZM51.237 62.795C51.237 60.904 49.846 59.331 47.885 59.331 45.925 59.331 44.567 60.874 44.567 62.766V62.796C44.567 64.688 45.959 66.261 47.919 66.261 49.879 66.261 51.237 64.717 51.237 62.826V62.795 62.795ZM67.001 68.217 72.374 57.091 77.746 68.218H75.052L72.374 62.681 69.705 68.218H67.001V68.217ZM66.007 57.327V68.5L57.813 61.884V68.199H55.264V57.091L63.458 63.681V57.327H66.007Z" fill="#2C323E"/>
+    <defs>
+        <linearGradient id="paint0_linear_2899_1968" x1="18.1513" y1="44.7152" x2="61.4356" y2="20.9786" gradientUnits="userSpaceOnUse">
+            <stop stop-color="#FC3519"/>
+            <stop offset="1" stop-color="#F0D136"/>
+        </linearGradient>
+    </defs>
+</svg>

docs/get-help.rst

@@ -0,0 +1,40 @@
+.. _get-help:
+
+Get help from Percona
+**********************
+
+This guide is packed with information but can’t cover everything you need to know about Percona Toolkit or every scenario you might encounter. Don’t be afraid to try things out and ask questions when you get stuck.
+ 
+Ask a question in the Community Forum
+=======================================
+
+Be a part of a space where you can tap into a wealth of knowledge from other database enthusiasts and experts who work with Percona’s software every day. While our service is entirely free, keep in mind that response times can vary depending on the complexity of the question. You are engaging with people who genuinely love solving database challenges.
+
+We recommend visiting our `Community Forum <https://forums.percona.com/t/welcome-to-perconas-community-forum/7>`_.
+It’s an excellent place for discussions, technical insights, and support around Percona database software. If you’re new and feeling a bit unsure, our `FAQ <https://forums.percona.com/faq>`_ and `Guide for New Users <https://forums.percona.com/t/faq-guide-for-new-users/8562>`_ ease you in.
+
+If you have thoughts, feedback, or ideas, the community team would like to hear from you at `Any ideas on how to make the forum better? <https://forums.percona.com/t/any-ideas-on-how-to-make-the-forum-better/11522>`_.
+We’re always excited to connect and improve everyone's experience.
+
+Work with a Percona Expert
+==============================
+
+Percona experts bring years of experience in tackling tough database performance issues and design challenges.
+
+.. raw:: html
+
+   <div data-tf-live="01JKGY9435F75X6DHG92DJZB26"></div>
+   <script src="//embed.typeform.com/next/embed.js"></script>
+
+We understand your challenges when managing complex database environments. That's why we offer various services to help you simplify your operations and achieve your goals.
+
++----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| Service                    | Description                                                                                                                                                           |
++----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| 24/7 Expert Support        | Our dedicated team of database experts is available 24/7 to assist you with any database issues. We provide flexible support plans tailored to your specific needs.   |
+| Hands-On Database Management | Our managed services team can take over the day-to-day management of your database infrastructure, freeing up your time to focus on other priorities.               |
+| Expert Consulting          | Our experienced consultants provide guidance on database topics like architecture design, migration planning, performance optimization, and security best practices.  |
+| Comprehensive Training     | Our training programs help your team develop skills to manage databases effectively, offering virtual and in-person courses.                                          |
++----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+
+We're here to help you every step of the way. Whether you need a quick fix or a long-term partnership, we're ready to provide your expertise and support.

docs/index.rst

@@ -29,6 +29,7 @@ Getting Percona Toolkit
    :glob:
 
    installation
+   get-help
 
 Tools
 =====

docs/percona-toolkit.pod

@@ -529,7 +529,7 @@ list of supported platforms and versions.
 
 In order to support IPv6 addresses to connect to MySQL, Perl DBD::MySQL driver v4.033_01 is
 required. Also, as stated in RFC 3986 L<https://www.ietf.org/rfc/rfc3986.txt> section 3.2.2
-brackes must be used to distinguish host and port.
+brackets must be used to distinguish host and port.
 Examples: L<https://metacpan.org/pod/DBD::mysql#port>
 
 =head1 BUGS

go.mod

@@ -2,16 +2,18 @@ module github.com/percona/percona-toolkit
 
 go 1.23.4
 
+toolchain go1.24.1
+
 require (
 	github.com/AlekSi/pointer v1.2.0
 	github.com/Ladicle/tabwriter v1.0.0
 	github.com/Masterminds/semver v1.5.0
 	github.com/alecthomas/kingpin v2.2.6+incompatible
-	github.com/alecthomas/kong v1.6.0
+	github.com/alecthomas/kong v1.9.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/go-ini/ini v1.67.0
 	github.com/golang/mock v1.6.0
-	github.com/google/go-cmp v0.6.0
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-version v1.7.0
 	github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef
@@ -21,17 +23,17 @@ require (
 	github.com/pborman/getopt v1.1.0
 	github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.33.0
+	github.com/rs/zerolog v1.34.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.10.0
 	github.com/xlab/treeprint v1.2.0
-	go.mongodb.org/mongo-driver v1.17.1
-	golang.org/x/crypto v0.31.0
+	go.mongodb.org/mongo-driver v1.17.3
+	golang.org/x/crypto v0.36.0
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.32.0
+	k8s.io/api v0.32.3
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
 )
 
@@ -45,7 +47,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
@@ -60,14 +62,14 @@ require (
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/term v0.27.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/net v0.36.0 // indirect
+	golang.org/x/sync v0.12.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/term v0.30.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apimachinery v0.32.0 // indirect
+	k8s.io/apimachinery v0.32.3 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect

go.sum

@@ -8,8 +8,8 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v
 github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
-github.com/alecthomas/kong v1.6.0 h1:mwOzbdMR7uv2vul9J0FU3GYxE7ls/iX1ieMg5WIM6gE=
-github.com/alecthomas/kong v1.6.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU=
+github.com/alecthomas/kong v1.9.0 h1:Wgg0ll5Ys7xDnpgYBuBn/wPeLGAuK0NvYmEcisJgrIs=
+github.com/alecthomas/kong v1.9.0/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU=
 github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
 github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -38,8 +38,8 @@ github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+Licev
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -55,8 +55,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
-github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -88,9 +88,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
-github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
 github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
 github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
@@ -125,14 +125,14 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
-go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
+go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ=
+go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -146,17 +146,15 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
-golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
+golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -172,18 +170,18 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
+golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -207,10 +205,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE=
-k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0=
-k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg=
-k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
+k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls=
+k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k=
+k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
+k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=

lib/ChangeHandler.pm

@@ -324,6 +324,16 @@ sub make_UPDATE {
       @cols = $self->sort_cols($row);
    }
    my $types = $self->{tbl_struct}->{type_for};
+
+   # MySQL uses utf8mb4 for all strings in JSON, but
+   # DBD::mysql does not decode it accordingly
+   foreach my $col ( @cols ) {
+      my $is_json = ($types->{$col} || '') =~ m/json/i;
+      if ( $is_json && defined $row->{$col} ) {
+         utf8::decode($row->{$col});
+      }
+   }
+
    return "UPDATE $self->{dst_db_tbl} SET "
       . join(', ', map {
             my $is_hex = ($types->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i;
@@ -403,6 +413,15 @@ sub make_row {
    my $q     = $self->{Quoter};
    my $type_for = $self->{tbl_struct}->{type_for};
 
+   # MySQL uses utf8mb4 for all strings in JSON, but
+   # DBD::mysql does not decode it accordingly
+   foreach my $col ( @cols ) {
+      my $is_json = ($type_for->{$col} || '') =~ m/json/i;
+      if ( $is_json && defined $row->{$col} ) {
+         utf8::decode($row->{$col});
+      }
+   }
+
    return "$verb INTO $self->{dst_db_tbl}("
       . join(', ', map { $q->quote($_) } @cols)
       . ') VALUES ('
@@ -462,7 +481,8 @@ sub get_changes {
 
 
 # Sub: sort_cols
-#   Sort a row's columns based on their real order in the table.
+#   Sort a row's columns based on their real order in the table, and remove
+#   generated columns.
 #   This requires that the optional tbl_struct arg was passed to <new()>.
 #   If not, the rows are sorted alphabetically.
 #
@@ -476,6 +496,7 @@ sub sort_cols {
    my @cols;
    if ( $self->{tbl_struct} ) {
       my $pos = $self->{tbl_struct}->{col_posn};
+      my $is_generated = $self->{tbl_struct}->{is_generated};
       my @not_in_tbl;
       @cols = sort {
             $pos->{$a} <=> $pos->{$b}
@@ -489,6 +510,9 @@ sub sort_cols {
                1;
             }
          }
+         grep {
+            !$is_generated->{$_}
+         }
          sort keys %$row;
       push @cols, @not_in_tbl if @not_in_tbl;
    }

lib/Lmo/Types.pm

@@ -56,7 +56,7 @@ sub check_type_constraints {
       . (defined $val ? Lmo::Dumper($val) : 'undef') )
 }
 
-# Nested (or parametized) constraints look like this: ArrayRef[CONSTRAINT] or
+# Nested (or parameritized) constraints look like this: ArrayRef[CONSTRAINT] or
 # Maybe[CONSTRAINT]. This function returns a coderef that implements one of
 # these.
 sub _nested_constraints {
@@ -64,7 +64,7 @@ sub _nested_constraints {
 
    my $inner_types;
    if ( $type =~ /\A(ArrayRef|Maybe)\[(.*)\]\z/ ) {
-      # If the inner constraint -- the part within brackets -- is also a parametized
+      # If the inner constraint -- the part within brackets -- is also a parametirized
       # constraint, then call this function recursively.
       $inner_types = _nested_constraints($1, $2);
    }

lib/NibbleIterator.pm

@@ -208,10 +208,6 @@ sub _nibble_params {
       );
       PTDEBUG && _d('Ascend params:', Dumper($asc));
 
-      # Check if enum fields items are sorted or not.
-      # If they are sorted we can skip adding CONCAT to improve the queries eficiency.
-      my $force_concat_enums;
-
       # Make SQL statements, prepared on first call to next().  FROM and
       # ORDER BY are the same for all statements.  FORCE IDNEX and ORDER BY
       # are needed to ensure deterministic nibbling.

lib/Processlist.pm

@@ -507,7 +507,7 @@ sub find {
          }
          my $reason = 'Exceeds busy time';
          PTDEBUG && _d($reason);
-         # Saving the reasons for each query in the objct is a bit nasty,
+         # Saving the reasons for each query in the object is a bit nasty,
          # but the alternatives are worse:
          # - Saving internal data in the query
          # - Instead of using the stringified hashref as a key, using

lib/Quoter.pm

@@ -78,7 +78,10 @@ sub quote_val {
                   && !$args{is_char};          # unless is_char is true
 
    # https://bugs.launchpad.net/percona-toolkit/+bug/1229861
-   return $val if $args{is_float};
+   if ( $args{is_float} ) {
+      return sprintf("%.17g", $val) if $val - "$val" != 0;
+      return $val;
+   }
 
    # Quote and return non-numeric vals.
    $val =~ s/(['\\])/\\$1/g;

lib/TableChunker.pm

@@ -1140,7 +1140,7 @@ sub timestampdiff {
 # Some column types can store invalid values, like most of the temporal
 # types.  When evaluated, invalid values return NULL.  If the value is
 # NULL to begin with, then it is not invalid because NULL is valid.
-# For example, TO_DAYS('2009-00-00') evalues to NULL because that date
+# For example, TO_DAYS('2009-00-00') evaluates to NULL because that date
 # is invalid, even though it's storable.
 sub get_valid_end_points {
    my ( $self, %args ) = @_;

requirements-docbuild.txt

@@ -14,9 +14,8 @@ sphinxcontrib-srclinks
 sphinx-tabs
 
 certifi>=2024.7.4 # not directly required, pinned by Snyk to avoid a vulnerability
-jinja2>=3.1.4 # not directly required, pinned by Snyk to avoid a vulnerability
+jinja2>=3.1.6 # not directly required, pinned by Snyk to avoid a vulnerability
 pygments>=2.15.0 # not directly required, pinned by Snyk to avoid a vulnerability
-requests>=2.32.2 # not directly required, pinned by Snyk to avoid a vulnerability
-setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
+requests>=2.31.0 # not directly required, pinned by Snyk to avoid a vulnerability
+setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability
 idna>=3.7 # not directly required, pinned by Snyk to avoid a vulnerability
-urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability

src/go/mongolib/fingerprinter/fingerprinter_test.go

@@ -10,11 +10,12 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/percona/percona-toolkit/src/go/lib/tutil"
-	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/percona/percona-toolkit/src/go/lib/tutil"
+	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
 )
 
 const (
@@ -33,7 +34,8 @@ func TestMain(m *testing.M) {
 		log.Printf("cannot get root path: %s", err.Error())
 		os.Exit(1)
 	}
-	os.Exit(m.Run())
+	code := m.Run()
+	os.Exit(code)
 }
 
 func TestSingleFingerprint(t *testing.T) {

src/go/mongolib/profiler/profiler_test.go

@@ -7,14 +7,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+
 	tu "github.com/percona/percona-toolkit/src/go/internal/testutils"
 	"github.com/percona/percona-toolkit/src/go/lib/tutil"
 	"github.com/percona/percona-toolkit/src/go/mongolib/fingerprinter"
 	"github.com/percona/percona-toolkit/src/go/mongolib/stats"
 	"github.com/percona/percona-toolkit/src/go/pt-mongodb-query-digest/filter"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
 const (
@@ -38,7 +39,8 @@ func TestMain(m *testing.M) {
 		log.Printf("cannot get root path: %s", err.Error())
 		os.Exit(1)
 	}
-	os.Exit(m.Run())
+	code := m.Run()
+	os.Exit(code)
 }
 
 func TestRegularIterator(t *testing.T) {

src/go/mongolib/proto/system.profile.go

@@ -8,85 +8,104 @@ import (
 )
 
 // docsExamined is renamed from nscannedObjects in 3.2.0
+// json tags are used for PMM purposes
 // https://docs.mongodb.com/manual/reference/database-profiler/#system.profile.docsExamined
 type SystemProfile struct {
-	AllUsers        []interface{} `bson:"allUsers"`
-	Client          string        `bson:"client"`
-	CursorExhausted bool          `bson:"cursorExhausted"`
-	DocsExamined    int           `bson:"docsExamined"`
-	NscannedObjects int           `bson:"nscannedObjects"`
+	AllUsers        []interface{} `bson:"allUsers" json:"allUsers"`
+	Client          string        `bson:"client" json:"client"`
+	CursorExhausted bool          `bson:"cursorExhausted" json:"cursorExhausted"`
 	ExecStats       struct {
-		Advanced                    int `bson:"advanced"`
-		ExecutionTimeMillisEstimate int `bson:"executionTimeMillisEstimate"`
+		Advanced                    int `bson:"advanced" json:"advanced"`
+		ExecutionTimeMillisEstimate int `bson:"executionTimeMillisEstimate" json:"executionTimeMillisEstimate"`
 		InputStage                  struct {
-			Advanced                    int    `bson:"advanced"`
-			Direction                   string `bson:"direction"`
-			DocsExamined                int    `bson:"docsExamined"`
-			ExecutionTimeMillisEstimate int    `bson:"executionTimeMillisEstimate"`
+			Advanced                    int    `bson:"advanced" json:"advanced"`
+			Direction                   string `bson:"direction" json:"direction"`
+			DocsExamined                int    `bson:"docsExamined" json:"docsExamined"`
+			ExecutionTimeMillisEstimate int    `bson:"executionTimeMillisEstimate" json:"executionTimeMillisEstimate"`
 			Filter                      struct {
 				Date struct {
-					Eq string `bson:"$eq"`
-				} `bson:"date"`
-			} `bson:"filter"`
-			Invalidates  int    `bson:"invalidates"`
-			IsEOF        int    `bson:"isEOF"`
-			NReturned    int    `bson:"nReturned"`
-			NeedTime     int    `bson:"needTime"`
-			NeedYield    int    `bson:"needYield"`
-			RestoreState int    `bson:"restoreState"`
-			SaveState    int    `bson:"saveState"`
-			Stage        string `bson:"stage"`
-			Works        int    `bson:"works"`
-		} `bson:"inputStage"`
-		Invalidates  int    `bson:"invalidates"`
-		IsEOF        int    `bson:"isEOF"`
-		LimitAmount  int    `bson:"limitAmount"`
-		NReturned    int    `bson:"nReturned"`
-		NeedTime     int    `bson:"needTime"`
-		NeedYield    int    `bson:"needYield"`
-		RestoreState int    `bson:"restoreState"`
-		SaveState    int    `bson:"saveState"`
-		Stage        string `bson:"stage"`
-		Works        int    `bson:"works"`
-	} `bson:"execStats"`
-	KeyUpdates   int `bson:"keyUpdates"`
-	KeysExamined int `bson:"keysExamined"`
+					Eq string `bson:"$eq" json:"$eq"`
+				} `bson:"date" json:"date"`
+			} `bson:"filter" json:"filter"`
+			Invalidates  int    `bson:"invalidates" json:"invalidates"`
+			IsEOF        int    `bson:"isEOF" json:"isEOF"`
+			NReturned    int    `bson:"nReturned" json:"nReturned"`
+			NeedTime     int    `bson:"needTime" json:"needTime"`
+			NeedYield    int    `bson:"needYield" json:"needYield"`
+			RestoreState int    `bson:"restoreState" json:"restoreState"`
+			SaveState    int    `bson:"saveState" json:"saveState"`
+			Stage        string `bson:"stage" json:"stage"`
+			Works        int    `bson:"works" json:"works"`
+		} `bson:"inputStage" json:"inputStage"`
+		Invalidates  int    `bson:"invalidates" json:"invalidates"`
+		IsEOF        int    `bson:"isEOF" json:"isEOF"`
+		LimitAmount  int    `bson:"limitAmount" json:"limitAmount"`
+		NReturned    int    `bson:"nReturned" json:"nReturned"`
+		NeedTime     int    `bson:"needTime" json:"needTime"`
+		NeedYield    int    `bson:"needYield" json:"needYield"`
+		RestoreState int    `bson:"restoreState" json:"restoreState"`
+		SaveState    int    `bson:"saveState" json:"saveState"`
+		Stage        string `bson:"stage" json:"stage"`
+		Works        int    `bson:"works" json:"works"`
+		DocsExamined int    `bson:"docsExamined" json:"docsExamined"`
+	} `bson:"execStats" json:"execStats"`
+	KeyUpdates   int `bson:"keyUpdates" json:"keyUpdates"`
+	KeysExamined int `bson:"keysExamined" json:"keysExamined"`
 	Locks        struct {
 		Collection struct {
 			AcquireCount struct {
-				R int `bson:"R"`
-			} `bson:"acquireCount"`
-		} `bson:"Collection"`
+				Read       int `bson:"R" json:"R"`
+				ReadShared int `bson:"r" json:"r"`
+			} `bson:"acquireCount" json:"acquireCount"`
+		} `bson:"Collection" json:"Collection"`
 		Database struct {
 			AcquireCount struct {
-				R int `bson:"r"`
-			} `bson:"acquireCount"`
-		} `bson:"Database"`
+				ReadShared int `bson:"r" json:"r"`
+			} `bson:"acquireCount" json:"acquireCount"`
+			AcquireWaitCount struct {
+				ReadShared int `bson:"r" json:"r"`
+			} `bson:"acquireWaitCount" json:"acquireWaitCount"`
+			TimeAcquiringMicros struct {
+				ReadShared int64 `bson:"r" json:"r"`
+			} `bson:"timeAcquiringMicros" json:"timeAcquiringMicros"`
+		} `bson:"Database" json:"Database"`
 		Global struct {
 			AcquireCount struct {
-				R int `bson:"r"`
-			} `bson:"acquireCount"`
-		} `bson:"Global"`
+				ReadShared  int `bson:"r" json:"r"`
+				WriteShared int `bson:"w" json:"w"`
+			} `bson:"acquireCount" json:"acquireCount"`
+		} `bson:"Global" json:"Global"`
 		MMAPV1Journal struct {
 			AcquireCount struct {
-				R int `bson:"r"`
-			} `bson:"acquireCount"`
-		} `bson:"MMAPV1Journal"`
-	} `bson:"locks"`
-	Millis             int       `bson:"millis"`
-	Nreturned          int       `bson:"nreturned"`
-	Ns                 string    `bson:"ns"`
-	NumYield           int       `bson:"numYield"`
-	Op                 string    `bson:"op"`
-	Protocol           string    `bson:"protocol"`
-	Query              bson.D    `bson:"query"`
-	UpdateObj          bson.D    `bson:"updateobj"`
-	Command            bson.D    `bson:"command"`
-	OriginatingCommand bson.D    `bson:"originatingCommand"`
-	ResponseLength     int       `bson:"responseLength"`
-	Ts                 time.Time `bson:"ts"`
-	User               string    `bson:"user"`
-	WriteConflicts     int       `bson:"writeConflicts"`
+				ReadShared int `bson:"r" json:"r"`
+			} `bson:"acquireCount" json:"acquireCount"`
+		} `bson:"MMAPV1Journal" json:"MMAPV1Journal"`
+	} `bson:"locks" json:"locks"`
+	Millis             int       `bson:"millis" json:"durationMillis"`
+	Nreturned          int       `bson:"nreturned" json:"nreturned"`
+	Ns                 string    `bson:"ns" json:"ns"`
+	NumYield           int       `bson:"numYield" json:"numYield"`
+	Op                 string    `bson:"op" json:"op"`
+	PlanSummary        string    `bson:"planSummary" json:"planSummary"`
+	Protocol           string    `bson:"protocol" json:"protocol"`
+	Query              bson.D    `bson:"query" json:"query"`
+	UpdateObj          bson.D    `bson:"updateobj" json:"updateobj"`
+	Command            bson.D    `bson:"command" json:"command"`
+	OriginatingCommand bson.D    `bson:"originatingCommand" json:"originatingCommand"`
+	ResponseLength     int       `bson:"responseLength" json:"reslen"`
+	Ts                 time.Time `bson:"ts" json:"ts"`
+	User               string    `bson:"user" json:"user"`
+	WriteConflicts     int       `bson:"writeConflicts" json:"writeConflicts"`
+	DocsExamined       int       `bson:"docsExamined" json:"docsExamined"`
+	QueryHash          string    `bson:"queryHash" json:"queryHash"`
+	Storage            struct {
+		Data struct {
+			BytesRead         int64 `bson:"bytesRead" json:"bytesRead"`
+			TimeReadingMicros int64 `bson:"timeReadingMicros" json:"timeReadingMicros"`
+		} `bson:"data" json:"data"`
+	} `bson:"storage" json:"storage"`
+	AppName  string `bson:"appName" json:"appName"`
+	Comments string `bson:"comments" json:"comments"`
 }
 
 func NewExampleQuery(doc SystemProfile) ExampleQuery {

src/go/mongolib/stats/stats.go

@@ -4,6 +4,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -13,6 +14,11 @@ import (
 	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
 )
 
+const (
+	planSummaryCollScan = "COLLSCAN"
+	planSummaryIXScan   = "IXSCAN"
+)
+
 type StatsError struct {
 	error
 }
@@ -86,18 +92,24 @@ func (s *Stats) Add(doc proto.SystemProfile) error {
 			Namespace:   fp.Namespace,
 			TableScan:   false,
 			Query:       string(queryBson),
+			PlanSummary: doc.PlanSummary,
+			QueryHash:   doc.QueryHash,
+			AppName:     doc.AppName,
+			Client:      doc.Client,
+			User:        strings.Split(doc.User, "@")[0],
+			Comments:    doc.Comments,
 		}
 		s.setQueryInfoAndCounters(key, qiac)
 	}
 	qiac.Count++
-	// docsExamined is renamed from nscannedObjects in 3.2.0.
-	// https://docs.mongodb.com/manual/reference/database-profiler/#system.profile.docsExamined
 	s.Lock()
-	if doc.NscannedObjects > 0 {
-		qiac.NScanned = append(qiac.NScanned, float64(doc.NscannedObjects))
-	} else {
-		qiac.NScanned = append(qiac.NScanned, float64(doc.DocsExamined))
+	if qiac.PlanSummary == planSummaryCollScan {
+		qiac.CollScanCount++
 	}
+	if strings.HasPrefix(qiac.PlanSummary, planSummaryIXScan) {
+		qiac.PlanSummary = planSummaryIXScan
+	}
+
 	qiac.NReturned = append(qiac.NReturned, float64(doc.Nreturned))
 	qiac.QueryTime = append(qiac.QueryTime, float64(doc.Millis))
 	qiac.ResponseLength = append(qiac.ResponseLength, float64(doc.ResponseLength))
@@ -107,6 +119,42 @@ func (s *Stats) Add(doc proto.SystemProfile) error {
 	if qiac.LastSeen.IsZero() || qiac.LastSeen.Before(doc.Ts) {
 		qiac.LastSeen = doc.Ts
 	}
+
+	if doc.DocsExamined > 0 {
+		qiac.DocsExamined = append(qiac.DocsExamined, float64(doc.DocsExamined))
+	}
+	if doc.KeysExamined > 0 {
+		qiac.KeysExamined = append(qiac.KeysExamined, float64(doc.KeysExamined))
+	}
+	if doc.Locks.Global.AcquireCount.ReadShared > 0 {
+		qiac.LocksGlobalAcquireCountReadSharedCount++
+		qiac.LocksGlobalAcquireCountReadShared += doc.Locks.Global.AcquireCount.ReadShared
+	}
+	if doc.Locks.Global.AcquireCount.WriteShared > 0 {
+		qiac.LocksGlobalAcquireCountWriteSharedCount++
+		qiac.LocksGlobalAcquireCountWriteShared += doc.Locks.Global.AcquireCount.WriteShared
+	}
+	if doc.Locks.Database.AcquireCount.ReadShared > 0 {
+		qiac.LocksDatabaseAcquireCountReadSharedCount++
+		qiac.LocksDatabaseAcquireCountReadShared += doc.Locks.Database.AcquireCount.ReadShared
+	}
+	if doc.Locks.Database.AcquireWaitCount.ReadShared > 0 {
+		qiac.LocksDatabaseAcquireWaitCountReadSharedCount++
+		qiac.LocksDatabaseAcquireWaitCountReadShared += doc.Locks.Database.AcquireWaitCount.ReadShared
+	}
+	if doc.Locks.Database.TimeAcquiringMicros.ReadShared > 0 {
+		qiac.LocksDatabaseTimeAcquiringMicrosReadShared = append(qiac.LocksDatabaseTimeAcquiringMicrosReadShared, float64(doc.Locks.Database.TimeAcquiringMicros.ReadShared))
+	}
+	if doc.Locks.Collection.AcquireCount.ReadShared > 0 {
+		qiac.LocksCollectionAcquireCountReadSharedCount++
+		qiac.LocksCollectionAcquireCountReadShared += doc.Locks.Collection.AcquireCount.ReadShared
+	}
+	if doc.Storage.Data.BytesRead > 0 {
+		qiac.StorageBytesRead = append(qiac.StorageBytesRead, float64(doc.Storage.Data.BytesRead))
+	}
+	if doc.Storage.Data.TimeReadingMicros > 0 {
+		qiac.StorageTimeReadingMicros = append(qiac.StorageTimeReadingMicros, float64(doc.Storage.Data.TimeReadingMicros))
+	}
 	s.Unlock()
 
 	return nil
@@ -185,9 +233,34 @@ type QueryInfoAndCounters struct {
 	BlockedTime    Times
 	LockTime       Times
 	NReturned      []float64
-	NScanned       []float64
 	QueryTime      []float64 // in milliseconds
 	ResponseLength []float64
+
+	PlanSummary   string
+	CollScanCount int
+
+	DocsExamined []float64
+	KeysExamined []float64
+	QueryHash    string
+	AppName      string
+	Client       string
+	User         string
+	Comments     string
+
+	LocksGlobalAcquireCountReadSharedCount       int
+	LocksGlobalAcquireCountReadShared            int
+	LocksGlobalAcquireCountWriteSharedCount      int
+	LocksGlobalAcquireCountWriteShared           int
+	LocksDatabaseAcquireCountReadSharedCount     int
+	LocksDatabaseAcquireCountReadShared          int
+	LocksDatabaseAcquireWaitCountReadSharedCount int
+	LocksDatabaseAcquireWaitCountReadShared      int
+	LocksDatabaseTimeAcquiringMicrosReadShared   []float64 // in microseconds
+	LocksCollectionAcquireCountReadSharedCount   int
+	LocksCollectionAcquireCountReadShared        int
+
+	StorageBytesRead         []float64
+	StorageTimeReadingMicros []float64 // in microseconds
 }
 
 // times is an array of time.Time that implements the Sorter interface
@@ -214,11 +287,15 @@ func (g GroupKey) String() string {
 }
 
 type totalCounters struct {
-	Count     int
-	Scanned   float64
-	Returned  float64
-	QueryTime float64
-	Bytes     float64
+	Count                                      int
+	Returned                                   float64
+	QueryTime                                  float64
+	Bytes                                      float64
+	DocsExamined                               float64
+	KeysExamined                               float64
+	LocksDatabaseTimeAcquiringMicrosReadShared float64
+	StorageBytesRead                           float64
+	StorageTimeReadingMicros                   float64
 }
 
 type QueryStats struct {
@@ -230,14 +307,44 @@ type QueryStats struct {
 	FirstSeen   time.Time
 	LastSeen    time.Time
 
-	Count          int
-	QPS            float64
-	Rank           int
-	Ratio          float64
-	QueryTime      Statistics
-	ResponseLength Statistics
-	Returned       Statistics
-	Scanned        Statistics
+	Count               int
+	QPS                 float64
+	Rank                int
+	Ratio               float64
+	QueryTime           Statistics
+	ResponseLengthCount int
+	ResponseLength      Statistics
+	Returned            Statistics
+
+	PlanSummary       string
+	CollScanCount     int
+	DocsExaminedCount int
+	DocsExamined      Statistics
+	KeysExaminedCount int
+	KeysExamined      Statistics
+	QueryHash         string
+	AppName           string
+	Client            string
+	User              string
+	Comments          string
+
+	LocksGlobalAcquireCountReadSharedCount          int
+	LocksGlobalAcquireCountReadShared               int
+	LocksGlobalAcquireCountWriteSharedCount         int
+	LocksGlobalAcquireCountWriteShared              int
+	LocksDatabaseAcquireCountReadSharedCount        int
+	LocksDatabaseAcquireCountReadShared             int
+	LocksDatabaseAcquireWaitCountReadSharedCount    int
+	LocksDatabaseAcquireWaitCountReadShared         int
+	LocksDatabaseTimeAcquiringMicrosReadSharedCount int
+	LocksDatabaseTimeAcquiringMicrosReadShared      Statistics // in microseconds
+	LocksCollectionAcquireCountReadSharedCount      int
+	LocksCollectionAcquireCountReadShared           int
+
+	StorageBytesReadCount         int
+	StorageBytesRead              Statistics
+	StorageTimeReadingMicrosCount int
+	StorageTimeReadingMicros      Statistics // in microseconds
 }
 
 type Statistics struct {
@@ -254,22 +361,46 @@ type Statistics struct {
 
 func countersToStats(query QueryInfoAndCounters, uptime int64, tc totalCounters) QueryStats {
 	queryStats := QueryStats{
-		Count:          query.Count,
-		ID:             query.ID,
-		Operation:      query.Operation,
-		Query:          query.Query,
-		Fingerprint:    query.Fingerprint,
-		Scanned:        calcStats(query.NScanned),
-		Returned:       calcStats(query.NReturned),
-		QueryTime:      calcStats(query.QueryTime),
-		ResponseLength: calcStats(query.ResponseLength),
-		FirstSeen:      query.FirstSeen,
-		LastSeen:       query.LastSeen,
-		Namespace:      query.Namespace,
-		QPS:            float64(query.Count) / float64(uptime),
-	}
-	if tc.Scanned > 0 {
-		queryStats.Scanned.Pct = queryStats.Scanned.Total * 100 / tc.Scanned
+		Count:                                    query.Count,
+		ID:                                       query.ID,
+		Operation:                                query.Operation,
+		Query:                                    query.Query,
+		Fingerprint:                              query.Fingerprint,
+		Returned:                                 calcStats(query.NReturned),
+		QueryTime:                                calcStats(query.QueryTime),
+		FirstSeen:                                query.FirstSeen,
+		LastSeen:                                 query.LastSeen,
+		Namespace:                                query.Namespace,
+		QPS:                                      float64(query.Count) / float64(uptime),
+		PlanSummary:                              query.PlanSummary,
+		CollScanCount:                            query.CollScanCount,
+		ResponseLengthCount:                      len(query.ResponseLength),
+		ResponseLength:                           calcStats(query.ResponseLength),
+		DocsExaminedCount:                        len(query.DocsExamined),
+		DocsExamined:                             calcStats(query.DocsExamined),
+		KeysExaminedCount:                        len(query.KeysExamined),
+		KeysExamined:                             calcStats(query.KeysExamined),
+		QueryHash:                                query.QueryHash,
+		AppName:                                  query.AppName,
+		Client:                                   query.Client,
+		User:                                     query.User,
+		Comments:                                 query.Comments,
+		LocksGlobalAcquireCountReadSharedCount:   query.LocksGlobalAcquireCountReadSharedCount,
+		LocksGlobalAcquireCountReadShared:        query.LocksGlobalAcquireCountReadShared,
+		LocksGlobalAcquireCountWriteSharedCount:  query.LocksGlobalAcquireCountWriteSharedCount,
+		LocksGlobalAcquireCountWriteShared:       query.LocksGlobalAcquireCountWriteShared,
+		LocksDatabaseAcquireCountReadSharedCount: query.LocksDatabaseAcquireCountReadSharedCount,
+		LocksDatabaseAcquireCountReadShared:      query.LocksDatabaseAcquireCountReadShared,
+		LocksDatabaseAcquireWaitCountReadSharedCount:    query.LocksDatabaseAcquireWaitCountReadSharedCount,
+		LocksDatabaseAcquireWaitCountReadShared:         query.LocksDatabaseAcquireWaitCountReadShared,
+		LocksDatabaseTimeAcquiringMicrosReadSharedCount: len(query.LocksDatabaseTimeAcquiringMicrosReadShared),
+		LocksDatabaseTimeAcquiringMicrosReadShared:      calcStats(query.LocksDatabaseTimeAcquiringMicrosReadShared),
+		LocksCollectionAcquireCountReadSharedCount:      query.LocksCollectionAcquireCountReadSharedCount,
+		LocksCollectionAcquireCountReadShared:           query.LocksCollectionAcquireCountReadShared,
+		StorageBytesReadCount:                           len(query.StorageBytesRead),
+		StorageBytesRead:                                calcStats(query.StorageBytesRead),
+		StorageTimeReadingMicrosCount:                   len(query.StorageTimeReadingMicros),
+		StorageTimeReadingMicros:                        calcStats(query.StorageTimeReadingMicros),
 	}
 	if tc.Returned > 0 {
 		queryStats.Returned.Pct = queryStats.Returned.Total * 100 / tc.Returned
@@ -281,7 +412,22 @@ func countersToStats(query QueryInfoAndCounters, uptime int64, tc totalCounters)
 		queryStats.ResponseLength.Pct = queryStats.ResponseLength.Total * 100 / tc.Bytes
 	}
 	if queryStats.Returned.Total > 0 {
-		queryStats.Ratio = queryStats.Scanned.Total / queryStats.Returned.Total
+		queryStats.Ratio = queryStats.DocsExamined.Total / queryStats.Returned.Total
+	}
+	if tc.DocsExamined > 0 {
+		queryStats.DocsExamined.Pct = queryStats.DocsExamined.Total * 100 / tc.DocsExamined
+	}
+	if tc.KeysExamined > 0 {
+		queryStats.KeysExamined.Pct = queryStats.KeysExamined.Total * 100 / tc.KeysExamined
+	}
+	if tc.LocksDatabaseTimeAcquiringMicrosReadShared > 0 {
+		queryStats.LocksDatabaseTimeAcquiringMicrosReadShared.Pct = queryStats.LocksDatabaseTimeAcquiringMicrosReadShared.Total * 100 / tc.LocksDatabaseTimeAcquiringMicrosReadShared
+	}
+	if tc.StorageBytesRead > 0 {
+		queryStats.StorageBytesRead.Pct = queryStats.StorageBytesRead.Total * 100 / tc.StorageBytesRead
+	}
+	if tc.StorageTimeReadingMicros > 0 {
+		queryStats.StorageTimeReadingMicros.Pct = queryStats.StorageTimeReadingMicros.Total * 100 / tc.StorageTimeReadingMicros
 	}
 
 	return queryStats
@@ -291,10 +437,14 @@ func aggregateCounters(queries []QueryInfoAndCounters) QueryInfoAndCounters {
 	qt := QueryInfoAndCounters{}
 	for _, query := range queries {
 		qt.Count += query.Count
-		qt.NScanned = append(qt.NScanned, query.NScanned...)
 		qt.NReturned = append(qt.NReturned, query.NReturned...)
 		qt.QueryTime = append(qt.QueryTime, query.QueryTime...)
 		qt.ResponseLength = append(qt.ResponseLength, query.ResponseLength...)
+		qt.DocsExamined = append(qt.DocsExamined, query.DocsExamined...)
+		qt.KeysExamined = append(qt.KeysExamined, query.KeysExamined...)
+		qt.LocksDatabaseTimeAcquiringMicrosReadShared = append(qt.LocksDatabaseTimeAcquiringMicrosReadShared, query.LocksDatabaseTimeAcquiringMicrosReadShared...)
+		qt.StorageBytesRead = append(qt.StorageBytesRead, query.StorageBytesRead...)
+		qt.StorageTimeReadingMicros = append(qt.StorageTimeReadingMicros, query.StorageTimeReadingMicros...)
 	}
 	return qt
 }
@@ -305,9 +455,6 @@ func calcTotalCounters(queries []QueryInfoAndCounters) totalCounters {
 	for _, query := range queries {
 		tc.Count += query.Count
 
-		scanned, _ := stats.Sum(query.NScanned)
-		tc.Scanned += scanned
-
 		returned, _ := stats.Sum(query.NReturned)
 		tc.Returned += returned
 
@@ -316,11 +463,30 @@ func calcTotalCounters(queries []QueryInfoAndCounters) totalCounters {
 
 		bytes, _ := stats.Sum(query.ResponseLength)
 		tc.Bytes += bytes
+
+		docsExamined, _ := stats.Sum(query.DocsExamined)
+		tc.DocsExamined += docsExamined
+
+		keysExamined, _ := stats.Sum(query.KeysExamined)
+		tc.KeysExamined += keysExamined
+
+		locksDatabaseTimeAcquiringMicrosReadShared, _ := stats.Sum(query.LocksDatabaseTimeAcquiringMicrosReadShared)
+		tc.LocksDatabaseTimeAcquiringMicrosReadShared += locksDatabaseTimeAcquiringMicrosReadShared
+
+		storageBytesRead, _ := stats.Sum(query.StorageBytesRead)
+		tc.StorageBytesRead += storageBytesRead
+
+		storageTimeReadingMicros, _ := stats.Sum(query.StorageTimeReadingMicros)
+		tc.StorageTimeReadingMicros += storageTimeReadingMicros
 	}
 	return tc
 }
 
 func calcStats(samples []float64) Statistics {
+	if len(samples) == 0 {
+		return Statistics{}
+	}
+
 	var s Statistics
 	s.Total, _ = stats.Sum(samples)
 	s.Min, _ = stats.Min(samples)

src/go/mongolib/stats/stats_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/golang/mock/gomock"
+
 	"github.com/percona/percona-toolkit/src/go/lib/tutil"
 	"github.com/percona/percona-toolkit/src/go/mongolib/fingerprinter"
 	"github.com/percona/percona-toolkit/src/go/mongolib/proto"
@@ -40,8 +41,8 @@ func TestMain(m *testing.M) {
 		log.Printf("cannot get root path: %s", err.Error())
 		os.Exit(1)
 	}
-	// TODO: Review with the new sandbox
-	// os.Exit(m.Run())
+	code := m.Run()
+	os.Exit(code)
 }
 
 func TestTimesLen(t *testing.T) {
@@ -158,9 +159,9 @@ func TestStats(t *testing.T) {
 		BlockedTime:    nil,
 		LockTime:       nil,
 		NReturned:      []float64{0},
-		NScanned:       []float64{10000},
 		QueryTime:      []float64{7},
 		ResponseLength: []float64{215},
+		DocsExamined:   []float64{10000},
 	}
 
 	want := Queries{

src/go/pt-mongodb-query-digest/README.rst

@@ -54,7 +54,7 @@ Options
 
 ``-o``, ``--order-by``
   Specifies the sorting order using fields:
-  ``count``, ``ratio``, ``query-time``, ``docs-scanned``, ``docs-returned``.
+  ``count``, ``ratio``, ``query-time``, ``docs-examined``, ``docs-returned``.
 
   Adding a hyphen (``-``) in front of a field denotes reverse order.
   For example: ``--order-by="count,-ratio"``.
@@ -94,13 +94,13 @@ Output Example
 .. code-block:: none
 
    # Query 3:  0.06 QPS, ID 0b906bd86148def663d11b402f3e41fa
-   # Ratio    1.00  (docs scanned/returned)
+   # Ratio    1.00  (docs examined/returned)
    # Time range: 2017-02-03 16:01:37.484 -0300 ART to 2017-02-03 16:02:08.43 -0300 ART
    # Attribute            pct     total        min         max        avg         95%        stddev      median
    # ==================   ===   ========    ========    ========    ========    ========     =======    ========
    # Count (docs)                   100
    # Exec Time ms           2         3           0           1           0           0           0           0
-   # Docs Scanned           5      7.50K      75.00       75.00       75.00       75.00        0.00       75.00
+   # Docs Examined          5      7.50K      75.00       75.00       75.00       75.00        0.00       75.00
    # Docs Returned         92      7.50K      75.00       75.00       75.00       75.00        0.00       75.00
    # Bytes recv             1    106.12M       1.06M       1.06M       1.06M       1.06M       0.00        1.06M
    # String:

src/go/pt-mongodb-query-digest/main.go

@@ -497,23 +497,23 @@ func sortQueries(queries []stats.QueryStats, orderby []string) []stats.QueryStat
 			}
 
 		//
-		case "docs-scanned":
+		case "docs-examined":
 			f = func(c1, c2 *stats.QueryStats) bool {
-				return c1.Scanned.Max < c2.Scanned.Max
+				return c1.DocsExamined.Max < c2.DocsExamined.Max
 			}
-		case "-docs-scanned":
+		case "-docs-examined":
 			f = func(c1, c2 *stats.QueryStats) bool {
-				return c1.Scanned.Max > c2.Scanned.Max
+				return c1.DocsExamined.Max > c2.DocsExamined.Max
 			}
 
 		//
 		case "docs-returned":
 			f = func(c1, c2 *stats.QueryStats) bool {
-				return c1.Returned.Max < c2.Scanned.Max
+				return c1.Returned.Max < c2.DocsExamined.Max
 			}
 		case "-docs-returned":
 			f = func(c1, c2 *stats.QueryStats) bool {
-				return c1.Returned.Max > c2.Scanned.Max
+				return c1.Returned.Max > c2.DocsExamined.Max
 			}
 		}
 		// count,query-time,docs-scanned, docs-returned. - in front of the field name denotes reverse order.")

src/go/pt-pg-summary/main_test.go

@@ -32,7 +32,8 @@ var logger = logrus.New()
 
 func TestMain(m *testing.M) {
 	logger.SetLevel(logrus.WarnLevel)
-	os.Exit(m.Run())
+	code := m.Run()
+	os.Exit(code)
 }
 
 func TestConnection(t *testing.T) {

src/go/pt-secure-collect/README.rst

@@ -15,7 +15,7 @@ Usage
 
 ::
 
-  pt-secure-data [<flags>] <command> [<args> ...]
+  pt-secure-collect [<flags>] <command> [<args> ...]
 
 By default, :program:`pt-secure-collect` will collect the output of:
 

t/lib/ChangeHandler.t

@@ -7,6 +7,7 @@ BEGIN {
 };
 
 use strict;
+use utf8;
 use warnings FATAL => 'all';
 use English qw(-no_match_vars);
 use Test::More;
@@ -339,6 +340,85 @@ SKIP: {
    );
 };
 
+# #############################################################################
+# PT-2375: pt-table-sync must handle generated columns correctly
+# #############################################################################
+$row = {
+   id  => 1,
+   foo => 'foo',
+   bar => 'bar',
+};
+$tbl_struct = {
+   col_posn => { id=>0, foo=>1, bar=>2 },
+   is_generated => {foo=>1}
+};
+$ch = new ChangeHandler(
+   Quoter     => $q,
+   right_db   => 'test',       # dst
+   right_tbl  => 'pt-2375',
+   left_db    => 'test',       # src
+   left_tbl   => 'pt-2375',
+   actions    => [ sub { push @rows, @_ } ],
+   replace    => 0,
+   queue      => 0,
+   tbl_struct => $tbl_struct,
+);
+
+@rows = ();
+@dbhs = ();
+
+is(
+   $ch->make_INSERT($row, [qw(id foo bar)]),
+   "INSERT INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'bar')",
+   'make_INSERT() omits generated columns'
+);
+
+is(
+   $ch->make_REPLACE($row, [qw(id foo bar)]),
+   "REPLACE INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'bar')",
+   'make_REPLACE() omits generated columns'
+);
+
+is(
+   $ch->make_UPDATE($row, [qw(id foo)]),
+   "UPDATE `test`.`pt-2375` SET `bar`='bar' WHERE `id`='1' AND `foo`='foo' LIMIT 1",
+   'make_UPDATE() omits generated columns from SET phrase but includes in WHERE phrase'
+);
+
+is(
+   $ch->make_DELETE($row, [qw(id foo bar)]),
+   "DELETE FROM `test`.`pt-2375` WHERE `id`='1' AND `foo`='foo' AND `bar`='bar' LIMIT 1",
+   'make_DELETE() includes generated columns in WHERE phrase'
+);
+
+SKIP: {
+   skip 'Cannot connect to sandbox master', 3 unless $master_dbh;
+
+   $master_dbh->do('DROP TABLE IF EXISTS test.`pt-2375`');
+   $master_dbh->do('CREATE TABLE test.`pt-2375` (id INT, foo varchar(16) as ("foo"), bar char)');
+   $master_dbh->do("INSERT INTO test.`pt-2375` (`id`, `bar`) VALUES (1,'a'),(2,'b')");
+
+   $ch->fetch_back($master_dbh);
+
+   is(
+      $ch->make_INSERT($row, [qw(id foo)]),
+      "INSERT INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'a')",
+      'make_INSERT() omits generated columns, with fetch-back'
+   );
+
+   is(
+      $ch->make_REPLACE($row, [qw(id foo)]),
+      "REPLACE INTO `test`.`pt-2375`(`id`, `bar`) VALUES ('1', 'a')",
+      'make_REPLACE() omits generated columns, with fetch-back'
+   );
+
+   is(
+      $ch->make_UPDATE($row, [qw(id foo)]),
+      "UPDATE `test`.`pt-2375` SET `bar`='a' WHERE `id`='1' AND `foo`='foo' LIMIT 1",
+      'make_UPDATE() omits generated columns from SET phrase, with fetch-back'
+   );
+};
+
 # #############################################################################
 # Issue 641: Make mk-table-sync use hex for binary/blob data
 # #############################################################################
@@ -538,6 +618,52 @@ SKIP: {
    );
 }
 
+# #############################################################################
+# PT-2377: pt-table-sync must handle utf8 in JSON columns correctly
+# #############################################################################
+SKIP: {
+   skip 'Cannot connect to sandbox master', 1 unless $master_dbh;
+   $master_dbh->do('DROP TABLE IF EXISTS `test`.`pt-2377`');
+   $master_dbh->do('CREATE TABLE `test`.`pt-2377` (id INT, data JSON)');
+   $master_dbh->do(q/INSERT INTO `test`.`pt-2377` VALUES (1, '{"name": "Müller"}')/);
+   $master_dbh->do(q/INSERT INTO `test`.`pt-2377` VALUES (2, NULL)/);
+
+   @rows = ();
+   $tbl_struct = {
+      cols      => [qw(id data)],
+      col_posn  => {id=>0, data=>1},
+      type_for  => {id=>'int', data=>'json'},
+   };
+   $ch = new ChangeHandler(
+      Quoter     => $q,
+      left_db    => 'test',
+      left_tbl   => 'pt-2377',
+      right_db   => 'test',
+      right_tbl  => 'pt-2377',
+      actions    => [ sub { push @rows, $_[0]; } ],
+      replace    => 0,
+      queue      => 0,
+      tbl_struct => $tbl_struct,
+   );
+   $ch->fetch_back($master_dbh);
+
+   $ch->change('UPDATE', {id=>1}, [qw(id)] );
+   $ch->change('INSERT', {id=>1}, [qw(id)] );
+   $ch->change('UPDATE', {id=>2}, [qw(id)] );
+   $ch->change('INSERT', {id=>2}, [qw(id)] );
+
+   is_deeply(
+      \@rows,
+      [
+         q/UPDATE `test`.`pt-2377` SET `data`='{"name": "Müller"}' WHERE `id`='1' LIMIT 1/,
+         q/INSERT INTO `test`.`pt-2377`(`id`, `data`) VALUES ('1', '{"name": "Müller"}')/,
+         q/UPDATE `test`.`pt-2377` SET `data`=NULL WHERE `id`='2' LIMIT 1/,
+         q/INSERT INTO `test`.`pt-2377`(`id`, `data`) VALUES ('2', NULL)/
+      ],
+      "UPDATE and INSERT quote data regardless of how it looks if tbl_struct->quote_val is true"
+   );
+}
+
 # #############################################################################
 # Done.
 # #############################################################################

t/lib/Quoter.t

@@ -71,6 +71,8 @@ is( $q->quote_val('0x89504E470', is_char => 0), '0x89504E470', 'hex string, with
 is( $q->quote_val('0x89504E470', is_char => 1), "'0x89504E470'", 'hex string, with is_char => 1');
 is( $q->quote_val('0x89504I470'), "'0x89504I470'", 'looks like hex string');
 is( $q->quote_val('eastside0x3'), "'eastside0x3'", 'looks like hex str (issue 1110');
+is( $q->quote_val(969.1 / 360, is_float => 1), "2.6919444444444447", 'float has full precision');
+is( $q->quote_val(0.1, is_float => 1), "0.1", 'full precision for float only used when required');
 
 # Splitting DB and tbl apart
 is_deeply(

t/pt-archiver/file.t

@@ -123,11 +123,11 @@ $output = output(
 );
 $output = `cat archive.test.table_2`;
 is($output, <<EOF
-1, 2, 3, 4
-2, "\\N", 3, 4
-3, 2, 3, "\\\t"
-4, 2, 3, "\\\n"
-5, 2, 3, "Zapp \\"Brannigan"
+1,2,3,4
+2,\\N,3,4
+3,2,3,"\\\t"
+4,2,3,"\\\n"
+5,2,3,"Zapp \\"Brannigan"
 EOF
 , '--output-format=csv');
 `rm -f archive.test.table_2`;

t/pt-archiver/pt-2410.t

@@ -0,0 +1,75 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use charnames ':full';
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-archiver";
+
+my $dp  = new DSNParser(opts=>$dsn_opts);
+my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $dbh = $sb->get_dbh_for('source');
+
+if ( !$dbh ) {
+   plan skip_all => 'Cannot connect to sandbox source';
+}
+
+my $output;
+my $exit_status;
+my $cnf = "/tmp/12345/my.sandbox.cnf";
+my $cmd = "$trunk/bin/pt-archiver";
+
+$sb->wipe_clean($dbh);
+$sb->create_dbs($dbh, ['test']);
+
+$sb->load_file('source', 't/pt-archiver/samples/pt-2410.sql');
+
+($output, $exit_status) = full_output(
+   sub { pt_archiver::main(
+      qw(--where 1=1 --output-format=csv),
+      '--source', "L=1,D=pt_2410,t=test,F=$cnf",
+      '--file',   '/tmp/pt-2410.csv') },
+);
+
+is(
+   $exit_status,
+   0,
+   'pt-archiver comleted'
+);
+
+$output = `cat /tmp/pt-2410.csv`;
+like(
+   $output,
+   qr/1,\\N,"testing..."/,
+   'NULL values stored correctly'
+) or diag($output);
+
+$dbh->do("load data local infile '/tmp/pt-2410.csv' into table pt_2410.test COLUMNS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'");
+
+$output = `/tmp/12345/use pt_2410 -N -e 'SELECT * FROM test'`;
+
+like(
+   $output,
+   qr/1	NULL	testing.../,
+   'NULL values loaded correctly'
+) or diag($output);
+
+# #############################################################################
+# Done.
+# #############################################################################
+diag(`rm -f /tmp/pt-2410.csv`);
+$sb->wipe_clean($dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;
+exit;

t/pt-archiver/samples/pt-2410.sql

@@ -0,0 +1,10 @@
+CREATE DATABASE pt_2410;
+USE pt_2410;
+
+CREATE TABLE test(
+    id int not null primary key auto_increment,
+    column1 int default null,
+    column2 varchar(50) not null);
+
+INSERT INTO test VALUES (null,null,'testing...');
+INSERT INTO test VALUES (null,null,'testing...');

t/pt-online-schema-change/pt-1717-resume.t

@@ -137,7 +137,7 @@ set_delay();
 # We need to sleep, otherwise pt-osc can finish before replica is delayed
 sleep($max_lag);
 
-my $args = "$source_dsn,D=test,t=pt1717 --execute --chunk-size ${chunk_size} --max-lag $max_lag --alter 'engine=INNODB' --pid $tmp_file_name --progress time,5 --no-drop-new-table --no-drop-triggers --history";
+my $args = "$source_dsn,D=test,t=pt1717 --execute --chunk-size ${chunk_size} --max-lag $max_lag --alter 'ADD COLUMN foo varchar(32)' --pid $tmp_file_name --progress time,5 --no-drop-new-table --no-drop-triggers --history";
 
 $output = run_broken_job($args);
 
@@ -165,7 +165,7 @@ my @args = (qw(--execute --chunk-size=10 --history));
 
 ($output, $exit) = full_output(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
-         '--alter', 'engine=INNODB', '--execute', "--resume=${job_id}",
+         '--alter', 'ADD COLUMN foo varchar(32)', '--execute', "--resume=${job_id}",
          '--chunk-index=f2'
          ) }
 );
@@ -186,7 +186,7 @@ like(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
          '--max-lag', $max_lag,
          '--resume', $job_id,
-         '--alter', 'engine=INNODB',
+         '--alter', 'ADD COLUMN foo varchar(32)',
          '--plugin', "$plugin/pt-1717.pm",
          ),
       },
@@ -208,8 +208,10 @@ ok(
    'All rows copied correctly'
 ) or diag("New table checksum: '${new_table_checksum}', original content checksum: '${old_table_checksum}'");
 
+diag(`/tmp/12345/use test -N -e "ALTER TABLE pt1717 DROP COLUMN foo"`);
+
 # Tests for chunk-index and chunk-index-columns options
-$args = "$source_dsn,D=test,t=pt1717 --alter engine=innodb --execute --history --chunk-size=10 --no-drop-new-table --no-drop-triggers --reverse-triggers --chunk-index=f2";
+$args = "$source_dsn,D=test,t=pt1717 --alter 'ADD COLUMN foo varchar(32)' --execute --history --chunk-size=10 --no-drop-new-table --no-drop-triggers --reverse-triggers --chunk-index=f2";
 
 set_delay();
 $output = run_broken_job($args);
@@ -220,7 +222,7 @@ $job_id = $1;
 
 ($output, $exit) = full_output(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
-         '--alter', 'engine=innodb', '--execute', "--resume=${job_id}",
+         '--alter', 'ADD COLUMN foo varchar(32)', '--execute', "--resume=${job_id}",
          ) }
 );
 
@@ -238,7 +240,7 @@ like(
 
 ($output, $exit) = full_output(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
-         '--alter', 'engine=innodb', '--execute', "--resume=${job_id}",
+         '--alter', 'ADD COLUMN foo varchar(32)', '--execute', "--resume=${job_id}",
          '--chunk-index=f1'
          ) }
 );
@@ -257,7 +259,7 @@ like(
 
 ($output, $exit) = full_output(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
-         '--alter', 'engine=innodb', '--execute', "--resume=${job_id}",
+         '--alter', 'ADD COLUMN foo varchar(32)', '--execute', "--resume=${job_id}",
          '--chunk-index=f2', '--chunk-index-columns=1'
          ) }
 );
@@ -288,7 +290,7 @@ is(
    $output + 0,
    3,
    'Triggers were not dropped'
-);
+) or diag($output);
 
 $output = `/tmp/12345/use -N -e "select count(*) from information_schema.triggers where TRIGGER_SCHEMA='test' AND EVENT_OBJECT_TABLE like '%pt1717%_new' AND trigger_name LIKE 'rt_%'"`;
 
@@ -300,7 +302,7 @@ is(
 
 ($output, $exit) = full_output(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
-         '--alter', 'engine=innodb', '--execute', "--resume=${job_id}",
+         '--alter', 'ADD COLUMN foo varchar(32)', '--execute', "--resume=${job_id}",
          '--chunk-size=4',
          '--chunk-index=f2'
       ) }
@@ -348,7 +350,7 @@ ok(
 
 ($output, $exit) = full_output(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
-         '--alter', 'engine=innodb', '--execute', "--resume=${job_id}",
+         '--alter', 'ADD COLUMN foo varchar(32)', '--execute', "--resume=${job_id}",
          '--chunk-size=4',
          '--chunk-index=f2'
       ) }
@@ -372,7 +374,7 @@ $output =~ /New table `test`.`([_]+pt1717_new)` not found, restart operation fro
 
 ($output, $exit) = full_output(
    sub { pt_online_schema_change::main(@args, "$source_dsn,D=test,t=pt1717",
-         '--alter', 'engine=innodb', '--execute', "--resume=${job_id}",
+         '--alter', 'ADD COLUMN foo varchar(32)', '--execute', "--resume=${job_id}",
          '--chunk-size=4',
          '--chunk-index=f2'
       ) }

t/pt-online-schema-change/pt-2355.t

@@ -0,0 +1,239 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use threads;
+
+use English qw(-no_match_vars);
+use Test::More;
+
+use Data::Dumper;
+use PerconaTest;
+use Sandbox;
+use SqlModes;
+use File::Temp qw/ tempdir tempfile /;
+
+our $delay = 10;
+my $max_lag = $delay / 2;
+my $output;
+my $exit;
+
+my $tmp_file = File::Temp->new();
+my $tmp_file_name = $tmp_file->filename;
+unlink $tmp_file_name;
+
+require "$trunk/bin/pt-online-schema-change";
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+if ($sb->is_cluster_mode) {
+    plan skip_all => 'Not for PXC';
+}
+
+my $source_dbh = $sb->get_dbh_for('source');
+my $replica_dbh1 = $sb->get_dbh_for('replica1');
+my $replica_dbh2 = $sb->get_dbh_for('replica2');
+my $source_dsn = 'h=127.0.0.1,P=12345,u=msandbox,p=msandbox';
+my $replica_dsn1 = 'h=127.0.0.1,P=12346,u=msandbox,p=msandbox';
+my $replica_dsn2 = 'h=127.0.0.1,P=12347,u=msandbox,p=msandbox';
+my $sample     = "t/pt-online-schema-change/samples";
+my $plugin     = "$trunk/$sample/plugins";
+
+# We need sync_relay_log=1 to keep changes after replica restart 
+my $cnf = '/tmp/12347/my.sandbox.cnf';
+diag(`cp $cnf $cnf.bak`);
+diag(`echo "[mysqld]" > /tmp/12347/my.sandbox.2.cnf`);
+diag(`echo "sync_relay_log=1" >> /tmp/12347/my.sandbox.2.cnf`);
+diag(`echo "sync_relay_log_info=1" >> /tmp/12347/my.sandbox.2.cnf`);
+diag(`echo "relay_log_recovery=1" >> /tmp/12347/my.sandbox.2.cnf`);
+diag(`echo "!include /tmp/12347/my.sandbox.2.cnf" >> $cnf`);
+diag(`/tmp/12347/stop >/dev/null`);
+sleep 1;
+diag(`/tmp/12347/start >/dev/null`);
+
+sub reset_query_cache {
+    my @dbhs = @_;
+    return if ($sandbox_version ge '8.0');
+    foreach my $dbh (@dbhs) {
+        $dbh->do('RESET QUERY CACHE');
+    }
+}
+
+sub run_broken_job {
+   my ($args) = @_;
+   my ($fh, $filename) = tempfile();
+   my $pid = fork();
+
+   if (!$pid) {
+      open(STDERR, '>', $filename);
+      open(STDOUT, '>', $filename);
+      exec("$trunk/bin/pt-online-schema-change $args");
+   }
+
+   sleep($max_lag + $max_lag/2);
+   # stop replica 12347
+   diag(`/tmp/12347/stop >/dev/null`);
+   sleep 1;
+
+   waitpid($pid, 0);
+   my $output = do {
+      local $/ = undef;
+      <$fh>;
+   };
+
+   return $output;
+}
+
+sub set_delay {
+   $sb->wait_for_replicas();
+
+   diag("Setting replica delay to $delay seconds");
+   diag(`/tmp/12345/use -N test -e "DROP TABLE IF EXISTS pt1717_back"`);
+
+   $replica_dbh1->do("STOP ${replica_name}");
+   $replica_dbh1->do("CHANGE ${source_change} TO ${source_name}_DELAY=$delay");
+   $replica_dbh1->do("START ${replica_name}");
+
+   # Run a full table scan query to ensure the replica is behind the source
+   # There is no query cache in MySQL 8.0+
+   reset_query_cache($source_dbh, $source_dbh);
+   # Update one row so replica is delayed
+   $source_dbh->do('UPDATE `test`.`pt1717` SET f2 = f2 + 1 LIMIT 1');
+   $source_dbh->do('UPDATE `test`.`pt1717` SET f2 = f2 + 1 WHERE f1 = ""');
+
+   # Creating copy of table pt1717, so we can compare data later
+   diag(`/tmp/12345/use -N test -e "CREATE TABLE pt1717_back like pt1717"`);
+   diag(`/tmp/12345/use -N test -e "INSERT INTO pt1717_back SELECT * FROM pt1717"`);
+}
+
+# 1) Set the replica delay to 0 just in case we are re-running the tests without restarting the sandbox.
+# 2) Load sample data
+# 3) Set the replica delay to 30 seconds to be able to see the 'waiting' message.
+diag("Setting replica delay to 0 seconds");
+$replica_dbh1->do("STOP ${replica_name}");
+$source_dbh->do("RESET ${source_reset}");
+$replica_dbh1->do("RESET ${replica_name}");
+$replica_dbh1->do("START ${replica_name}");
+
+diag('Loading test data');
+$sb->load_file('source', "t/pt-online-schema-change/samples/pt-1717.sql");
+
+# Should be greater than chunk-size and big enough, so pt-osc will wait for delay
+my $num_rows = 5000;
+my $chunk_size = 10;
+diag("Loading $num_rows into the table. This might take some time.");
+diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test pt1717 $num_rows`);
+
+diag("Starting tests...");
+
+set_delay();
+
+# We need to sleep, otherwise pt-osc can finish before replica is delayed
+sleep($max_lag);
+
+my $args = "$source_dsn,D=test,t=pt1717 --execute --chunk-size ${chunk_size} --max-lag $max_lag --alter 'ADD INDEX idx1(f1)' --pid $tmp_file_name --progress time,5 --no-drop-new-table --no-drop-triggers --history";
+
+$output = run_broken_job($args);
+
+like(
+   $output,
+   qr/`test`.`pt1717` was not altered/s,
+   "pt-osc stopped with error as expected",
+) or diag($output);
+
+diag(`/tmp/12347/start >/dev/null`);
+$sb->wait_for_replicas();
+
+$output = `/tmp/12345/use -N -e "select job_id, upper_boundary from percona.pt_osc_history"`;
+my ($job_id, $upper_boundary) = split(/\s+/, $output);
+
+my $copied_rows = `/tmp/12345/use -N -e "select count(*) from test._pt1717_new"`;
+chomp($copied_rows);
+
+ok(
+   $copied_rows eq $upper_boundary,
+   'Upper chunk boundary stored correctly'
+) or diag("Copied_rows: ${copied_rows}, upper boundary: ${upper_boundary}");;
+
+($output, $exit) = full_output(
+   sub { pt_online_schema_change::main("$source_dsn,D=test,t=pt1717",
+         "--execute", "--chunk-size=${chunk_size}", "--max-lag=${max_lag}",
+         "--alter=ADD INDEX idx1(f1)",
+         "--resume=${job_id}",
+         ) }
+);
+
+is(
+   $exit,
+   0,
+   'pt-osc works correctly with --resume'
+) or diag($exit);
+
+like(
+   $output,
+   qr/Successfully altered/,
+   'Success message printed'
+) or diag($output);
+
+# Corrupting job record, so we can test error message
+diag(`/tmp/12345/use -N -e "update percona.pt_osc_history set new_table_name=NULL where job_id=${job_id}"`);
+
+($output, $exit) = full_output(
+   sub { pt_online_schema_change::main("$source_dsn,D=test,t=pt1717",
+         "--execute", "--chunk-size=${chunk_size}", "--max-lag=${max_lag}",
+         "--alter=ADD INDEX idx1(f1)",
+         "--resume=${job_id}",
+         ) }
+);
+
+is(
+   $exit,
+   17,
+   'pt-osc works correctly fails with empty boundaries'
+) or diag($exit);
+
+like(
+   $output,
+   qr/Option --resume refers job \d+ with empty boundaries. Exiting./,
+   'Correct error message printed'
+) or diag($output);
+
+unlike(
+   $output,
+   qr/Option --resume refers non-existing job ID: \d+. Exiting./,
+   'Misleading error message not printed'
+) or diag($output);
+
+# #############################################################################
+# Done.
+# #############################################################################
+diag("Cleaning");
+$replica_dbh2 = $sb->get_dbh_for('replica2');
+diag("Setting replica delay to 0 seconds");
+$replica_dbh1->do("STOP ${replica_name}");
+$replica_dbh2->do("STOP ${replica_name}");
+$source_dbh->do("RESET ${source_reset}");
+$replica_dbh1->do("RESET ${source_reset}");
+$replica_dbh1->do("RESET ${replica_name}");
+$replica_dbh2->do("RESET ${replica_name}");
+$replica_dbh1->do("START ${replica_name}");
+$replica_dbh2->do("START ${replica_name}");
+
+diag(`mv $cnf.bak $cnf`);
+
+diag(`/tmp/12347/stop >/dev/null`);
+diag(`/tmp/12347/start >/dev/null`);
+
+diag("Dropping test database");
+$source_dbh->do("DROP DATABASE IF EXISTS test");
+$sb->wait_for_replicas();
+
+$sb->wipe_clean($source_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;

t/pt-online-schema-change/pt-2407.t

@@ -0,0 +1,71 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-online-schema-change";
+require VersionParser;
+
+use Data::Dumper;
+
+my $dp         = new DSNParser(opts=>$dsn_opts);
+my $sb         = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $source_dbh = $sb->get_dbh_for('source');
+my $replica_dbh  = $sb->get_dbh_for('replica1');
+
+if ( !$source_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox source';
+}
+elsif ( !$replica_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox replica';
+}
+
+my @args   = qw(--set-vars innodb_lock_wait_timeout=3);
+my $output = "";
+my $dsn    = "h=127.1,P=12345,u=msandbox,p=msandbox";
+my $exit   = 0;
+my $sample = "t/pt-online-schema-change/samples";
+
+$sb->load_file('source', "$sample/pt-2407.sql");
+
+($output, $exit) = full_output(
+   sub { pt_online_schema_change::main(@args, "$dsn,D=pt_2407,t=t1",
+         '--alter', 'alter table t1 ADD COLUMN payout_group_id VARCHAR(255) DEFAULT NULL, ALGORITHM=INSTANT;', '--execute') }
+);
+
+is(
+   $exit,
+   11,
+   'Return code non-zero for failed operation'
+) or diag($exit);
+
+like(
+   $output,
+   qr/You have an error in your SQL syntax/,
+   'Job failed due to SQL syntax error'
+) or diag($output);
+
+like(
+   $output,
+   qr/Error altering new table/,
+   'Error altering new table message printed'
+) or diag($output);
+
+# #############################################################################
+# Done.
+# #############################################################################
+
+$sb->wipe_clean($source_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+#
+done_testing;

t/pt-online-schema-change/pt-2422.t

@@ -0,0 +1,120 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-online-schema-change";
+require VersionParser;
+
+use Data::Dumper;
+
+my $dp         = new DSNParser(opts=>$dsn_opts);
+my $sb         = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $source_dbh = $sb->get_dbh_for('source');
+my $replica_dbh  = $sb->get_dbh_for('replica1');
+
+if ( !$source_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox source';
+}
+elsif ( !$replica_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox replica';
+}
+
+my @args   = qw(--set-vars innodb_lock_wait_timeout=3);
+my $output = "";
+my $dsn    = "h=127.1,P=12345,u=msandbox,p=msandbox";
+my $exit   = 0;
+my $sample = "t/pt-online-schema-change/samples";
+
+$sb->load_file('source', "$sample/basic_no_fks_innodb.sql");
+$source_dbh->do('CREATE TABLE pt_osc.pt_2422 LIKE pt_osc.t');
+$source_dbh->do('INSERT INTO pt_osc.pt_2422 SELECT * FROM pt_osc.t');
+
+($output, $exit) = full_output(
+   sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=t",
+         '--alter', 'engine=innodb', '--execute', '--history') }
+);
+
+is(
+   $exit,
+   0,
+   'basic test with option --history finished OK'
+) or diag($output);
+
+like(
+   $output,
+   qr/Job \d started/,
+   'Job id printed in the beginning of the tool output'
+);
+
+like(
+   $output,
+   qr/Job \d finished successfully/,
+   'Job id printed for successful copy'
+);
+
+$output = `/tmp/12345/use -N -e "SELECT new_table_name FROM percona.pt_osc_history WHERE job_id=1"`;
+
+like(
+   $output,
+   qr/_t_new/,
+   'Correct new table name inserted'
+) or diag($output);
+
+($output, $exit) = full_output(
+   sub { pt_online_schema_change::main(@args, "$dsn,D=pt_osc,t=pt_2422",
+         '--alter', 'engine=innodb', '--execute', '--history') }
+);
+
+is(
+   $exit,
+   0,
+   'basic test with second table and option --history finished OK'
+) or diag($output);
+
+like(
+   $output,
+   qr/Job \d started/,
+   'Job id printed in the beginning of the tool output for the second table'
+);
+
+like(
+   $output,
+   qr/Job \d finished successfully/,
+   'Job id printed for successful copy of the second table'
+);
+
+$output = `/tmp/12345/use -N -e "SELECT new_table_name FROM percona.pt_osc_history WHERE job_id=1"`;
+
+like(
+   $output,
+   qr/_t_new/,
+   'New table name for previouse job was not updated'
+) or diag($output);
+
+$output = `/tmp/12345/use -N -e "SELECT new_table_name FROM percona.pt_osc_history WHERE job_id=2"`;
+
+like(
+   $output,
+   qr/_pt_2422_new/,
+   'Correct new table name inserted for the second table'
+) or diag($output);
+
+# #############################################################################
+# Done.
+# #############################################################################
+
+$sb->wipe_clean($source_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+#
+done_testing;

t/pt-online-schema-change/samples/pt-2407.sql

@@ -0,0 +1,12 @@
+CREATE DATABASE pt_2407;
+
+USE pt_2407;
+
+CREATE TABLE t1 (
+    c1 int NOT NULL,
+    c2 varchar(100) NOT NULL,
+    PRIMARY KEY (c1),
+    KEY idx (c2)
+) ENGINE=InnoDB;
+
+INSERT INTO t1 VALUES(1,1),(2,2),(3,3),(4,4),(5,5);

t/pt-table-checksum/pt-2400.t

@@ -0,0 +1,70 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+use Data::Dumper;
+
+# Hostnames make testing less accurate.  Tests need to see
+# that such-and-such happened on specific replica hosts, but
+# the sandbox servers are all on one host so all replicas have
+# the same hostname.
+$ENV{PERCONA_TOOLKIT_TEST_USE_DSN_NAMES} = 1;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-table-checksum";
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $node1 = $sb->get_dbh_for('node1');
+my $sb_version = VersionParser->new($node1);
+my $node2 = $sb->get_dbh_for('node2');
+my $node3 = $sb->get_dbh_for('node3');
+
+my %checks = (
+    'Cannot connect to cluster node1' => !$node1,
+    'Cannot connect to cluster node2' => !$node2,
+    'Cannot connect to cluster node3' => !$node3,
+    'PXC tests'                       => !$sb->is_cluster_mode,
+);
+
+for my $message (keys %checks) {
+    if ( $checks{$message} ) {
+        plan skip_all => $message;
+    }
+}
+
+my $node1_dsn = $sb->dsn_for('node1');
+my @args      = ($node1_dsn, qw(--databases pt-2400 --tables apple),
+                             qw(--recursion-method none),
+                             qw(--replicate percona.checksums --create-replicate-table --empty-replicate-table )
+                );
+my $sample  = "t/pt-table-checksum/samples/";
+
+$sb->load_file('node1', "$sample/pt-2400.sql");
+
+my ($output, $error) = full_output(
+   sub { pt_table_checksum::main(@args) },
+   stderr => 1,
+);
+
+unlike(
+   $output,
+   qr/Immediately starting the version comment after the version number is deprecated and may change behavior in a future release. Please insert a white-space character after the version number./,
+   "No typo in version comment"
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($node1);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;

t/pt-table-checksum/samples/pt-2400.sql

@@ -0,0 +1,13 @@
+DROP DATABASE IF EXISTS `pt-2400`;
+CREATE DATABASE `pt-2400`;
+USE `pt-2400`;
+CREATE TABLE `apple` (
+  `id` int NOT NULL,
+  `name` varchar(255) NOT NULL,
+  PRIMARY KEY (`id`,`name`)
+) ENGINE=InnoDB;
+
+INSERT INTO `apple` VALUES
+  (1, 'Granny Smith'),
+  (2, 'Red Delicious'),
+  (3, 'Golden Apple');

t/pt-table-sync/float_precision.t

@@ -63,6 +63,23 @@ is(
    '--float-precision so no more diff (issue 410)'
 );
 
+# Although the SQL statement contains serialized values with more than necessary decimal digits
+# we produce the expected value on execution
+$output = `$trunk/bin/pt-table-sync --sync-to-source h=127.1,P=12346,u=msandbox,p=msandbox,D=test,t=fl --execute 2>&1`;
+is(
+   $output,
+   '',
+   'REPLACE statement can be successfully applied'
+);
+
+$sb->wait_for_replicas();
+my @rows = $replica_dbh->selectrow_array('SELECT `d` FROM `test`.`fl` WHERE `d` = 2.0000012');
+is_deeply(
+   \@rows,
+   [2.0000012],
+   'Floating point values are set correctly in round trip'
+);
+
 # #############################################################################
 # pt-table-sync quotes floats, prevents syncing
 # https://bugs.launchpad.net/percona-toolkit/+bug/1229861
@@ -86,7 +103,6 @@ is_deeply(
    [],
    "Sync rows with float values (bug 1229861)"
 ) or diag(Dumper($rows), $output);
-
 # #############################################################################
 # Done.
 # #############################################################################

t/pt-table-sync/pt-2375.t

@@ -0,0 +1,80 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-table-sync";
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $source_dbh = $sb->get_dbh_for('source');
+my $replica1_dbh = $sb->get_dbh_for('replica1');
+
+if ( !$source_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox source';
+}
+elsif ( !$replica1_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox replica1';
+}
+else {
+   plan tests => 3;
+}
+
+my $output;
+
+# #############################################################################
+# Test generated REPLACE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2375.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do("delete from `test`.`test_table` where `id`=1");
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main('--sync-to-source',
+      'h=127.0.0.1,P=12346,u=msandbox,p=msandbox',
+      qw(-t test.test_table --print --execute))
+   },
+));
+chomp($output);
+is(
+   $output,
+   "REPLACE INTO `test`.`test_table`(`id`, `value`) VALUES ('1', '24');",
+   "Generated columns are not used in REPLACE statements"
+);
+
+# #############################################################################
+# Test generated UPDATE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2375.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do("update `test`.`test_table` set `value`=55 where `id`=2");
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main(qw(--print --execute),
+      "h=127.0.0.1,P=12346,u=msandbox,p=msandbox,D=test,t=test_table",
+      "h=127.0.0.1,P=12345,u=msandbox,p=msandbox,D=test,t=test_table");
+   }
+));
+chomp($output);
+is(
+   $output,
+   "UPDATE `test`.`test_table` SET `value`='55' WHERE `id`='2' LIMIT 1;",
+   "Generated columns are not used in UPDATE statements"
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($source_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;

t/pt-table-sync/pt-2377.t

@@ -0,0 +1,83 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-table-sync";
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $source_dbh = $sb->get_dbh_for('source');
+my $replica1_dbh = $sb->get_dbh_for('replica1');
+
+if ( !$source_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox source';
+}
+elsif ( !$replica1_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox replica1';
+}
+elsif ( $sandbox_version lt '8.0') {
+   plan skip_all => 'Requires MySQL >= 8.0';
+}
+else {
+   plan tests => 3;
+}
+
+my $output;
+
+# #############################################################################
+# Test generated REPLACE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2377.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do("delete from `test`.`test_table` where `id`=1");
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main('--sync-to-source',
+      'h=127.0.0.1,P=12346,u=msandbox,p=msandbox',
+      qw(-t test.test_table --print --execute))
+   },
+));
+chomp($output);
+is(
+   $output,
+   q/REPLACE INTO `test`.`test_table`(`id`, `data`) VALUES ('1', '{"name": "Müller"}');/,
+   "UTF8 characters of JSON values are printed correctly in REPLACE statements"
+);
+
+# #############################################################################
+# Test generated UPDATE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2377.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do(q/update `test`.`test_table` set `data`='{"reaction": "哈哈哈"}' where `id`=2/);
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main(qw(--print --execute),
+      "h=127.0.0.1,P=12346,u=msandbox,p=msandbox,D=test,t=test_table",
+      "h=127.0.0.1,P=12345,u=msandbox,p=msandbox,D=test,t=test_table");
+   }
+));
+chomp($output);
+is(
+   $output,
+   q/UPDATE `test`.`test_table` SET `data`='{"reaction": "哈哈哈"}' WHERE `id`='2' LIMIT 1;/,
+   "UTF8 characters of JSON values are printed correctly in UPDATE statements"
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($source_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;

t/pt-table-sync/pt-2378.t

@@ -0,0 +1,96 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-table-sync";
+
+my $dp = new DSNParser(opts=>$dsn_opts);
+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $source_dbh = $sb->get_dbh_for('source');
+my $replica1_dbh = $sb->get_dbh_for('replica1');
+
+if ( !$source_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox source';
+}
+elsif ( !$replica1_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox replica1';
+}
+else {
+   plan tests => 5;
+}
+
+my ($output, @rows);
+
+# #############################################################################
+# Test generated REPLACE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2378.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do("update `test`.`test_table` set `some_string` = 'c' where `id` = 1");
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main('--sync-to-source',
+      'h=127.0.0.1,P=12346,u=msandbox,p=msandbox',
+      qw(-t test.test_table --print --execute))
+   },
+));
+chomp($output);
+is(
+   $output,
+   "REPLACE INTO `test`.`test_table`(`id`, `value1`, `value2`, `some_string`) VALUES ('1', 315.25999999999942, 2.6919444444444447, 'a');",
+   "Floating point numbers are generated with sufficient precision in REPLACE statements"
+);
+
+$sb->wait_for_replicas();
+my $query = 'SELECT * FROM `test`.`test_table` WHERE `value1` = 315.2599999999994 AND `value2` = 2.6919444444444447';
+@rows = $replica1_dbh->selectrow_array($query);
+is_deeply(
+   \@rows,
+   [1, 315.2599999999994, 2.6919444444444447, 'a'],
+   'Floating point values are set correctly in round trip'
+);
+
+# #############################################################################
+# Test generated UPDATE statements.
+# #############################################################################
+$sb->load_file('source', "t/pt-table-sync/samples/pt-2378.sql");
+$sb->wait_for_replicas();
+$replica1_dbh->do("update `test`.`test_table` set `some_string` = 'c' where `id` = 1");
+
+$output = remove_traces(output(
+   sub { pt_table_sync::main(qw(--print --execute),
+      "h=127.0.0.1,P=12346,u=msandbox,p=msandbox,D=test,t=test_table",
+      "h=127.0.0.1,P=12345,u=msandbox,p=msandbox,D=test,t=test_table");
+   }
+));
+chomp($output);
+is(
+   $output,
+   "UPDATE `test`.`test_table` SET `value1`=315.25999999999942, `value2`=2.6919444444444447, `some_string`='c' WHERE `id`='1' LIMIT 1;",
+   "Floating point numbers are generated with sufficient precision in UPDATE statements"
+);
+
+@rows = $source_dbh->selectrow_array($query);
+is_deeply(
+   \@rows,
+   [1, 315.2599999999994, 2.6919444444444447, 'c'],
+   'Floating point values are set correctly in round trip'
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+$sb->wipe_clean($source_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+exit;

t/pt-table-sync/samples/pt-2375.sql

@@ -0,0 +1,12 @@
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE test;
+USE test;
+
+CREATE TABLE `test_table` (
+  `id` INT AUTO_INCREMENT PRIMARY KEY,
+  `value` INT NOT NULL,
+  `derived_value` INT AS (2*`value`)
+) ENGINE=InnoDB;
+
+INSERT INTO `test_table` (`value`) VALUES (24);
+INSERT INTO `test_table` (`value`) VALUES (42);

t/pt-table-sync/samples/pt-2377.sql

@@ -0,0 +1,14 @@
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE test;
+USE test;
+
+CREATE TABLE `test_table` (
+  `id` INT AUTO_INCREMENT PRIMARY KEY,
+  `data` JSON NOT NULL
+) ENGINE=InnoDB;
+
+INSERT INTO
+  `test_table` (`data`)
+VALUES
+  ('{"name": "Müller"}'),
+  ('{"reaction": "哈哈"}');

t/pt-table-sync/samples/pt-2378.sql

@@ -0,0 +1,15 @@
+DROP DATABASE IF EXISTS test;
+CREATE DATABASE test;
+USE test;
+
+CREATE TABLE `test_table` (
+  `id` BIGINT AUTO_INCREMENT PRIMARY KEY,
+  `value1` DOUBLE NOT NULL,
+  `value2` DOUBLE NOT NULL,
+  `some_string` VARCHAR(32) NOT NULL
+) ENGINE=InnoDB;
+
+INSERT INTO `test_table`
+ (`value1`, `value2`, `some_string`)
+VALUES
+ (315.2599999999994, 2.6919444444444447, 'a');

util/update-modules

@@ -137,6 +137,10 @@ if [ ! -f $tool_file ]; then
    die "$tool_file does not exist"
 fi
 
+if [ -h $tool_file ]; then
+   die "$tool_file is a symbolic link"
+fi
+
 if [ -n "$(head -n 1 $tool_file | grep perl)" ]; then
    tool_lang="perl"
 else