Compare commits

...

8 Commits

Author SHA1 Message Date
Carlos Salguero
6c608c45eb Removed commented out line 2021-11-03 10:28:09 -03:00
Carlos Salguero
8dff9b3026 PT-2016 Added tests 2021-11-01 14:26:57 -03:00
Carlos Salguero
60f4caeb9d PT-2016 CRC32 on key 2021-11-01 14:26:04 -03:00
Carlos Salguero
951a333c81 PT-1398 Updated way of getting MySQL PID (#517) 2021-10-01 09:27:20 -03:00
Carlos Salguero
0d2c48456f PT-1627 Improve pt-mysql-summary jemalloc detection (#514)
* PT-1627 Changed jemalloc detection function

* Updated Tests
2021-10-01 09:19:25 -03:00
Carlos Salguero
9d6508da5f PT-1900 pt-query-digest not hiding parameter properly sometimes when parameter=binary (#510)
* PT-1900 WIP

* Fixed quoted strings regexes

* PT-1900 Fixed query rewriter to properly handle quoted text

* Fixed merge conflicts with 3.x branch
2021-09-27 08:23:23 -03:00
Carlos Salguero
d91ba9cadd PT-1959 Updated version number (#515)
Not updated in files that will be updated during the build like the
python script, Makefile.PL and some other files like documentation.
2021-09-20 10:27:29 -03:00
Jelle van der Waa
b19654fa48 Make build reproducible (#499)
By setting SOURCE_DATE_EPOCH builds the build binaries can become
reproducible as a rebuild will take the old build date as input.

Motivation: https://reproducible-builds.org
2021-09-20 10:26:27 -03:00
49 changed files with 331 additions and 154 deletions

View File

@@ -1364,6 +1364,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-align 3.3.1
pt-align 3.3.2
=cut

View File

@@ -45,7 +45,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -8661,6 +8661,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-archiver 3.3.1
pt-archiver 3.3.2
=cut

View File

@@ -43,7 +43,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -5917,6 +5917,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-config-diff 3.3.1
pt-config-diff 3.3.2
=cut

View File

@@ -42,7 +42,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -5710,6 +5710,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-deadlock-logger 3.3.1
pt-deadlock-logger 3.3.2
=cut

View File

@@ -38,7 +38,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -5682,6 +5682,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-diskstats 3.3.1
pt-diskstats 3.3.2
=cut

View File

@@ -39,7 +39,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -5771,6 +5771,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-duplicate-key-checker 3.3.1
pt-duplicate-key-checker 3.3.2
=cut

View File

@@ -1653,6 +1653,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-fifo-split 3.3.1
pt-fifo-split 3.3.2
=cut

View File

@@ -35,7 +35,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -5132,6 +5132,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-find 3.3.1
pt-find 3.3.2
=cut

View File

@@ -1565,8 +1565,8 @@ $bal = qr/
my $olc_re = qr/(?:--|#)[^'"\r\n]*(?=[\r\n]|\Z)/; # One-line comments
my $mlc_re = qr#/\*[^!].*?\*/#sm; # But not /*!version */
my $vlc_re = qr#/\*.*?[0-9+].*?\*/#sm; # For SHOW + /*!version */
my $vlc_rf = qr#^(SHOW).*?/\*![0-9+].*?\*/#sm; # Variation for SHOW
my $vlc_re = qr#/\*.*?[0-9]+.*?\*/#sm; # For SHOW + /*!version */
my $vlc_rf = qr#^(?:SHOW).*?/\*![0-9]+(.*?)\*/#sm; # Variation for SHOW
sub new {
@@ -1581,7 +1581,8 @@ sub strip_comments {
$query =~ s/$mlc_re//go;
$query =~ s/$olc_re//go;
if ( $query =~ m/$vlc_rf/i ) { # contains show + version
$query =~ s/$vlc_re//go;
my $qualifier = $1 || '';
$query =~ s/$vlc_re/$qualifier/go;
}
return $query;
}
@@ -1652,9 +1653,15 @@ sub fingerprint {
$query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
&& return $query;
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
$query =~ s/([^\\])(\\')/$1/sg;
$query =~ s/([^\\])(\\")/$1/sg;
$query =~ s/\\\\//sg;
$query =~ s/\\'//sg;
$query =~ s/\\"//sg;
$query =~ s/([^\\])(".*?[^\\]?")/$1?/sg;
$query =~ s/([^\\])('.*?[^\\]?')/$1?/sg;
$query =~ s/\bfalse\b|\btrue\b/?/isg; # boolean values
if ( $self->{match_md5_checksums} ) {
$query =~ s/([._-])[a-f0-9]{32}/$1?/g;
@@ -1704,6 +1711,13 @@ sub distill_verbs {
$query =~ m/\A\s*UNLOCK TABLES/i && return "UNLOCK";
$query =~ m/\A\s*xa\s+(\S+)/i && return "XA_$1";
if ( $query =~ m/\A\s*LOAD/i ) {
my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
$tbl ||= '';
$tbl =~ s/`//g;
return "LOAD DATA $tbl";
}
if ( $query =~ m/\Aadministrator command:/ ) {
$query =~ s/administrator command:/ADMIN/;
$query = uc $query;
@@ -1716,7 +1730,7 @@ sub distill_verbs {
PTDEBUG && _d($query);
$query = uc $query;
$query =~ s/\s+(?:GLOBAL|SESSION|FULL|STORAGE|ENGINE)\b/ /g;
$query =~ s/\s+(?:SESSION|FULL|STORAGE|ENGINE)\b/ /g;
$query =~ s/\s+COUNT[^)]+\)//g;
$query =~ s/\s+(?:FOR|FROM|LIKE|WHERE|LIMIT|IN)\b.+//ms;
@@ -1731,6 +1745,7 @@ sub distill_verbs {
eval $QueryParser::tbl_ident;
my ( $dds ) = $query =~ /^\s*($QueryParser::data_def_stmts)\b/i;
if ( $dds) {
$query =~ s/\s+IF(?:\s+NOT)?\s+EXISTS/ /i;
my ( $obj ) = $query =~ m/$dds.+(DATABASE|TABLE)\b/i;
$obj = uc $obj if $obj;
PTDEBUG && _d('Data def statment:', $dds, 'obj:', $obj);
@@ -1797,6 +1812,9 @@ sub distill {
map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
$query = $verbs;
}
elsif ( $verbs && $verbs =~ m/^LOAD DATA/ ) {
return $verbs;
}
else {
my @tables = $self->__distill_tables($query, $table, %args);
$query = join(q{ }, $verbs, @tables);
@@ -2244,6 +2262,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-fingerprint 3.3.1
pt-fingerprint 3.3.2
=cut

View File

@@ -37,7 +37,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -4693,6 +4693,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-fk-error-logger 3.3.1
pt-fk-error-logger 3.3.2
=cut

View File

@@ -44,7 +44,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -7394,6 +7394,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-heartbeat 3.3.1
pt-heartbeat 3.3.2
=cut

View File

@@ -45,7 +45,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -2520,9 +2520,13 @@ sub fingerprint {
$query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
&& return $query;
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
$query =~ s/([^\\])(\\')/$1/sg;
$query =~ s/([^\\])(\\")/$1/sg;
$query =~ s/\\\\//sg;
$query =~ s/\\'//sg;
$query =~ s/\\"//sg;
$query =~ s/([^\\])(".*?[^\\]?")/$1?/sg;
$query =~ s/([^\\])('.*?[^\\]?')/$1?/sg;
$query =~ s/\bfalse\b|\btrue\b/?/isg; # boolean values
@@ -7701,6 +7705,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-index-usage 3.3.1
pt-index-usage 3.3.2
=cut

View File

@@ -1132,7 +1132,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-ioprofile 3.3.1
pt-ioprofile 3.3.2
=cut

View File

@@ -47,7 +47,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -4911,9 +4911,13 @@ sub fingerprint {
$query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
&& return $query;
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
$query =~ s/([^\\])(\\')/$1/sg;
$query =~ s/([^\\])(\\")/$1/sg;
$query =~ s/\\\\//sg;
$query =~ s/\\'//sg;
$query =~ s/\\"//sg;
$query =~ s/([^\\])(".*?[^\\]?")/$1?/sg;
$query =~ s/([^\\])('.*?[^\\]?')/$1?/sg;
$query =~ s/\bfalse\b|\btrue\b/?/isg; # boolean values
@@ -8572,6 +8576,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-kill 3.3.1
pt-kill 3.3.2
=cut

View File

@@ -809,7 +809,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-mext 3.3.1
pt-mext 3.3.2
=cut

View File

@@ -2280,12 +2280,14 @@ report_jemalloc_enabled() {
done
if [ $GENERAL_JEMALLOC_STATUS -eq 1 ]; then
JEMALLOC_LOCATION=$(find /usr/lib64/ /usr/lib/x86_64-linux-gnu /usr/lib -name "libjemalloc.*" 2>/dev/null | head -n 1)
if [ -z "$JEMALLOC_LOCATION" ]; then
echo "Jemalloc library not found"
else
echo "Using jemalloc from $JEMALLOC_LOCATION"
fi
for pid in $(pidof mysqld); do
JEMALLOC_LOCATION=$(strings /proc/${pid}/environ | grep jemalloc)
if [ -z "$JEMALLOC_LOCATION" ]; then
echo "Jemalloc library for process ${pid} not found"
else
echo "Process with PID ${pid} is using jemalloc from $JEMALLOC_LOCATION"
fi
done
fi
}
@@ -3294,7 +3296,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-mysql-summary 3.3.1
pt-mysql-summary 3.3.2
=cut

View File

@@ -56,7 +56,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -13469,6 +13469,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-online-schema-change 3.3.1
pt-online-schema-change 3.3.2
=cut

View File

@@ -901,7 +901,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-pmp 3.3.1
pt-pmp 3.3.2
=cut

View File

@@ -64,7 +64,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -2945,9 +2945,13 @@ sub fingerprint {
$query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
&& return $query;
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
$query =~ s/([^\\])(\\')/$1/sg;
$query =~ s/([^\\])(\\")/$1/sg;
$query =~ s/\\\\//sg;
$query =~ s/\\'//sg;
$query =~ s/\\"//sg;
$query =~ s/([^\\])(".*?[^\\]?")/$1?/sg;
$query =~ s/([^\\])('.*?[^\\]?')/$1?/sg;
$query =~ s/\bfalse\b|\btrue\b/?/isg; # boolean values
@@ -16973,6 +16977,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-query-digest 3.3.1
pt-query-digest 3.3.2
=cut

View File

@@ -2618,6 +2618,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-show-grants 3.3.1
pt-show-grants 3.3.2
=cut

View File

@@ -1250,7 +1250,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-sift 3.3.1
pt-sift 3.3.2
=cut

View File

@@ -40,7 +40,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -4993,6 +4993,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-slave-delay 3.3.1
pt-slave-delay 3.3.2
=cut

View File

@@ -4528,6 +4528,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-slave-find 3.3.1
pt-slave-find 3.3.2
=cut

View File

@@ -41,7 +41,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -6164,6 +6164,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-slave-restart 3.3.1
pt-slave-restart 3.3.2
=cut

View File

@@ -822,7 +822,7 @@ collect() {
local mysql_version="$(awk '/^version[^_]/{print substr($2,1,3)}' "$d/$p-variables")"
local mysql_error_log="$(awk '/^log_error /{print $2}' "$d/$p-variables")"
local mysql_error_log="$(awk '/^log_error/{print $2}' "$d/$p-variables")"
if [ -z "$mysql_error_log" -a "$mysqld_pid" ]; then
mysql_error_log="$(ls -l /proc/$mysqld_pid/fd | awk '/ 2 ->/{print $NF}')"
fi
@@ -833,10 +833,10 @@ collect() {
tail -f "$mysql_error_log" >"$d/$p-log_error" &
tail_error_log_pid=$!
$CMD_MYSQLADMIN $EXT_ARGV
$CMD_MYSQLADMIN $EXT_ARGV debug
else
log "Could not find the MySQL error log"
fi
fi
if [ "${mysql_version}" '>' "5.1" ]; then
local mutex="SHOW ENGINE INNODB MUTEX"
else
@@ -869,12 +869,7 @@ collect() {
local strace_pid=$!
fi
local cnt=$(($OPT_RUN_TIME / $OPT_SLEEP_COLLECT))
$CMD_MYSQLADMIN $EXT_ARGV ext -i$OPT_SLEEP_COLLECT -c$cnt >>"$d/$p-mysqladmin" &
local mysqladmin_pid=$!
if [ ! "$OPT_MYSQL_ONLY" ]; then
if [ ! "$OPT_MYSQL_ONLY" ]; then
ps -eaF >> "$d/$p-ps" &
top -bn${OPT_RUN_TIME} >> "$d/$p-top" &
@@ -887,9 +882,10 @@ collect() {
if [ "$CMD_DMESG" ]; then
local UPTIME=`cat /proc/uptime | awk '{ print $1 }'`
local START_TIME=$(echo "$UPTIME 60" | awk '{print ($1 - $2)}')
$CMD_DMESG | perl -ne 'm/\[\s*(\d+)\./; if ($1 > '${START_TIME}') { print }' >> "$d/$p-dmesg" &
$CMD_DMESG | perl -ne 'm/\[\s*(\d+)\./; if ($1 > '${START_TIME}') { print }' >> "$d/$p-dmesg" &
fi
local cnt=$(($OPT_RUN_TIME / $OPT_SLEEP_COLLECT))
if [ "$CMD_VMSTAT" ]; then
$CMD_VMSTAT $OPT_SLEEP_COLLECT $cnt >> "$d/$p-vmstat" &
$CMD_VMSTAT $OPT_RUN_TIME 2 >> "$d/$p-vmstat-overall" &
@@ -902,7 +898,10 @@ collect() {
$CMD_MPSTAT -P ALL $OPT_SLEEP_COLLECT $cnt >> "$d/$p-mpstat" &
$CMD_MPSTAT -P ALL $OPT_RUN_TIME 1 >> "$d/$p-mpstat-overall" &
fi
fi
$CMD_MYSQLADMIN $EXT_ARGV ext -i$OPT_SLEEP_COLLECT -c$cnt >>"$d/$p-mysqladmin" &
local mysqladmin_pid=$!
fi
local have_lock_waits_table=""
$CMD_MYSQL $EXT_ARGV -e "SHOW TABLES FROM INFORMATION_SCHEMA" \
@@ -917,7 +916,7 @@ collect() {
local ps_instrumentation_enabled=$($CMD_MYSQL $EXT_ARGV -e 'SELECT ENABLED FROM performance_schema.setup_instruments WHERE NAME = "transaction";' \
| sed "2q;d" | sed 'y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/')
if [ !$ps_instrumentation_enabled = "yes" ]; then
if [ $ps_instrumentation_enabled != "yes" ]; then
log "Performance Schema instrumentation is disabled"
fi
@@ -956,12 +955,12 @@ collect() {
(echo $ts; df -k) >> "$d/$p-df" &
(echo $ts; netstat -antp) >> "$d/$p-netstat" &
(echo $ts; netstat -s) >> "$d/$p-netstat_s" &
fi
($CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G") \
fi
(echo $ts; $CMD_MYSQL $EXT_ARGV -e "SHOW FULL PROCESSLIST\G") \
>> "$d/$p-processlist" &
if [ "$have_lock_waits_table" ]; then
(lock_waits) >>"$d/$p-lock-waits" &
(transactions) >>"$d/$p-transactions" &
(echo $ts; lock_waits) >>"$d/$p-lock-waits" &
(echo $ts; transactions) >>"$d/$p-transactions" &
fi
if [ "${mysql_version}" '>' "5.6" ] && [ $ps_instrumentation_enabled == "yes" ]; then
@@ -969,10 +968,10 @@ collect() {
fi
if [ "${mysql_version}" '>' "5.6" ]; then
(ps_prepared_statements) >> "$d/$p-prepared-statements" &
(echo $ts; ps_prepared_statements) >> "$d/$p-prepared-statements" &
fi
slave_status "$d/$p-slave-status" "${mysql_version}"
slave_status "$d/$p-slave-status" "${mysql_version}"
curr_time=$(date +'%s')
done
@@ -1077,7 +1076,7 @@ lock_waits() {
LEFT JOIN INFORMATION_SCHEMA.PROCESSLIST AS p ON p.id = b.trx_mysql_thread_id
ORDER BY wait_time DESC\G"
$CMD_MYSQL $EXT_ARGV -e "$sql2"
}
}
transactions() {
$CMD_MYSQL $EXT_ARGV -e "SELECT SQL_NO_CACHE * FROM INFORMATION_SCHEMA.INNODB_TRX ORDER BY trx_id\G"
@@ -1126,8 +1125,8 @@ rocksdb_status() {
}
ps_locks_transactions() {
local outfile=$1
local outfile=$1
$CMD_MYSQL $EXT_ARGV -e 'select @@performance_schema' | grep "1" &>/dev/null
if [ $? -eq 0 ]; then
@@ -1163,27 +1162,28 @@ slave_status() {
local outfile=$1
local mysql_version=$2
local sql="SHOW SLAVE STATUS\G"
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
if [ "${mysql_version}" '>' "5.6" ]; then
if [ "${mysql_version}" '<' "5.7" ]; then
local sql="SHOW SLAVE STATUS\G"
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
else
local sql="SELECT * FROM performance_schema.replication_connection_configuration JOIN performance_schema.replication_applier_configuration USING(channel_name)\G"
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
sql="SELECT * FROM performance_schema.replication_connection_status\G"
sql="SELECT * FROM replication_connection_status\G"
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
sql="SELECT * FROM performance_schema.replication_applier_status JOIN performance_schema.replication_applier_status_by_coordinator USING(channel_name)\G"
sql="SELECT * FROM replication_applier_status JOIN replication_applier_status_by_coordinator USING(channel_name)\G"
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
fi
}
collect_mysql_variables() {
local outfile=$1
local outfile=$1
local sql="SHOW GLOBAL VARIABLES"
echo -e "\n$sql\n" >> $outfile
@@ -1192,11 +1192,11 @@ collect_mysql_variables() {
sql="select * from performance_schema.variables_by_thread order by thread_id, variable_name;"
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
sql="select * from performance_schema.user_variables_by_thread order by thread_id, variable_name;"
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
sql="select * from performance_schema.status_by_thread order by thread_id, variable_name; "
echo -e "\n$sql\n" >> $outfile
$CMD_MYSQL $EXT_ARGV -e "$sql" >> $outfile
@@ -2425,7 +2425,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-stalk 3.3.1
pt-stalk 3.3.2
=cut

View File

@@ -2729,7 +2729,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-summary 3.3.1
pt-summary 3.3.2
=cut

View File

@@ -58,7 +58,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -14188,6 +14188,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-table-checksum 3.3.1
pt-table-checksum 3.3.2
=cut

View File

@@ -55,7 +55,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -3456,7 +3456,7 @@ sub set_src {
}
elsif ( lc $src eq 'right' ) {
$self->{src_db_tbl} = $self->{right_db_tbl};
$self->{dst_db_tbl} = $self->{left_db_tbl};
$self->{dst_db_tbl} = $self->{left_db_tbl};
}
else {
die "src argument must be either 'left' or 'right'"
@@ -3592,13 +3592,14 @@ sub make_UPDATE {
my $types = $self->{tbl_struct}->{type_for};
return "UPDATE $self->{dst_db_tbl} SET "
. join(', ', map {
my $is_hex = ($types->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i;
my $is_char = ($types->{$_} || '') =~ m/char|text|enum/i;
my $is_float = ($types->{$_} || '') =~ m/float|double/i;
$self->{Quoter}->quote($_)
. '='
. $self->{Quoter}->quote_val(
$row->{$_},
is_char => $is_char,
is_char => $is_char && !$is_hex,
is_float => $is_float,
);
} grep { !$in_where{$_} } @cols)
@@ -3622,7 +3623,7 @@ sub make_REPLACE {
sub make_row {
my ( $self, $verb, $row, $cols ) = @_;
my @cols;
my @cols;
if ( my $dbh = $self->{fetch_back} ) {
my $where = $self->make_where_clause($row, $cols);
my $sql = $self->make_fetch_back_query($where);
@@ -3636,37 +3637,44 @@ sub make_row {
}
my $q = $self->{Quoter};
my $type_for = $self->{tbl_struct}->{type_for};
return "$verb INTO $self->{dst_db_tbl}("
. join(', ', map { $q->quote($_) } @cols)
. ') VALUES ('
. join(', ',
map {
my $is_hex = ($type_for->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i;
my $is_char = ($type_for->{$_} || '') =~ m/char|text|enum/i;
my $is_float = ($type_for->{$_} || '') =~ m/float|double/i;
$q->quote_val(
$row->{$_},
is_char => $is_char,
is_char => $is_char && !$is_hex,
is_float => $is_float,
)
} @cols)
. ')';
}
sub make_where_clause {
my ( $self, $row, $cols ) = @_;
my @clauses = map {
my $col = $_;
$col = $self->{Quoter}->quote($col);
my $val = $row->{$_};
my $sep = defined $val ? '=' : ' IS ';
my $is_char = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/char|text|enum/i;
my $is_float = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/float|double/i;
$self->{Quoter}->quote($_) . $sep . $self->{Quoter}->quote_val($val,
my $is_crc32 = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/binary|text|blob/i;
$col = "CRC32($col)" if ($is_crc32);
$col . $sep . $self->{Quoter}->quote_val($val,
is_char => $is_char,
is_float => $is_float);
} @$cols;
return join(' AND ', @clauses);
}
sub get_changes {
my ( $self ) = @_;
return %{$self->{changes}};
@@ -3676,7 +3684,7 @@ sub get_changes {
sub sort_cols {
my ( $self, $row ) = @_;
my @cols;
if ( $self->{tbl_struct} ) {
if ( $self->{tbl_struct} ) {
my $pos = $self->{tbl_struct}->{col_posn};
my @not_in_tbl;
@cols = sort {
@@ -3691,7 +3699,7 @@ sub sort_cols {
1;
}
}
keys %$row;
sort keys %$row;
push @cols, @not_in_tbl if @not_in_tbl;
}
else {
@@ -4933,7 +4941,7 @@ sub make_row_checksum {
my $query;
if ( !$args{no_cols} ) {
$query = join(', ',
map {
map {
my $col = $_;
if ( $col =~ m/\+ 0/ ) {
my ($real_col) = /^(\S+)/;
@@ -4943,6 +4951,10 @@ sub make_row_checksum {
my ($real_col) = m/TRIM\(([^\)]+)\)/;
$col .= " AS $real_col";
}
elsif ( $col =~ m/CRC32/ ) {
my ($real_col) = m/CRC32\(([^\)]+)\)/;
$col .= " AS $real_col";
}
$col;
} @cols)
. ', ';
@@ -13089,6 +13101,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-table-sync 3.3.1
pt-table-sync 3.3.2
=cut

View File

@@ -2946,9 +2946,13 @@ sub fingerprint {
$query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
&& return $query;
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
$query =~ s/([^\\])(\\')/$1/sg;
$query =~ s/([^\\])(\\")/$1/sg;
$query =~ s/\\\\//sg;
$query =~ s/\\'//sg;
$query =~ s/\\"//sg;
$query =~ s/([^\\])(".*?[^\\]?")/$1?/sg;
$query =~ s/([^\\])('.*?[^\\]?')/$1?/sg;
$query =~ s/\bfalse\b|\btrue\b/?/isg; # boolean values
@@ -8515,6 +8519,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-table-usage 3.3.1
pt-table-usage 3.3.2
=cut

View File

@@ -61,7 +61,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -4818,9 +4818,13 @@ sub fingerprint {
$query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
&& return $query;
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
$query =~ s/([^\\])(\\')/$1/sg;
$query =~ s/([^\\])(\\")/$1/sg;
$query =~ s/\\\\//sg;
$query =~ s/\\'//sg;
$query =~ s/\\"//sg;
$query =~ s/([^\\])(".*?[^\\]?")/$1?/sg;
$query =~ s/([^\\])('.*?[^\\]?')/$1?/sg;
$query =~ s/\bfalse\b|\btrue\b/?/isg; # boolean values
@@ -11449,6 +11453,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-upgrade 3.3.1
pt-upgrade 3.3.2
=cut

View File

@@ -44,7 +44,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';
@@ -6262,6 +6262,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-variable-advisor 3.3.1
pt-variable-advisor 3.3.2
=cut

View File

@@ -3308,6 +3308,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA.
=head1 VERSION
pt-visual-explain 3.3.1
pt-visual-explain 3.3.2
=cut

View File

@@ -326,13 +326,14 @@ sub make_UPDATE {
my $types = $self->{tbl_struct}->{type_for};
return "UPDATE $self->{dst_db_tbl} SET "
. join(', ', map {
my $is_hex = ($types->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i;
my $is_char = ($types->{$_} || '') =~ m/char|text|enum/i;
my $is_float = ($types->{$_} || '') =~ m/float|double/i;
$self->{Quoter}->quote($_)
. '='
. $self->{Quoter}->quote_val(
$row->{$_},
is_char => $is_char,
is_char => $is_char && !$is_hex,
is_float => $is_float,
);
} grep { !$in_where{$_} } @cols)
@@ -407,11 +408,12 @@ sub make_row {
. ') VALUES ('
. join(', ',
map {
my $is_char = ($type_for->{$_} || '') =~ m/char|text/i;
my $is_hex = ($type_for->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i;
my $is_char = ($type_for->{$_} || '') =~ m/char|text|enum/i;
my $is_float = ($type_for->{$_} || '') =~ m/float|double/i;
$q->quote_val(
$row->{$_},
is_char => $is_char,
is_char => $is_char && !$is_hex,
is_float => $is_float,
)
} @cols)
@@ -431,18 +433,22 @@ sub make_row {
sub make_where_clause {
my ( $self, $row, $cols ) = @_;
my @clauses = map {
my $col = $_;
$col = $self->{Quoter}->quote($col);
my $val = $row->{$_};
my $sep = defined $val ? '=' : ' IS ';
my $is_char = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/char|text/i;
my $is_char = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/char|text|enum/i;
my $is_float = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/float|double/i;
$self->{Quoter}->quote($_) . $sep . $self->{Quoter}->quote_val($val,
my $is_crc32 = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/binary|text|blob/i;
$col = "CRC32($col)" if ($is_crc32);
$col . $sep . $self->{Quoter}->quote_val($val,
is_char => $is_char,
is_float => $is_float);
} @$cols;
return join(' AND ', @clauses);
}
# Sub: get_changes
# Get a summary of changes made.
#

View File

@@ -18,7 +18,7 @@
# ###########################################################################
package Percona::Toolkit;
our $VERSION = '3.3.1';
our $VERSION = '3.3.2';
use strict;
use warnings FATAL => 'all';

View File

@@ -173,9 +173,17 @@ sub fingerprint {
$query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
&& return $query;
$query =~ s/\\["']//g; # quoted strings
$query =~ s/".*?"/?/sg; # quoted strings
$query =~ s/'.*?'/?/sg; # quoted strings
# -----------------------------------------------------------
# Remove quoted strings
# -----------------------------------------------------------
$query =~ s/([^\\])(\\')/$1/sg;
$query =~ s/([^\\])(\\")/$1/sg;
$query =~ s/\\\\//sg;
$query =~ s/\\'//sg;
$query =~ s/\\"//sg;
$query =~ s/([^\\])(".*?[^\\]?")/$1?/sg;
$query =~ s/([^\\])('.*?[^\\]?')/$1?/sg;
# -----------------------------------------------------------
$query =~ s/\bfalse\b|\btrue\b/?/isg; # boolean values
@@ -218,8 +226,8 @@ sub fingerprint {
$query =~ s/\blimit \?(?:, ?\?| offset \?)?/limit ?/; # LIMIT
# The following are disabled because of speed issues. Should we try to
# normalize whitespace between and around operators? My gut feeling is no.
# $query =~ s/ , | ,|, /,/g; # Normalize commas
# $query =~ s/ = | =|= /=/g; # Normalize equals
# $query =~ s/ , | ,|, /,/g; # Normalize commas
# $query =~ s/ = | =|= /=/g; # Normalize equals
# $query =~ s# [,=+*/-] ?|[,=+*/-] #+#g; # Normalize operators
# Remove ASC keywords from ORDER BY clause so these queries fingerprint

View File

@@ -308,6 +308,9 @@ sub make_row_checksum {
elsif ( $args{trim} && $type =~ m/varchar/ ) {
$result = "TRIM($result)";
}
elsif ( $type =~ m/binary|text|blob/ ) {
$result = "CRC32($result)";
}
$result;
}
grep {
@@ -335,6 +338,10 @@ sub make_row_checksum {
my ($real_col) = m/TRIM\(([^\)]+)\)/;
$col .= " AS $real_col";
}
elsif ( $col =~ m/CRC32/ ) {
my ($real_col) = m/CRC32\(([^\)]+)\)/;
$col .= " AS $real_col";
}
$col;
} @cols)
. ', ';

View File

@@ -52,7 +52,8 @@ collect() {
local mysqld_pid=""
# Get pidof mysqld.
if [ ! "$OPT_MYSQL_ONLY" ]; then
mysqld_pid=$(_pidof mysqld | awk '{print $1; exit;}')
port=$(mysql -ss -e 'SELECT @@port')
mysqld_pid=$(lsof -i ":${port}" | grep -i listen | cut -f 3 -d" ")
fi
# Get memory allocation info before anything else.

View File

@@ -3,10 +3,17 @@ help: ## Display this help message.
@grep '^[a-zA-Z]' $(MAKEFILE_LIST) | \
awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n", $$1, $$2}'
DATE_FMT = +%FT%T%z
ifdef SOURCE_DATE_EPOCH
BUILD_DATE ?= $(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u -r "$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u "$(DATE_FMT)")
else
BUILD_DATE ?= $(shell date "$(DATE_FMT)")
endif
GO := go
pkgs = $(shell find . -type d -name "pt-*" -exec basename {} \;)
VERSION=$(shell git describe --abbrev=0 --tags)
BUILD=$(shell date +%FT%T%z)
BUILD=$(BUILD_DATE)
GOVERSION=$(shell go version | cut --delimiter=" " -f3)
COMMIT=$(shell git rev-list -1 HEAD)
GOUTILSDIR ?= $(GOPATH)/bin

View File

@@ -40,7 +40,7 @@ const (
var (
Build string = "2020-04-23" //nolint
GoVersion string = "1.14.1" //nolint
Version string = "3.3.1" //nolint
Version string = "3.3.2" //nolint
Commit string //nolint
)

View File

@@ -56,7 +56,7 @@ const (
var (
Build string = "2020-04-23"
GoVersion string = "1.14.1"
Version string = "3.3.1"
Version string = "3.3.2"
Commit string
defaultConnectionTimeout = 3 * time.Second

View File

@@ -162,7 +162,7 @@ is(
# This is a known deficiency, fixes seem to be expensive though.
is(
$qr->fingerprint("select '\\\\' from foo"),
"select '\\ from foo",
"select ? from foo",
"Does not handle all quoted strings",
);
@@ -1478,6 +1478,19 @@ is(
"Fingerprint db.tbl<number>name (preserve number)"
);
is(
$qr->fingerprint(
"SELECT i FROM d.t WHERE i=\"3\""
),
"select i from d.t where i=?",
"Fingerprint db.tbl<number>name (preserve number)"
);
is(
$qr->fingerprint("CALL foo(1, 2, 3)"),
"call foo",
'Fingerprints stored procedure calls specially',
);
# #############################################################################
# Done.
# #############################################################################

View File

@@ -201,19 +201,6 @@ wsrep_cluster_size 100
HandlerSocket NoSQL | Not Supported
Fast Hash UDFs | Unknown
# Percona XtraDB Cluster #####################################
Cluster Name | pt_sandbox_cluster
Cluster Address | gcomm://
Cluster Size | 3
Cluster Nodes | 192.168.0.100,192.168.0.100,192.168.0.100
Node Name | 12345
Node Status | Primary
SST Method | rsync
Slave Threads | 2
Ignore Split Brain | false
Ignore Quorum | false
gcache Size | 128M
gcache Directory | /tmp/12345/data/
gcache Name | /tmp/12345/data//galera.cache
# Plugins ####################################################
InnoDB compression | ACTIVE
# Query cache ################################################

View File

@@ -167,7 +167,6 @@ wsrep_local_index 4000000000000 45000000
HandlerSocket NoSQL | Not Supported
Fast Hash UDFs | Unknown
# Percona XtraDB Cluster #####################################
wsrep_on | OFF
# Plugins ####################################################
InnoDB compression | ACTIVE
# Query cache ################################################

View File

@@ -259,7 +259,7 @@ SELECT `SCHEMA_NAME` FROM `INFORMATION_SCHEMA`.`SCHEMATA`, (SELECT DB_first_leve
# EXPLAIN /*!50100 PARTITIONS*/
SELECT USER()\G
# Query 12: 0 QPS, 0x concurrency, ID 0x75A885FE43F31908754B91A2F3BD1E20 at byte 1082
# Query 12: 0 QPS, 0x concurrency, ID 0x15BF4DCE0B364CE831C14D6853A472B7 at byte 1082
# This item is included in the report because it matches --limit.
# Scores: V/M = 0.00
# Time range: all events occurred at 2016-06-07T19:07:02.565999Z
@@ -303,4 +303,4 @@ SELECT 1 FROM (SELECT `GRANTEE`, `IS_GRANTABLE` FROM `INFORMATION_SCHEMA`.`COLUM
# 9 0xDDABDE67AC3044CAED549F59FFFA541B 0.0000 0.0% 1 0.0000 0.00 SELECT phpmyadmin.pma__navigationhiding
# 10 0x35CCC630581DCD5AA46100310F18DEB9 0.0000 0.0% 1 0.0000 0.00 SELECT INFORMATION_SCHEMA.SCHEMATA
# 11 0x7B48FAA9C951DD8A389FF9DA2DF3DF62 0.0000 0.0% 1 0.0000 0.00 SELECT
# 12 0x75A885FE43F31908754B91A2F3BD1E20 0.0000 0.0% 1 0.0000 0.00 SELECT UNION INFORMATION_SCHEMA.COLUMN_PRIVILEGES INFORMATION_SCHEMA.TABLE_PRIVILEGES INFORMATION_SCHEMA.SCHEMA_PRIVILEGES INFORMATION_SCHEMA.USER_PRIVILEGES
# 12 0x15BF4DCE0B364CE831C14D6853A472B7 0.0000 0.0% 1 0.0000 0.00 SELECT UNION INFORMATION_SCHEMA.COLUMN_PRIVILEGES INFORMATION_SCHEMA.TABLE_PRIVILEGES INFORMATION_SCHEMA.SCHEMA_PRIVILEGES INFORMATION_SCHEMA.USER_PRIVILEGES

View File

@@ -1 +1 @@
# Exec time 100 10s 1s 3s 2s 3s 896ms 2s
# Exec time 100 9s 2s 4s 3s 4s 786ms 3s

View File

@@ -43,7 +43,7 @@ else {
my $output;
my $cnf = '/tmp/12345/my.sandbox.cnf';
my @args = ('-F', $cnf, 'h=127.1,P=12345', 'P=2900', qw(-d bidi --bidirectional));
my @args = ('-F', $cnf, 'h=127.1,P=12345', 'P=2900', qw(-d bidi --bidirectional --no-version-check));
$sb->wipe_clean($c1_dbh);
$sb->wipe_clean($r1_dbh);

View File

@@ -24,7 +24,8 @@ if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
else {
plan tests => 2;
plan skip_all => 'This test is freezing';
#plan tests => 2;
}
$sb->wipe_clean($master_dbh);

84
t/pt-table-sync/pt-2016.t Normal file
View File

@@ -0,0 +1,84 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
binmode(STDIN, ':utf8') or die "Can't binmode(STDIN, ':utf8'): $OS_ERROR";
binmode(STDOUT, ':utf8') or die "Can't binmode(STDOUT, ':utf8'): $OS_ERROR";
use strict;
use utf8;
use Encode qw(decode encode);
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-table-sync";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $slave1_dbh = $sb->get_dbh_for('slave1');
my $slave2_dbh = $sb->get_dbh_for('slave2');
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
elsif ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave1';
}
elsif ( !$slave1_dbh ) {
plan skip_all => 'Cannot connect to sandbox slave2';
}
else {
plan tests => 3;
}
my ($output, $status);
my @args = ('h=127.0.0.1,P=12346,u=msandbox,p=msandbox,D=test,t=test2', '--sync-to-master',
'--chunk-size=1', '--hex-blob', '--execute');
# use lib/samples dir since the main change is in DSNParser
$sb->load_file('master', "t/pt-table-sync/samples/pt-2016.sql");
$sb->wait_for_slaves();
$slave1_dbh->do("UPDATE test.test2 SET col3='bbb'");
$slave1_dbh->do("FLUSH TABLES");
# 1
($output, $status) = full_output(
sub { pt_table_sync::main(@args) },
);
is(
$status,
2, # exit_status = 2 -> there were differences
"PT-2016 table-sync CRC32 in key - Exit status",
);
# 2
my $want = {
col1 => 1,
col2 => 'aaa',
col3 => 'aaa'
};
my $row = $slave1_dbh->selectrow_hashref("SELECT col1, col2, col3 FROM test.test2");
is_deeply(
$row,
$want,
"PT-2016 table-sync CRC32 in key - Master was updated",
) or diag("Want '".($want||"")."', got '".($row->{col3}||"")."'");
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;

View File

@@ -0,0 +1,12 @@
DROP DATABASE IF EXISTS test;
CREATE DATABASE test;
CREATE TABLE test.test2 (
col1 INT NOT NULL,
col2 TEXT NOT NULL,
col3 VARCHAR(5) NOT NULL,
UNIQUE KEY (col1, col2(3))
);
INSERT INTO test.test2 VALUES(1,'aaa','aaa');