percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-07 21:09:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '3.0' into PMM-4278-official-mongo-driver

Browse Source
This commit is contained in:
Carlos Salguero
2019-07-25 21:43:14 -03:00
parent f955451fe5 748945a45c
commit f9cb161483
16 changed files with 199 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bin/pt-archiver
View File

@@ -3745,6 +3745,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

5
bin/pt-diskstats
View File

@@ -2240,8 +2240,11 @@ sub design_print_formats {
sub parse_diskstats_line {
my ( $self, $line, $block_size ) = @_;
# linux kernel source => Documentation/iostats.txt
# 2.6+ => 14 fields
# 4.18+ => 18 fields
my @dev_stats = split ' ', $line;
return unless @dev_stats == 14;
return unless @dev_stats == 14 or @dev_stats == 18;
my $read_bytes = $dev_stats[READ_SECTORS] * $block_size;
my $written_bytes = $dev_stats[WRITTEN_SECTORS] * $block_size;

43
bin/pt-heartbeat
View File

@@ -112,22 +112,22 @@ use warnings FATAL => 'all';
use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
sub check_recursion_method {
sub check_recursion_method {
my ($methods) = @_;
if ( @$methods != 1 ) {
if ( grep({ !m/processlist|hosts/i } @$methods)
&& $methods->[0] !~ /^dsn=/i )
{
die "Invalid combination of recursion methods: "
. join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". "
. "Only hosts and processlist may be combined.\n"
}
}
else {
if ( @$methods != 1 ) {
if ( grep({ !m/processlist|hosts/i } @$methods)
&& $methods->[0] !~ /^dsn=/i )
{
die "Invalid combination of recursion methods: "
. join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". "
. "Only hosts and processlist may be combined.\n"
}
}
else {
my ($method) = @$methods;
die "Invalid recursion method: " . ( $method || 'undef' )
unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i;
}
die "Invalid recursion method: " . ( $method || 'undef' )
unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i;
}
}
sub new {
@@ -156,7 +156,7 @@ sub get_slaves {
my $methods = $self->_resolve_recursion_methods($args{dsn});
return $slaves unless @$methods;
if ( grep { m/processlist|hosts/i } @$methods ) {
my @required_args = qw(dbh dsn);
foreach my $arg ( @required_args ) {
@@ -201,7 +201,7 @@ sub get_slaves {
else {
die "Unexpected recursion methods: @$methods";
}
return $slaves;
}
@@ -321,6 +321,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}
@@ -719,7 +720,7 @@ sub short_host {
}
sub is_replication_thread {
my ( $self, $query, %args ) = @_;
my ( $self, $query, %args ) = @_;
return unless $query;
my $type = lc($args{type} || 'all');
@@ -734,7 +735,7 @@ sub is_replication_thread {
if ( !$match ) {
if ( ($query->{User} || $query->{user} || '') eq "system user" ) {
PTDEBUG && _d("Slave replication thread");
if ( $type ne 'all' ) {
if ( $type ne 'all' ) {
my $state = $query->{State} || $query->{state} || '';
if ( $state =~ m/^init|end$/ ) {
@@ -747,7 +748,7 @@ sub is_replication_thread {
|Reading\sevent\sfrom\sthe\srelay\slog
|Has\sread\sall\srelay\slog;\swaiting
|Making\stemp\sfile
|Waiting\sfor\sslave\smutex\son\sexit)/xi;
|Waiting\sfor\sslave\smutex\son\sexit)/xi;
$match = $type eq 'slave_sql' && $slave_sql ? 1
: $type eq 'slave_io' && !$slave_sql ? 1
@@ -811,7 +812,7 @@ sub get_replication_filters {
replicate_do_db
replicate_ignore_db
replicate_do_table
replicate_ignore_table
replicate_ignore_table
replicate_wild_do_table
replicate_wild_ignore_table
);
@@ -822,7 +823,7 @@ sub get_replication_filters {
$filters{slave_skip_errors} = $row->[1] if $row->[1] && $row->[1] ne 'OFF';
}
return \%filters;
return \%filters;
}

1
bin/pt-kill
View File

@@ -4027,6 +4027,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

4
bin/pt-online-schema-change
View File

@@ -4281,8 +4281,7 @@ sub recurse_to_slaves {
PTDEBUG && _d('Connected to', $dp->as_string($slave_dsn));
};
if ( $EVAL_ERROR ) {
#TODO REMOVE DEBUG
print STDERR "1> Cannot connect to ", $dp->as_string($slave_dsn), "\n"
print STDERR "Cannot connect to ", $dp->as_string($slave_dsn), "\n"
or die "Cannot print: $OS_ERROR";
return;
}
@@ -4350,6 +4349,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

1
bin/pt-pmp
View File

@@ -447,7 +447,6 @@ _parse_command_line() {
else
spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
option_error "Unknown option: $real_opt"
continue
fi
fi

1
bin/pt-query-digest
View File

@@ -10675,6 +10675,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

1
bin/pt-slave-find
View File

@@ -2435,6 +2435,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

1
bin/pt-slave-restart
View File

@@ -2846,6 +2846,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

44
bin/pt-stalk
View File

@@ -1352,6 +1352,8 @@ sleep_ok() {
purge_samples() {
local dir="$1"
local retention_time="$2"
local retention_count="$3"
local retention_size="$4"
# Delete collect files which more than --retention-time days old.
find "$dir" -type f -mtime +$retention_time -exec rm -f '{}' \;
@@ -1363,6 +1365,30 @@ purge_samples() {
find "$oprofile_dir" -depth -type d -name 'pt_collect_*' \
-mtime +$retention_time -exec rm -rf '{}' \;
fi
targetCnt=$(($retention_count + 0))
if [ $targetCnt -gt 0 ]; then
targetCnt=$(($retention_count + 1))
files_to_delete=$(find $dir -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +${targetCnt})
for prefix in $files_to_delete; do
echo "deleting files ${dir}${prefix}* according to the --retention-count param"
rm -f ${dir}${prefix}* 2>/dev/null
done
fi
targetSize=$(($retention_size + 0))
if [ $targetSize -gt 0 ]; then
files_to_delete=$(find $dir -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +1)
for prefix in $files_to_delete; do
current_size=$(du -BM $dir | cut -f1 -d"M")
if [ $current_size -gt $targetSize ]; then
echo "deleting files ${dir}${prefix}* according to the --retention-size param"
rm -f ${dir}${prefix}* 2>/dev/null
else
break
fi
done
fi
}
sigtrap() {
@@ -1471,7 +1497,7 @@ stalk() {
# Purge old collect files.
if [ -d "$OPT_DEST" ]; then
purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME"
purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" "$OPT_RETENTION_SIZE"
fi
fi
@@ -1496,7 +1522,7 @@ stalk() {
# One final purge of old collect files, but only if in collect mode.
if [ "$OPT_COLLECT" -a -d "$OPT_DEST" ]; then
purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME"
purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" "$OPT_RETENTION_SIZE"
fi
# Before exiting, the last collector may still be running.
@@ -2120,6 +2146,20 @@ The filename prefix for diagnostic samples. By default, all files created
by the same L<"--collect"> instance have a timestamp prefix based on the current
local time, like C<2011_12_06_14_02_02>, which is December 6, 2011 at 14:02:02.
=item --retention-count
type: int; default: 0
Keep the data for the last N runs. If N > 0, the program will keep the data for the last
N runs and will delete the older data.
=item --retention-size
type: int; default: 0
Keep up to --retention-size MB of data. It will keep at least 1 run even if the size is bigger
than the specified in this parameter
=item --retention-time
type: int; default: 30

16
bin/pt-table-checksum
View File

@@ -5299,6 +5299,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}
@@ -6726,9 +6727,7 @@ sub can_nibble {
if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) {
$one_nibble = 1;
}
} else {
$one_nibble = 1;
}
}
PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no');
@@ -9660,9 +9659,7 @@ sub _get_first_values {
die "I need a $arg argument" unless $args{$arg};
}
my ($cxn, $tbl, $index, $n_index_cols) = @args{@required_args};
my $q = $self->{Quoter};
my $q = new Quoter();
my $index_struct = $tbl->{tbl_struct}->{keys}->{$index};
my $index_cols = $index_struct->{cols};
my $index_columns = join (', ',
@@ -9690,9 +9687,7 @@ sub _make_range_query {
die "I need a $arg argument" unless $args{$arg};
}
my ($tbl, $index, $n_index_cols, $vals) = @args{@required_args};
my $q = $self->{Quoter};
my $q = new Quoter();
my $index_struct = $tbl->{tbl_struct}->{keys}->{$index};
my $index_cols = $index_struct->{cols};
@@ -11595,6 +11590,9 @@ sub nibble_is_safe {
# See https://bugs.launchpad.net/percona-toolkit/+bug/987393
my $sth = $nibble_iter->statements();
my $boundary = $nibble_iter->boundaries();
if (!defined($boundary) || !$boundary || (!$boundary->{lower} || !$boundary->{upper})) {
return 0;
}
my $expl = explain_statement(
tbl => $tbl,
sth => $sth->{explain_nibble},

1
bin/pt-table-sync
View File

@@ -6844,6 +6844,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

1
lib/MasterSlave.pm
View File

@@ -272,6 +272,7 @@ sub _find_slaves_by_processlist {
grep { $_ }
map {
my ( $host ) = $_->{host} =~ m/^([^:]+):/;
$host ||= $_->{host};
if ( $host eq 'localhost' ) {
$host = '127.0.0.1'; # Replication never uses sockets.
}

5
lib/bash/parse_options.sh
View File

@@ -468,7 +468,10 @@ _parse_command_line() {
else
spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1)
if [ -z "$spec" ]; then
option_error "Unknown option: $real_opt"
# Not all programs uses the same options and since these options can be stored
# in a common config file, we need to skip general options not used by a particular
# program
# option_error "Unknown option: $real_opt"
continue
fi
fi

8
t/lib/TableParser.t
View File

@@ -1267,6 +1267,14 @@ is_deeply(
'Column having the word "generated" as part of the comment is OK',
) or diag Data::Dumper::Dumper($tbl);
$tbl = $tp->parse( load_file('t/lib/samples/generated_cols_comments.sql') );
warn Data::Dumper::Dumper($tbl);
is_deeply(
$tbl,
{},
'pt-1728',
);
# #############################################################################
# Done.
# #############################################################################

103
t/pt-table-checksum/pt-1728.t Normal file
View File

@@ -0,0 +1,103 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use PerconaTest;
use Sandbox;
use SqlModes;
use threads;
use Time::HiRes qw( usleep );
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
require "$trunk/bin/pt-table-checksum";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
else {
plan tests => 2;
}
my $num_rows = 1_000_000;
my $table = 't1';
$dbh->do("DROP DATABASE IF EXISTS test");
$dbh->do("CREATE DATABASE IF NOT EXISTS test");
$dbh->do("CREATE TABLE `test`.`$table` (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(5)) Engine=InnoDB");
diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test $table $num_rows`);
sub start_thread {
my ($dsn_opts, $initial_sleep_time) = @_;
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $slave_dbh = $sb->get_dbh_for('slave1');
diag("Sleeping");
sleep($initial_sleep_time);
diag("Woke up");
$slave_dbh->do("STOP SLAVE IO_THREAD FOR CHANNEL ''");
$slave_dbh->do("STOP SLAVE");
$master_dbh->do("TRUNCATE TABLE test.$table");
# PTDEBUG && diag("Exit thread")
sleep(2);
$slave_dbh->do("START SLAVE");
diag("Exit thread")
}
# This is not a realiable sleep value. It works for a i7, hybrid HDD
my $initial_sleep_time = 17;
my $thr = threads->create('start_thread', $dsn_opts, $initial_sleep_time);
threads->yield();
diag("Starting checksum");
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die.
# And --max-load "" prevents waiting for status variables.
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = ($master_dsn, qw(--no-check-binlog-format --chunk-size 10));
my $output;
$output = output(
sub { pt_table_checksum::main(@args) },
stderr => 1,
);
diag($output);
unlike(
$output,
qr/Can't use an undefined value as an ARRAY/,
"Truncating tables while checksum is running"
);
$sb->load_file('master', 't/pt-table-checksum/samples/pt-1728.sql');
@args = ($master_dsn, qw(--no-check-binlog-format));
my $new_rows_count = $num_rows * 5;
diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test $table $new_rows_count`);
$output = output(
sub { pt_table_checksum::main(@args) },
stderr => 1,
);
diag($output);
$thr->join();
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
exit;