diff --git a/bin/pt-archiver b/bin/pt-archiver index c868cbc4..69d258df 100755 --- a/bin/pt-archiver +++ b/bin/pt-archiver @@ -3745,6 +3745,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/bin/pt-diskstats b/bin/pt-diskstats index f26ac5ab..c43e4904 100755 --- a/bin/pt-diskstats +++ b/bin/pt-diskstats @@ -2240,8 +2240,11 @@ sub design_print_formats { sub parse_diskstats_line { my ( $self, $line, $block_size ) = @_; + # linux kernel source => Documentation/iostats.txt + # 2.6+ => 14 fields + # 4.18+ => 18 fields my @dev_stats = split ' ', $line; - return unless @dev_stats == 14; + return unless @dev_stats == 14 or @dev_stats == 18; my $read_bytes = $dev_stats[READ_SECTORS] * $block_size; my $written_bytes = $dev_stats[WRITTEN_SECTORS] * $block_size; diff --git a/bin/pt-heartbeat b/bin/pt-heartbeat index da52e9d2..a2949705 100755 --- a/bin/pt-heartbeat +++ b/bin/pt-heartbeat @@ -112,22 +112,22 @@ use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; -sub check_recursion_method { +sub check_recursion_method { my ($methods) = @_; - if ( @$methods != 1 ) { - if ( grep({ !m/processlist|hosts/i } @$methods) - && $methods->[0] !~ /^dsn=/i ) - { - die "Invalid combination of recursion methods: " - . join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". " - . "Only hosts and processlist may be combined.\n" - } - } - else { + if ( @$methods != 1 ) { + if ( grep({ !m/processlist|hosts/i } @$methods) + && $methods->[0] !~ /^dsn=/i ) + { + die "Invalid combination of recursion methods: " + . join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". " + . "Only hosts and processlist may be combined.\n" + } + } + else { my ($method) = @$methods; - die "Invalid recursion method: " . ( $method || 'undef' ) - unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i; - } + die "Invalid recursion method: " . ( $method || 'undef' ) + unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i; + } } sub new { @@ -156,7 +156,7 @@ sub get_slaves { my $methods = $self->_resolve_recursion_methods($args{dsn}); return $slaves unless @$methods; - + if ( grep { m/processlist|hosts/i } @$methods ) { my @required_args = qw(dbh dsn); foreach my $arg ( @required_args ) { @@ -201,7 +201,7 @@ sub get_slaves { else { die "Unexpected recursion methods: @$methods"; } - + return $slaves; } @@ -321,6 +321,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } @@ -719,7 +720,7 @@ sub short_host { } sub is_replication_thread { - my ( $self, $query, %args ) = @_; + my ( $self, $query, %args ) = @_; return unless $query; my $type = lc($args{type} || 'all'); @@ -734,7 +735,7 @@ sub is_replication_thread { if ( !$match ) { if ( ($query->{User} || $query->{user} || '') eq "system user" ) { PTDEBUG && _d("Slave replication thread"); - if ( $type ne 'all' ) { + if ( $type ne 'all' ) { my $state = $query->{State} || $query->{state} || ''; if ( $state =~ m/^init|end$/ ) { @@ -747,7 +748,7 @@ sub is_replication_thread { |Reading\sevent\sfrom\sthe\srelay\slog |Has\sread\sall\srelay\slog;\swaiting |Making\stemp\sfile - |Waiting\sfor\sslave\smutex\son\sexit)/xi; + |Waiting\sfor\sslave\smutex\son\sexit)/xi; $match = $type eq 'slave_sql' && $slave_sql ? 1 : $type eq 'slave_io' && !$slave_sql ? 1 @@ -811,7 +812,7 @@ sub get_replication_filters { replicate_do_db replicate_ignore_db replicate_do_table - replicate_ignore_table + replicate_ignore_table replicate_wild_do_table replicate_wild_ignore_table ); @@ -822,7 +823,7 @@ sub get_replication_filters { $filters{slave_skip_errors} = $row->[1] if $row->[1] && $row->[1] ne 'OFF'; } - return \%filters; + return \%filters; } diff --git a/bin/pt-kill b/bin/pt-kill index dc0b67df..1eac8fb3 100755 --- a/bin/pt-kill +++ b/bin/pt-kill @@ -4027,6 +4027,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 54c97e31..ff29c278 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -4281,8 +4281,7 @@ sub recurse_to_slaves { PTDEBUG && _d('Connected to', $dp->as_string($slave_dsn)); }; if ( $EVAL_ERROR ) { - #TODO REMOVE DEBUG - print STDERR "1> Cannot connect to ", $dp->as_string($slave_dsn), "\n" + print STDERR "Cannot connect to ", $dp->as_string($slave_dsn), "\n" or die "Cannot print: $OS_ERROR"; return; } @@ -4350,6 +4349,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/bin/pt-pmp b/bin/pt-pmp index e330bc6d..b34fa0bf 100755 --- a/bin/pt-pmp +++ b/bin/pt-pmp @@ -447,7 +447,6 @@ _parse_command_line() { else spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1) if [ -z "$spec" ]; then - option_error "Unknown option: $real_opt" continue fi fi diff --git a/bin/pt-query-digest b/bin/pt-query-digest index 90d31679..96a6de73 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -10675,6 +10675,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/bin/pt-slave-find b/bin/pt-slave-find index 5c18448f..96c6498c 100755 --- a/bin/pt-slave-find +++ b/bin/pt-slave-find @@ -2435,6 +2435,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/bin/pt-slave-restart b/bin/pt-slave-restart index dabb81b9..6da8aea3 100755 --- a/bin/pt-slave-restart +++ b/bin/pt-slave-restart @@ -2846,6 +2846,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/bin/pt-stalk b/bin/pt-stalk index 2d65a88f..33adef2b 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -1352,6 +1352,8 @@ sleep_ok() { purge_samples() { local dir="$1" local retention_time="$2" + local retention_count="$3" + local retention_size="$4" # Delete collect files which more than --retention-time days old. find "$dir" -type f -mtime +$retention_time -exec rm -f '{}' \; @@ -1363,6 +1365,30 @@ purge_samples() { find "$oprofile_dir" -depth -type d -name 'pt_collect_*' \ -mtime +$retention_time -exec rm -rf '{}' \; fi + + targetCnt=$(($retention_count + 0)) + if [ $targetCnt -gt 0 ]; then + targetCnt=$(($retention_count + 1)) + files_to_delete=$(find $dir -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +${targetCnt}) + for prefix in $files_to_delete; do + echo "deleting files ${dir}${prefix}* according to the --retention-count param" + rm -f ${dir}${prefix}* 2>/dev/null + done + fi + + targetSize=$(($retention_size + 0)) + if [ $targetSize -gt 0 ]; then + files_to_delete=$(find $dir -type f -exec basename {} \; | cut -f1 -d- | sort -r | uniq | tail -n +1) + for prefix in $files_to_delete; do + current_size=$(du -BM $dir | cut -f1 -d"M") + if [ $current_size -gt $targetSize ]; then + echo "deleting files ${dir}${prefix}* according to the --retention-size param" + rm -f ${dir}${prefix}* 2>/dev/null + else + break + fi + done + fi } sigtrap() { @@ -1471,7 +1497,7 @@ stalk() { # Purge old collect files. if [ -d "$OPT_DEST" ]; then - purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" + purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" "$OPT_RETENTION_SIZE" fi fi @@ -1496,7 +1522,7 @@ stalk() { # One final purge of old collect files, but only if in collect mode. if [ "$OPT_COLLECT" -a -d "$OPT_DEST" ]; then - purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" + purge_samples "$OPT_DEST" "$OPT_RETENTION_TIME" "$OPT_RETENTION_COUNT" "$OPT_RETENTION_SIZE" fi # Before exiting, the last collector may still be running. @@ -2120,6 +2146,20 @@ The filename prefix for diagnostic samples. By default, all files created by the same L<"--collect"> instance have a timestamp prefix based on the current local time, like C<2011_12_06_14_02_02>, which is December 6, 2011 at 14:02:02. +=item --retention-count + +type: int; default: 0 + +Keep the data for the last N runs. If N > 0, the program will keep the data for the last +N runs and will delete the older data. + +=item --retention-size + +type: int; default: 0 + +Keep up to --retention-size MB of data. It will keep at least 1 run even if the size is bigger +than the specified in this parameter + =item --retention-time type: int; default: 30 diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 92c0edcd..86d5aed8 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -5299,6 +5299,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } @@ -6726,9 +6727,7 @@ sub can_nibble { if ( !$key || !$key_len || lc($key) ne lc($mysql_index)) { $one_nibble = 1; } - } else { - $one_nibble = 1; - } + } PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); @@ -9660,9 +9659,7 @@ sub _get_first_values { die "I need a $arg argument" unless $args{$arg}; } my ($cxn, $tbl, $index, $n_index_cols) = @args{@required_args}; - - my $q = $self->{Quoter}; - + my $q = new Quoter(); my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_cols = $index_struct->{cols}; my $index_columns = join (', ', @@ -9690,9 +9687,7 @@ sub _make_range_query { die "I need a $arg argument" unless $args{$arg}; } my ($tbl, $index, $n_index_cols, $vals) = @args{@required_args}; - - my $q = $self->{Quoter}; - + my $q = new Quoter(); my $index_struct = $tbl->{tbl_struct}->{keys}->{$index}; my $index_cols = $index_struct->{cols}; @@ -11595,6 +11590,9 @@ sub nibble_is_safe { # See https://bugs.launchpad.net/percona-toolkit/+bug/987393 my $sth = $nibble_iter->statements(); my $boundary = $nibble_iter->boundaries(); + if (!defined($boundary) || !$boundary || (!$boundary->{lower} || !$boundary->{upper})) { + return 0; + } my $expl = explain_statement( tbl => $tbl, sth => $sth->{explain_nibble}, diff --git a/bin/pt-table-sync b/bin/pt-table-sync index 036a449d..97544686 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -6844,6 +6844,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/lib/MasterSlave.pm b/lib/MasterSlave.pm index 5bb628d3..239d14a6 100644 --- a/lib/MasterSlave.pm +++ b/lib/MasterSlave.pm @@ -272,6 +272,7 @@ sub _find_slaves_by_processlist { grep { $_ } map { my ( $host ) = $_->{host} =~ m/^([^:]+):/; + $host ||= $_->{host}; if ( $host eq 'localhost' ) { $host = '127.0.0.1'; # Replication never uses sockets. } diff --git a/lib/bash/parse_options.sh b/lib/bash/parse_options.sh index a3e4beb4..5324585b 100644 --- a/lib/bash/parse_options.sh +++ b/lib/bash/parse_options.sh @@ -468,7 +468,10 @@ _parse_command_line() { else spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1) if [ -z "$spec" ]; then - option_error "Unknown option: $real_opt" + # Not all programs uses the same options and since these options can be stored + # in a common config file, we need to skip general options not used by a particular + # program + # option_error "Unknown option: $real_opt" continue fi fi diff --git a/t/lib/TableParser.t b/t/lib/TableParser.t index 57772ada..24d69ce9 100644 --- a/t/lib/TableParser.t +++ b/t/lib/TableParser.t @@ -1267,6 +1267,14 @@ is_deeply( 'Column having the word "generated" as part of the comment is OK', ) or diag Data::Dumper::Dumper($tbl); +$tbl = $tp->parse( load_file('t/lib/samples/generated_cols_comments.sql') ); +warn Data::Dumper::Dumper($tbl); + +is_deeply( + $tbl, + {}, + 'pt-1728', +); # ############################################################################# # Done. # ############################################################################# diff --git a/t/pt-table-checksum/pt-1728.t b/t/pt-table-checksum/pt-1728.t new file mode 100644 index 00000000..6b0bf67a --- /dev/null +++ b/t/pt-table-checksum/pt-1728.t @@ -0,0 +1,103 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More; + +use PerconaTest; +use Sandbox; +use SqlModes; +use threads; +use Time::HiRes qw( usleep ); +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +require "$trunk/bin/pt-table-checksum"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $dbh = $sb->get_dbh_for('master'); + +if ( !$dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} +else { + plan tests => 2; +} + +my $num_rows = 1_000_000; +my $table = 't1'; + +$dbh->do("DROP DATABASE IF EXISTS test"); +$dbh->do("CREATE DATABASE IF NOT EXISTS test"); +$dbh->do("CREATE TABLE `test`.`$table` (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(5)) Engine=InnoDB"); + +diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test $table $num_rows`); + +sub start_thread { + my ($dsn_opts, $initial_sleep_time) = @_; + my $dp = new DSNParser(opts=>$dsn_opts); + my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); + my $master_dbh = $sb->get_dbh_for('master'); + my $slave_dbh = $sb->get_dbh_for('slave1'); + diag("Sleeping"); + sleep($initial_sleep_time); + diag("Woke up"); + $slave_dbh->do("STOP SLAVE IO_THREAD FOR CHANNEL ''"); + $slave_dbh->do("STOP SLAVE"); + $master_dbh->do("TRUNCATE TABLE test.$table"); + # PTDEBUG && diag("Exit thread") + sleep(2); + $slave_dbh->do("START SLAVE"); + diag("Exit thread") +} +# This is not a realiable sleep value. It works for a i7, hybrid HDD +my $initial_sleep_time = 17; +my $thr = threads->create('start_thread', $dsn_opts, $initial_sleep_time); +threads->yield(); + +diag("Starting checksum"); +# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic +# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die. +# And --max-load "" prevents waiting for status variables. +my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox'; +my @args = ($master_dsn, qw(--no-check-binlog-format --chunk-size 10)); +my $output; + +$output = output( + sub { pt_table_checksum::main(@args) }, + stderr => 1, +); + +diag($output); +unlike( + $output, + qr/Can't use an undefined value as an ARRAY/, + "Truncating tables while checksum is running" +); + +$sb->load_file('master', 't/pt-table-checksum/samples/pt-1728.sql'); +@args = ($master_dsn, qw(--no-check-binlog-format)); +my $new_rows_count = $num_rows * 5; +diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test $table $new_rows_count`); +$output = output( + sub { pt_table_checksum::main(@args) }, + stderr => 1, +); + +diag($output); + +$thr->join(); + +# ############################################################################# +# Done. +# ############################################################################# +$sb->wipe_clean($dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +exit;