From 24119553a209f28b9c4e2f82bde052a2246f25f6 Mon Sep 17 00:00:00 2001 From: Frantzcy Date: Mon, 28 Nov 2016 12:26:46 -0500 Subject: [PATCH 01/18] point is not decimal prevent field type 'point' to be taken as decimal (poINT) --- bin/pt-table-sync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pt-table-sync b/bin/pt-table-sync index b56ee929..90157859 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -2866,7 +2866,7 @@ sub parse { my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/; die "Can't determine column type for $def" unless $type; $type_for{$col} = $type; - if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) { + if ( $type =~ m/(?:(?:tiny|big|medium|small)?\bint|float|double|decimal|year)/ ) { push @nums, $col; $is_numeric{$col} = 1; } From 46a21a91aaa0e0e746ef72e34aebea136c418761 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Wed, 1 Feb 2017 11:04:19 -0300 Subject: [PATCH 02/18] Updated readme --- README | 1 - 1 file changed, 1 deletion(-) diff --git a/README b/README index 4857e41b..53a34665 100644 --- a/README +++ b/README @@ -34,4 +34,3 @@ to read the embedded documentation for a specific tool. You can also read the documentation online at [http://www.percona.com/software/percona-toolkit/](http://www.percona.com/software/percona-toolkit/). - From 298969e57f9b2f5c39ca65dc027453987e15fe74 Mon Sep 17 00:00:00 2001 From: Moritz Lenz Date: Thu, 25 Jan 2018 13:51:23 +0100 Subject: [PATCH 03/18] Only set binlog_format when necessary The documentation states that --sync-to-master or --replicate need the binlog_format STATEMENT, but the code set the binlog format regardless of the operation mode. With this patch, the binlog_format is only set if --sync-to-master or --replicate have actually been supplied on the command line, enabling a node-to-node sync without these options without the SUPER privilege. --- bin/pt-table-sync | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/pt-table-sync b/bin/pt-table-sync index 67d3d6dc..feba005f 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -10957,7 +10957,8 @@ sub get_cxn { # instead, it should check if it's already set to STATEMENT. # This is becase starting with MySQL 5.1.29, changing the format # requires a SUPER user. - if ( VersionParser->new($dbh) >= '5.1.29' ) { + if ( VersionParser->new($dbh) >= '5.1.29' + && ($o->get('replicate') || $o->get('sync-to-master'))) { $sql = 'SELECT @@binlog_format'; PTDEBUG && _d($dbh, $sql); my ($original_binlog_format) = $dbh->selectrow_array($sql); From 3bcdf913fbb94a0ba909bd0ce6c1685e2637c75d Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 13 Feb 2018 09:30:55 +0200 Subject: [PATCH 04/18] Adding --read-only-interval flag, and read-only check on wake-up --- bin/pt-heartbeat | 172 ++++++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 76 deletions(-) diff --git a/bin/pt-heartbeat b/bin/pt-heartbeat index e577d886..32ffcf83 100755 --- a/bin/pt-heartbeat +++ b/bin/pt-heartbeat @@ -112,22 +112,22 @@ use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; -sub check_recursion_method { +sub check_recursion_method { my ($methods) = @_; - if ( @$methods != 1 ) { - if ( grep({ !m/processlist|hosts/i } @$methods) - && $methods->[0] !~ /^dsn=/i ) - { - die "Invalid combination of recursion methods: " - . join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". " - . "Only hosts and processlist may be combined.\n" - } - } - else { + if ( @$methods != 1 ) { + if ( grep({ !m/processlist|hosts/i } @$methods) + && $methods->[0] !~ /^dsn=/i ) + { + die "Invalid combination of recursion methods: " + . join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". " + . "Only hosts and processlist may be combined.\n" + } + } + else { my ($method) = @$methods; - die "Invalid recursion method: " . ( $method || 'undef' ) - unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i; - } + die "Invalid recursion method: " . ( $method || 'undef' ) + unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i; + } } sub new { @@ -156,7 +156,7 @@ sub get_slaves { my $methods = $self->_resolve_recursion_methods($args{dsn}); return $slaves unless @$methods; - + if ( grep { m/processlist|hosts/i } @$methods ) { my @required_args = qw(dbh dsn); foreach my $arg ( @required_args ) { @@ -200,7 +200,7 @@ sub get_slaves { else { die "Unexpected recursion methods: @$methods"; } - + return $slaves; } @@ -668,7 +668,7 @@ sub short_host { } sub is_replication_thread { - my ( $self, $query, %args ) = @_; + my ( $self, $query, %args ) = @_; return unless $query; my $type = lc($args{type} || 'all'); @@ -683,7 +683,7 @@ sub is_replication_thread { if ( !$match ) { if ( ($query->{User} || $query->{user} || '') eq "system user" ) { PTDEBUG && _d("Slave replication thread"); - if ( $type ne 'all' ) { + if ( $type ne 'all' ) { my $state = $query->{State} || $query->{state} || ''; if ( $state =~ m/^init|end$/ ) { @@ -696,7 +696,7 @@ sub is_replication_thread { |Reading\sevent\sfrom\sthe\srelay\slog |Has\sread\sall\srelay\slog;\swaiting |Making\stemp\sfile - |Waiting\sfor\sslave\smutex\son\sexit)/xi; + |Waiting\sfor\sslave\smutex\son\sexit)/xi; $match = $type eq 'slave_sql' && $slave_sql ? 1 : $type eq 'slave_io' && !$slave_sql ? 1 @@ -760,7 +760,7 @@ sub get_replication_filters { replicate_do_db replicate_ignore_db replicate_do_table - replicate_ignore_table + replicate_ignore_table replicate_wild_do_table replicate_wild_ignore_table ); @@ -771,7 +771,7 @@ sub get_replication_filters { $filters{slave_skip_errors} = $row->[1] if $row->[1] && $row->[1] ne 'OFF'; } - return \%filters; + return \%filters; } @@ -905,7 +905,7 @@ sub new { rules => [], # desc of rules for --help mutex => [], # rule: opts are mutually exclusive atleast1 => [], # rule: at least one opt is required - disables => {}, # rule: opt disables other opts + disables => {}, # rule: opt disables other opts defaults_to => {}, # rule: opt defaults to value of other opt DSNParser => undef, default_files => [ @@ -1068,7 +1068,7 @@ sub _pod_to_specs { } push @specs, { - spec => $self->{parse_attributes}->($self, $option, \%attribs), + spec => $self->{parse_attributes}->($self, $option, \%attribs), desc => $para . (defined $attribs{default} ? " (default $attribs{default})" : ''), group => ($attribs{'group'} ? $attribs{'group'} : 'default'), @@ -1159,7 +1159,7 @@ sub _parse_specs { $self->{opts}->{$long} = $opt; } else { # It's an option rule, not a spec. - PTDEBUG && _d('Parsing rule:', $opt); + PTDEBUG && _d('Parsing rule:', $opt); push @{$self->{rules}}, $opt; my @participants = $self->_get_participants($opt); my $rule_ok = 0; @@ -1204,7 +1204,7 @@ sub _parse_specs { PTDEBUG && _d('Option', $long, 'disables', @participants); } - return; + return; } sub _get_participants { @@ -1291,7 +1291,7 @@ sub _set_option { } sub get_opts { - my ( $self ) = @_; + my ( $self ) = @_; foreach my $long ( keys %{$self->{opts}} ) { $self->{opts}->{$long}->{got} = 0; @@ -1422,7 +1422,7 @@ sub _check_opts { else { $err = join(', ', map { "--$self->{opts}->{$_}->{long}" } - grep { $_ } + grep { $_ } @restricted_opts[0..scalar(@restricted_opts) - 2] ) . ' or --'.$self->{opts}->{$restricted_opts[-1]}->{long}; @@ -1432,7 +1432,7 @@ sub _check_opts { } } - elsif ( $opt->{is_required} ) { + elsif ( $opt->{is_required} ) { $self->save_error("Required option --$long must be specified"); } @@ -1808,7 +1808,7 @@ sub clone { $clone{$scalar} = $self->{$scalar}; } - return bless \%clone; + return bless \%clone; } sub _parse_size { @@ -2322,7 +2322,7 @@ sub extends { sub _load_module { my ($class) = @_; - + (my $file = $class) =~ s{::|'}{/}g; $file .= '.pm'; { local $@; eval { require "$file" } } # or warn $@; @@ -2353,7 +2353,7 @@ sub has { my $caller = scalar caller(); my $class_metadata = Lmo::Meta->metadata_for($caller); - + for my $attribute ( ref $names ? @$names : $names ) { my %args = @_; my $method = ($args{is} || '') eq 'ro' @@ -2372,16 +2372,16 @@ sub has { if ( my $type_check = $args{isa} ) { my $check_name = $type_check; - + if ( my ($aggregate_type, $inner_type) = $type_check =~ /\A(ArrayRef|Maybe)\[(.*)\]\z/ ) { $type_check = Lmo::Types::_nested_constraints($attribute, $aggregate_type, $inner_type); } - + my $check_sub = sub { my ($new_val) = @_; Lmo::Types::check_type_constaints($attribute, $type_check, $check_name, $new_val); }; - + $class_metadata->{$attribute}{isa} = [$check_name, $check_sub]; my $orig_method = $method; $method = sub { @@ -2823,7 +2823,7 @@ sub get_dbh { my $dbh; my $tries = 2; while ( !$dbh && $tries-- ) { - PTDEBUG && _d($cxn_string, ' ', $user, ' ', $pass, + PTDEBUG && _d($cxn_string, ' ', $user, ' ', $pass, join(', ', map { "$_=>$defaults->{$_}" } keys %$defaults )); $dbh = eval { DBI->connect($cxn_string, $user, $pass, $defaults) }; @@ -2999,7 +2999,7 @@ sub set_vars { } } - return; + return; } sub _d { @@ -3086,7 +3086,7 @@ sub daemonize { close STDERR; open STDERR, ">&STDOUT" - or die "Cannot dupe STDERR to STDOUT: $OS_ERROR"; + or die "Cannot dupe STDERR to STDOUT: $OS_ERROR"; } else { if ( -t STDOUT ) { @@ -3277,7 +3277,7 @@ sub split_unquote { s/`\z//; s/``/`/g; } - + return ($db, $tbl); } @@ -4164,7 +4164,7 @@ sub value_to_json { my $b_obj = B::svref_2object(\$value); # for round trip problem my $flags = $b_obj->FLAGS; - return $value # as is + return $value # as is if $flags & ( B::SVp_IOK | B::SVp_NOK ) and !( $flags & B::SVp_POK ); # SvTYPE is IV or NV? my $type = ref($value); @@ -4448,7 +4448,7 @@ sub _split_url { or die(qq/SSL certificate not valid for $host\n/); } } - + $self->{host} = $host; $self->{port} = $port; @@ -4922,7 +4922,7 @@ eval { } PTDEBUG && _d('Version check file', $file, 'in', $ENV{PWD}); return $file; # in the CWD - } + } } sub version_check_time_limit { @@ -4939,11 +4939,11 @@ sub version_check { PTDEBUG && _d('FindBin::Bin:', $FindBin::Bin); if ( !$args{force} ) { if ( $FindBin::Bin - && (-d "$FindBin::Bin/../.bzr" || + && (-d "$FindBin::Bin/../.bzr" || -d "$FindBin::Bin/../../.bzr" || - -d "$FindBin::Bin/../.git" || - -d "$FindBin::Bin/../../.git" - ) + -d "$FindBin::Bin/../.git" || + -d "$FindBin::Bin/../../.git" + ) ) { PTDEBUG && _d("$FindBin::Bin/../.bzr disables --version-check"); return; @@ -4967,7 +4967,7 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); @@ -5153,7 +5153,7 @@ sub pingback { ); die "Failed to parse server requested programs: $response->{content}" if !scalar keys %$items; - + my $versions = get_versions( items => $items, instances => $instances, @@ -5412,7 +5412,7 @@ sub get_from_mysql { if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { @{$item->{vars}} = grep { $_ eq 'version' || $_ eq 'version_comment' } @{$item->{vars}}; } - + my @versions; my %version_for; @@ -5674,7 +5674,7 @@ sub main { # Reset all global vars between test runs else weird things happen. @dbhs = (); - @sths = (); + @sths = (); # ######################################################################## # Get configuration information. @@ -5697,7 +5697,7 @@ sub main { $o->save_error("Invalid --frames argument"); } else { - push @times, + push @times, $suf eq 's' ? $num # Seconds : $suf eq 'm' ? $num * 60 # Minutes : $suf eq 'h' ? $num * 3600 # Hours @@ -5791,9 +5791,9 @@ sub main { $dbh->do("USE `$db`"); # ######################################################################## - # If --check-read-only option was given and we are in --update mode + # If --check-read-only option was given and we are in --update mode # we wait until server is writable , or run-time is over, or sentinel file - # We also do this check after daemon is up and running, but it is necessary + # We also do this check after daemon is up and running, but it is necessary # to check this before attempting to create the table and inserting rows # https://bugs.launchpad.net/percona-toolkit/+bug/1328686 # ####################################################################### @@ -5802,12 +5802,13 @@ sub main { if ( server_is_readonly($dbh) && PTDEBUG ) { _d('Server is read-only, waiting') } - my $start_time = time; - my $run_time = $o->get('run-time'); - my $interval = $o->get('interval') || 5; + my $start_time = time; + my $run_time = $o->get('run-time'); + my $interval = $o->get('interval') || 5; + my $read_only_interval = $o->get('read-only-interval') || $interval; while (server_is_readonly($dbh)) { - sleep($interval); - if ( + sleep($read_only_interval); + if ( ($run_time && $run_time < time - $start_time) || -f $sentinel ) { @@ -5837,7 +5838,7 @@ sub main { if ( $EVAL_ERROR ) { die "Error creating heartbeat table:". $EVAL_ERROR; } - + # Now we insert first row. # Some caveats: @@ -5851,18 +5852,18 @@ sub main { . qq/ INTO $db_tbl (ts, server_id) VALUES ($now_func, $server_id)/; # 2) - # RBR (Row Based Replication) converts REPLACE to INSERT if row isn't + # RBR (Row Based Replication) converts REPLACE to INSERT if row isn't # present in master. This breakes replication when the row is present in slave. - # Other workarounds also fail. + # Other workarounds also fail. # INSERT IGNORE (ignore is not replicated if no error in master) - # DELETE then INSERT (DELETE is ignored, INSERT breaks replication) + # DELETE then INSERT (DELETE is ignored, INSERT breaks replication) # INSERT ON DUPLICATE UPDATE (converts to simple INSERT) - # TRUNCATE gets trough and replicates! So we use that to wipe slave(s). + # TRUNCATE gets trough and replicates! So we use that to wipe slave(s). if ($o->get('replace')) { my $sql_truncate = "TRUNCATE TABLE $db_tbl"; PTDEBUG && _d($sql_truncate); eval { $dbh->do($sql_truncate) }; - } + } PTDEBUG && _d($sql_insert_row); eval { $dbh->do($sql_insert_row); }; } @@ -6018,8 +6019,8 @@ sub main { my @vals; return if $ro_check && server_is_readonly($dbh); - - my $sql; + + my $sql; if ( @master_status_cols ) { $sql = "SHOW MASTER STATUS"; PTDEBUG && _d($dbh, $sql); @@ -6032,7 +6033,7 @@ sub main { push @vals, map { $row->{$_} } @master_status_cols; } } - + if ( @slave_status_cols ) { $sql = "SHOW SLAVE STATUS"; PTDEBUG && _d($dbh, $sql); @@ -6070,7 +6071,7 @@ sub main { die $args{error}; } ); - + return; }; } @@ -6106,7 +6107,7 @@ sub main { PTDEBUG && _d('Delay', sprintf('%.6f', $delay), 'on', $hostname); # Because we adjust for skew, if the ts are less than skew seconds - # apart (i.e. replication is very fast) then delay will be negative. + # apart (i.e. replication is very fast) then delay will be negative. # So it's effectively 0 seconds of lag. $delay = 0.00 if $delay < 0; @@ -6151,7 +6152,7 @@ sub main { # ######################################################################## # --check and exit if --check was given. # ######################################################################## - if ( $o->get('check') ) { + if ( $o->get('check') ) { PTDEBUG && _d('--check and exit'); check_delay( dsn => $dsn, @@ -6223,6 +6224,18 @@ sub main { sleep $next_interval - $time; PTDEBUG && _d('Woke up at', ts(time)); + if ( $o->get('check-read-only') && $o->get('update') ) { + my $read_only_interval = $o->get('read-only-interval') || $interval; + while (server_is_readonly($dbh)) { + sleep($read_only_interval); + if ( + -f $sentinel + ) { + return 0; + } + } + } + # Connect or reconnect if necessary. if ( !$dbh->ping() ) { $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), { AutoCommit => 1 }); @@ -6247,7 +6260,7 @@ sub main { } @$frames; my $output = sprintf $format, $delay, @vals, $pk_val; - if ( my $file = $o->get('file') ) { + if ( my $file = $o->get('file') ) { open my $file, '>', $file or die "Can't open $file: $OS_ERROR"; print $file $output @@ -6389,7 +6402,7 @@ sub check_delay { # next interval is or should be. The caller can then sleep(time-next_interval) # to wake up at that interval. If the caller misses the next interval, # they just call the iterator until the next interval is later then the -# current time. +# current time. sub make_interval_iter { my ( $interval, $skew ) = @_; die "I need an interval argument" unless defined $interval; @@ -6549,7 +6562,7 @@ specify the L<"--master-server-id"> to use. For example, if the replication hierarchy is "master -> slave1 -> slave2" with corresponding server IDs 1, 2 and 3, you can: - pt-heartbeat --daemonize -D test --update -h master + pt-heartbeat --daemonize -D test --update -h master pt-heartbeat --daemonize -D test --update -h slave1 Then check (or monitor) the replication delay from master to slave2: @@ -6578,7 +6591,7 @@ Cluster (PXC), we recommend using 5.5.28-23.7 and newer. If you are setting up heartbeat instances between cluster nodes, keep in mind that, since the speed of the cluster is determined by its slowest node, pt-heartbeat will not report how fast the cluster itself is, but only how -fast events are replicating from one node to another. +fast events are replicating from one node to another. You must specify L<"--master-server-id"> for L<"--monitor"> and L<"--check"> instances. @@ -6619,7 +6632,7 @@ before its delay. L<"--recurse"> only works with MySQL. =item --check-read-only Check if the server has read_only enabled; If it does, the tool skips doing -any inserts. +any inserts. See also L<"--read-only-interval"> =item --config @@ -6640,7 +6653,7 @@ be created with the following MAGIC_create_heartbeat table definition: server_id int unsigned NOT NULL PRIMARY KEY, file varchar(255) DEFAULT NULL, -- SHOW MASTER STATUS position bigint unsigned DEFAULT NULL, -- SHOW MASTER STATUS - relay_master_log_file varchar(255) DEFAULT NULL, -- SHOW SLAVE STATUS + relay_master_log_file varchar(255) DEFAULT NULL, -- SHOW SLAVE STATUS exec_master_log_pos bigint unsigned DEFAULT NULL -- SHOW SLAVE STATUS ); @@ -6835,6 +6848,13 @@ Print the auto-detected or given L<"--master-server-id">. If L<"--check"> or L<"--monitor"> is specified, specifying this option will print the auto-detected or given L<"--master-server-id"> at the end of each line. +=item --read-only-interval + +type: int + +When L<"--check-read-only"> is specified, the interval to sleep while the +server is found to be read-only. If unspecified, L<"--interval"> is used. + =item --recurse type: int @@ -7023,7 +7043,7 @@ as 5.5.25a. Any updates or known problems are printed to STDOUT before the tool's normal output. This feature should never interfere with the normal operation of the -tool. +tool. For more information, visit L. From 14889da924a82fb1309ee1888f467548224b61e7 Mon Sep 17 00:00:00 2001 From: Nick Veenhof Date: Sun, 25 Feb 2018 17:02:29 +0100 Subject: [PATCH 05/18] pt-summary does not reliably read in the transparent huge pages setting --- bin/pt-summary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pt-summary b/bin/pt-summary index 4ef26179..e9a2277b 100755 --- a/bin/pt-summary +++ b/bin/pt-summary @@ -2259,7 +2259,7 @@ report_transparent_huge_pages () { STATUS_THP_SYSTEM="" if [ -f /sys/kernel/mm/transparent_hugepage/enabled ]; then - CONTENT_TRANSHP=$( Date: Mon, 26 Feb 2018 09:35:08 +0200 Subject: [PATCH 06/18] --fail-successive-errors --- bin/pt-heartbeat | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bin/pt-heartbeat b/bin/pt-heartbeat index 32ffcf83..c456d01c 100755 --- a/bin/pt-heartbeat +++ b/bin/pt-heartbeat @@ -6205,6 +6205,8 @@ sub main { PTDEBUG && _d($end ? ('Will exit at', ts($end)) : 'Running forever'); my $get_next_interval = make_interval_iter($interval, $skew); + my $max_successive_errors = $o->get('fail-successive-errors') || 0; + my $num_successive_errors = 0; while ( # Stop if... (!$end || int(time) < $end) # runtime exceeded, or @@ -6278,6 +6280,7 @@ sub main { } }; if ( $EVAL_ERROR ) { + $num_successive_errors = $num_successive_errors + 1; my ( $err ) = $EVAL_ERROR =~ m/^(?:DBI|DBD).*failed: (.*?)\s*at \S+ line .*/; if ( $err ) { warn "$err\n"; @@ -6285,6 +6288,12 @@ sub main { else { die $EVAL_ERROR; } + if ($max_successive_errors > 0 && $num_successive_errors >= $max_successive_errors) { + die $EVAL_ERROR; + } + } + else { + $num_successive_errors = 0; } } @@ -6819,6 +6828,13 @@ L<"--frames">. For example, 5s [ 0.25s, 0.05s, 0.02s ] +=item --fail-successive-errors + +type: int + +If specified, pt-heartbeat will fail after given number of successive DBI errors +(failure to connect to server or issue a query). + =item --password short form: -p; type: string From e8eb3679f80958fbff30562278e49dc6fe9528b1 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 20 Mar 2018 07:27:25 +0200 Subject: [PATCH 07/18] compat with 3.0 --- bin/pt-table-sync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pt-table-sync b/bin/pt-table-sync index 2812f146..90589da7 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -2866,7 +2866,7 @@ sub parse { my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/; die "Can't determine column type for $def" unless $type; $type_for{$col} = $type; - if ( $type =~ m/(?:(?:tiny|big|medium|small)?\bint|float|double|decimal|year)/ ) { + if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) { push @nums, $col; $is_numeric{$col} = 1; } From e69507f45effcb2de86d7140fd61ea3796fa7f19 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 20 Mar 2018 07:51:31 +0200 Subject: [PATCH 08/18] README conflict --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 53a34665..bac15eff 100644 --- a/README +++ b/README @@ -22,7 +22,7 @@ perl Makefile.PL make make test make install -``` +``` You probably need to be root to `make install`. On most systems, the tools are installed in /usr/local/bin. See the INSTALL file for more information. From 2e061461d0ad8f1f60df5fac1aced988c26f385f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 20 Mar 2018 07:52:46 +0200 Subject: [PATCH 09/18] README conflict --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index bac15eff..09acddd8 100644 --- a/README +++ b/README @@ -34,3 +34,4 @@ to read the embedded documentation for a specific tool. You can also read the documentation online at [http://www.percona.com/software/percona-toolkit/](http://www.percona.com/software/percona-toolkit/). + From 79c8c4b1b827d40ca00b5bb40028169980105870 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 20 Mar 2018 07:53:26 +0200 Subject: [PATCH 10/18] README conflict --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 09acddd8..4857e41b 100644 --- a/README +++ b/README @@ -22,7 +22,7 @@ perl Makefile.PL make make test make install -``` +``` You probably need to be root to `make install`. On most systems, the tools are installed in /usr/local/bin. See the INSTALL file for more information. From 998b37f3c0367f09c21a20533532baee7c9df6c0 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Mon, 26 Mar 2018 10:24:46 -0300 Subject: [PATCH 11/18] Fixed type in pt-table-sync test and updated changelog --- Changelog | 4 ++++ t/pt-table-sync/pt-1256.t | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog b/Changelog index 0ad1ac36..5f92ad67 100644 --- a/Changelog +++ b/Changelog @@ -1,5 +1,9 @@ Changelog for Percona Toolkit +v3.0.9 + + * Feature PT-1509 : Only set binlog_format when necessary (Thanks Moritz Lenz) + v3.0.8 released 2018-03-13 * Feature PT-1500 : add --secure-slowlog option to pt-query digest diff --git a/t/pt-table-sync/pt-1256.t b/t/pt-table-sync/pt-1256.t index d6458dff..3fedfb2c 100644 --- a/t/pt-table-sync/pt-1256.t +++ b/t/pt-table-sync/pt-1256.t @@ -79,7 +79,7 @@ is( "Character set is correct", ); -SKIP { +SKIP: { skip "Skipping in MySQL 8.0.4-rc since there is an error in the server itself", 1 if ($sandbox_version ge '8.0'); # 3 $output = `$trunk/bin/pt-table-sync --execute --lock-and-rename h=127.1,P=12345,u=msandbox,p=msandbox,D=test,t=t1 t=t2 2>&1`; From 9eaf5e217a7de4202403614295c58720f2009127 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Mon, 26 Mar 2018 16:23:32 -0300 Subject: [PATCH 12/18] PT-1508 Added test & updated changelog --- Changelog | 1 + bin/pt-heartbeat | 2 + t/pt-heartbeat/pt-1508.t | 94 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 t/pt-heartbeat/pt-1508.t diff --git a/Changelog b/Changelog index 5f92ad67..eac3a72e 100644 --- a/Changelog +++ b/Changelog @@ -3,6 +3,7 @@ Changelog for Percona Toolkit v3.0.9 * Feature PT-1509 : Only set binlog_format when necessary (Thanks Moritz Lenz) + * Feature PT-1508 : Adding --read-only-interval flag, and read-only check on wake-up (Thanks Shlomi Noach) v3.0.8 released 2018-03-13 diff --git a/bin/pt-heartbeat b/bin/pt-heartbeat index d27cebdf..c4c76764 100755 --- a/bin/pt-heartbeat +++ b/bin/pt-heartbeat @@ -5925,6 +5925,7 @@ sub main { my $interval = $o->get('interval') || 5; my $read_only_interval = $o->get('read-only-interval') || $interval; while (server_is_readonly($dbh)) { + PTDEBUG && _d("Sleeping for $read_only_interval seconds"); sleep($read_only_interval); if ( ($run_time && $run_time < time - $start_time) @@ -6347,6 +6348,7 @@ sub main { if ( $o->get('check-read-only') && $o->get('update') ) { my $read_only_interval = $o->get('read-only-interval') || $interval; while (server_is_readonly($dbh)) { + PTDEBUG && _d("Server is read only. Sleeping for $read_only_interval seconds..."); sleep($read_only_interval); if ( -f $sentinel diff --git a/t/pt-heartbeat/pt-1508.t b/t/pt-heartbeat/pt-1508.t new file mode 100644 index 00000000..d032a411 --- /dev/null +++ b/t/pt-heartbeat/pt-1508.t @@ -0,0 +1,94 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use threads ('yield'); + +use English qw(-no_match_vars); +use Test::More; + +use Data::Dumper; +use PerconaTest; +use Sandbox; +use SqlModes; +use File::Temp qw/ tempfile /; + +plan tests => 2; + +require "$trunk/bin/pt-heartbeat"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $master_dbh = $sb->get_dbh_for('master'); +my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox'; + +my ($tfh, $pid_file) = tempfile(); +close($tfh); +unlink($pid_file); + +my $slave1_dbh = $sb->get_dbh_for('slave1'); +my $slave1_dsn = 'h=127.1,P=12346,u=unprivileged,p=password'; + +if ( !$master_dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} + +sub start_thread { + my ($dsn_opts, $sleep, $pid_file) = @_; + + my $dp = new DSNParser(opts=>$dsn_opts); + my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); + my $dbh= $sb->get_dbh_for('slave1'); + diag("Thread started"); + + warn "Sleeping $sleep seconds"; + sleep($sleep); + $dbh->do("UNLOCK TABLES"); + $dbh->do("SET GLOBAL read_only = 0;"); +} + +my $create_table_sql = <<__EOQ; + CREATE TABLE IF NOT EXISTS sakila.heartbeat ( + ts varchar(26) NOT NULL, + server_id int unsigned NOT NULL PRIMARY KEY, + file varchar(255) DEFAULT NULL, -- SHOW MASTER STATUS + position bigint unsigned DEFAULT NULL, -- SHOW MASTER STATUS + relay_master_log_file varchar(255) DEFAULT NULL, -- SHOW SLAVE STATUS + exec_master_log_pos bigint unsigned DEFAULT NULL -- SHOW SLAVE STATUS + ); +__EOQ + +$sb->do_as_root('master', "$create_table_sql"); +$sb->do_as_root('slave1', 'GRANT SELECT, INSERT, UPDATE, REPLICATION CLIENT ON *.* TO "unprivileged"@"localhost" IDENTIFIED BY "password"'); +$sb->do_as_root('slave1', "FLUSH TABLES WITH READ LOCK;"); +$sb->do_as_root('slave1', "SET GLOBAL read_only = 1;"); + +my $thread = threads->create('start_thread', $dsn_opts, 4, $pid_file); +$thread->detach(); +threads->yield(); + +my $output = `PTDEBUG=1 $trunk/bin/pt-heartbeat --database=sakila --table heartbeat --read-only-interval 2 --check-read-only --run-time 5 --update $slave1_dsn 2>&1`; + +like ( + $output, + qr/Sleeping for 2 seconds/, + 'PT-1508 --read-only-interval', +); + +$master_dbh->do("DROP DATABASE IF EXISTS test"); + +# ############################################################################# +# Done. +# ############################################################################# +$sb->do_as_root('master', 'DROP TABLE IF EXISTS sakila.heartbeat'); +$sb->do_as_root('slave1', 'DROP USER "unprivileged"@"localhost"'); + +$sb->wipe_clean($master_dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +done_testing; From a48a767eb43cd749458de702e8a49d062acdbd5a Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Tue, 27 Mar 2018 14:23:59 -0300 Subject: [PATCH 13/18] PT-243 added --max-hostname-length & max-line-length to pt-query-digest --- bin/pt-query-digest | 49 ++++++++---- lib/QueryReportFormatter.pm | 21 +++-- t/lib/QueryReportFormatter.t | 78 +++++++++++++++++++ .../report014_no_trim.txt | 8 ++ .../report014_trim_12.txt | 8 ++ .../QueryReportFormatter/report015.txt | 6 +- t/lib/samples/slowlogs/slow-pt-243.txt | 16 ++++ t/pt-query-digest/pt-243.t | 34 ++++++++ .../samples/issue_1196-output-5.7.txt | 2 +- .../samples/slow007_explain_1-57.txt | 2 +- .../samples/slow007_explain_2-57.txt | 2 +- 11 files changed, 201 insertions(+), 25 deletions(-) create mode 100644 t/lib/samples/QueryReportFormatter/report014_no_trim.txt create mode 100644 t/lib/samples/QueryReportFormatter/report014_trim_12.txt create mode 100644 t/lib/samples/slowlogs/slow-pt-243.txt create mode 100644 t/pt-query-digest/pt-243.t diff --git a/bin/pt-query-digest b/bin/pt-query-digest index f4ab8424..99cd5881 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -6745,6 +6745,12 @@ sub BUILDARGS { ts => 1, }, }; + if (!defined($self->{max_hostname_length})) { + $self->{max_hostname_length} = MAX_STRING_LENGTH; + } + if (!defined($self->{max_line_length})) { + $self->{max_line_length} = LINE_LENGTH; + } return $self; } @@ -7613,16 +7619,19 @@ sub format_string_list { if ( $str =~ m/(?:\d+\.){3}\d+/ ) { $print_str = $str; # Do not shorten IP addresses. } - elsif ( length $str > MAX_STRING_LENGTH ) { - $print_str = substr($str, 0, MAX_STRING_LENGTH) . '...'; - } - else { + elsif ( $self->{max_hostname_length} > 0 and length $str > $self->{max_hostname_length} ) { + $print_str = substr($str, 0, $self->{max_hostname_length}) . '...'; + } else { $print_str = $str; } my $p = percentage_of($cnt_for->{$str}, $class_cnt); $print_str .= " ($cnt_for->{$str}/$p%)"; - if ( !$show_all->{$attrib} ) { - last if (length $line) + (length $print_str) > LINE_LENGTH - 27; + my $trim_length = LINE_LENGTH; + if ($self->{max_hostname_length} == 0 or $self->{max_hostname_length} > LINE_LENGTH) { + $trim_length = $self->{max_hostname_length}; + } + if ( $self->{max_line_length} > 0 and !$show_all->{$attrib} ) { + last if (length $line) + (length $print_str) > $self->{max_line_length} - 27; } $line .= "$print_str, "; $i++; @@ -14522,13 +14531,15 @@ sub print_reports { ? 'JSONReportFormatter' : 'QueryReportFormatter'; my $qrf = $report_class->new( - dbh => $ep_dbh, - QueryReview => $args{QueryReview}, - QueryRewriter => $args{QueryRewriter}, - OptionParser => $args{OptionParser}, - QueryParser => $args{QueryParser}, - Quoter => $args{Quoter}, - show_all => $show_all, + dbh => $ep_dbh, + QueryReview => $args{QueryReview}, + QueryRewriter => $args{QueryRewriter}, + OptionParser => $args{OptionParser}, + QueryParser => $args{QueryParser}, + Quoter => $args{Quoter}, + show_all => $show_all, + max_hostname_length => $o->get('max-hostname-length'), + max_line_length => $o->get('max-line-length'), ); $qrf->print_reports( @@ -15921,6 +15932,18 @@ type: string Print all output to this file when daemonized. +=item --max-hostname-length + +type: int; default: 10 + +Trim host names in reports to this length. 0=Do not trim host names. + +=item --max-line-length + +type: int; default: 74 + +Trim lines to this length. 0=Do not trim lines. + =item --order-by type: Array; default: Query_time:sum diff --git a/lib/QueryReportFormatter.pm b/lib/QueryReportFormatter.pm index 4dccd6ed..8dd649c9 100644 --- a/lib/QueryReportFormatter.pm +++ b/lib/QueryReportFormatter.pm @@ -138,6 +138,12 @@ sub BUILDARGS { ts => 1, }, }; + if (!defined($self->{max_hostname_length})) { + $self->{max_hostname_length} = MAX_STRING_LENGTH; + } + if (!defined($self->{max_line_length})) { + $self->{max_line_length} = LINE_LENGTH; + } return $self; } @@ -1160,16 +1166,19 @@ sub format_string_list { if ( $str =~ m/(?:\d+\.){3}\d+/ ) { $print_str = $str; # Do not shorten IP addresses. } - elsif ( length $str > MAX_STRING_LENGTH ) { - $print_str = substr($str, 0, MAX_STRING_LENGTH) . '...'; - } - else { + elsif ( $self->{max_hostname_length} > 0 and length $str > $self->{max_hostname_length} ) { + $print_str = substr($str, 0, $self->{max_hostname_length}) . '...'; + } else { $print_str = $str; } my $p = percentage_of($cnt_for->{$str}, $class_cnt); $print_str .= " ($cnt_for->{$str}/$p%)"; - if ( !$show_all->{$attrib} ) { - last if (length $line) + (length $print_str) > LINE_LENGTH - 27; + my $trim_length = LINE_LENGTH; + if ($self->{max_hostname_length} == 0 or $self->{max_hostname_length} > LINE_LENGTH) { + $trim_length = $self->{max_hostname_length}; + } + if ( $self->{max_line_length} > 0 and !$show_all->{$attrib} ) { + last if (length $line) + (length $print_str) > $self->{max_line_length} - 27; } $line .= "$print_str, "; $i++; diff --git a/t/lib/QueryReportFormatter.t b/t/lib/QueryReportFormatter.t index 1f444c0b..c0f4e84d 100644 --- a/t/lib/QueryReportFormatter.t +++ b/t/lib/QueryReportFormatter.t @@ -881,6 +881,84 @@ ok( "IPs not shortened with more" ); +# Don't shorten hostnames +$events = [ + { + cmd => 'Query', + arg => "foo", + Query_time => '8.000652', + host => 'a-really-long-host-name', + }, + { + cmd => 'Query', + arg => "foo", + Query_time => '8.000652', + host => '123.123.123.789', + }, +]; + +$ea = new EventAggregator( + groupby => 'arg', + worst => 'Query_time', + ignore_attributes => [qw(arg cmd)], +); +foreach my $event (@$events) { + $ea->aggregate($event); +} +$ea->calculate_statistical_metrics(); +my $no_trim_qrf = new QueryReportFormatter( + OptionParser => $o, + QueryRewriter => $qr, + QueryParser => $qp, + Quoter => $q, + ExplainAnalyzer => $ex, + max_hostname_length => 0, +); + +$result = $no_trim_qrf->event_report( + ea => $ea, + select => [ qw(Query_time host) ], + item => 'foo', + rank => 1, + orderby => 'Query_time', +); + +ok( + no_diff( + $result, + "t/lib/samples/QueryReportFormatter/report014_no_trim.txt", + cmd_output => 1, + ), + "Hostnames were not trimmed" +); + +$no_trim_qrf = new QueryReportFormatter( + OptionParser => $o, + QueryRewriter => $qr, + QueryParser => $qp, + Quoter => $q, + ExplainAnalyzer => $ex, + max_hostname_length => 12, + max_line_length => 200, +); + +$result = $no_trim_qrf->event_report( + ea => $ea, + select => [ qw(Query_time host) ], + item => 'foo', + rank => 1, + orderby => 'Query_time', +); + +ok( + no_diff( + $result, + "t/lib/samples/QueryReportFormatter/report014_trim_12.txt", + cmd_output => 1, + ), + "Hostnames were not trimmed" +); + $result = $qrf->event_report( ea => $ea, select => [ qw(Query_time host) ], diff --git a/t/lib/samples/QueryReportFormatter/report014_no_trim.txt b/t/lib/samples/QueryReportFormatter/report014_no_trim.txt new file mode 100644 index 00000000..9044b1ae --- /dev/null +++ b/t/lib/samples/QueryReportFormatter/report014_no_trim.txt @@ -0,0 +1,8 @@ +# Item 1: 0 QPS, 0x concurrency, ID 0xEDEF654FCCC4A4D8 at byte 0 _________ +# Scores: V/M = 0.00 +# Attribute pct total min max avg 95% stddev median +# ============ === ======= ======= ======= ======= ======= ======= ======= +# Count 100 2 +# Exec time 100 16s 8s 8s 8s 8s 0 8s +# String: +# Hosts 123.123.123.789 (1/50%)... 1 more diff --git a/t/lib/samples/QueryReportFormatter/report014_trim_12.txt b/t/lib/samples/QueryReportFormatter/report014_trim_12.txt new file mode 100644 index 00000000..d66604cb --- /dev/null +++ b/t/lib/samples/QueryReportFormatter/report014_trim_12.txt @@ -0,0 +1,8 @@ +# Item 1: 0 QPS, 0x concurrency, ID 0xEDEF654FCCC4A4D8 at byte 0 _________ +# Scores: V/M = 0.00 +# Attribute pct total min max avg 95% stddev median +# ============ === ======= ======= ======= ======= ======= ======= ======= +# Count 100 2 +# Exec time 100 16s 8s 8s 8s 8s 0 8s +# String: +# Hosts 123.123.123.789 (1/50%), a-really-lon... (1/50%) diff --git a/t/lib/samples/QueryReportFormatter/report015.txt b/t/lib/samples/QueryReportFormatter/report015.txt index 782b0010..cb559159 100644 --- a/t/lib/samples/QueryReportFormatter/report015.txt +++ b/t/lib/samples/QueryReportFormatter/report015.txt @@ -2,7 +2,7 @@ # Scores: V/M = 0.00 # Attribute pct total min max avg 95% stddev median # ============ === ======= ======= ======= ======= ======= ======= ======= -# Count 100 3 -# Exec time 100 24s 8s 8s 8s 8s 0 8s +# Count 100 2 +# Exec time 100 16s 8s 8s 8s 8s 0 8s # String: -# Hosts 123.123.123.456 (1/33%)... 2 more +# Hosts 123.123.123.789 (1/50%), a-really-l... (1/50%) diff --git a/t/lib/samples/slowlogs/slow-pt-243.txt b/t/lib/samples/slowlogs/slow-pt-243.txt new file mode 100644 index 00000000..9012b618 --- /dev/null +++ b/t/lib/samples/slowlogs/slow-pt-243.txt @@ -0,0 +1,16 @@ +/usr/sbin/mysqld, Version: 5.0.77-percona-b13-log (MySQL Percona Edition (GPL)). started with: +Tcp port: 3306 Unix socket: /var/run/mysqld/mysqld.sock +Time Id Command Argument +# Time: 090311 18:11:50 +# User@Host: root[root] @ alonghotnamelikelocalhost [] +# Thread_id: 47 Schema: +# Query_time: 0.017850 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 Rows_affected: 0 Rows_read: 0 +# QC_Hit: No Full_scan: No Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# administrator command: Refresh; +# User@Host: root[root] @ alonghotnamelikelocalhost [] +# Thread_id: 47 Schema: +# Query_time: 0.000002 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 Rows_affected: 0 Rows_read: 0 +# QC_Hit: No Full_scan: No Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# administrator command: Quit; diff --git a/t/pt-query-digest/pt-243.t b/t/pt-query-digest/pt-243.t new file mode 100644 index 00000000..0c1539be --- /dev/null +++ b/t/pt-query-digest/pt-243.t @@ -0,0 +1,34 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More tests => 1; + +use PerconaTest; + +my $run_with = "$trunk/bin/pt-query-digest --max-hostname-length 0 --max-line-length 100 --report-format=query_report --limit 10 $trunk/t/lib/samples/slowlogs/"; + +# ############################################################################# +# Issue 232: mk-query-digest does not properly handle logs with an empty Schema: +# ############################################################################# +my $output = 'foo'; # clear previous test results +my $cmd = "${run_with}slow-pt-243.txt"; +$output = `$cmd 2>&1`; + +like( + $output, + qr/Hosts\s+alonghotnamelikelocalhost/, + 'Hostname is not being truncated', +); + +# ############################################################################# +# Done. +# ############################################################################# +exit; diff --git a/t/pt-query-digest/samples/issue_1196-output-5.7.txt b/t/pt-query-digest/samples/issue_1196-output-5.7.txt index 20fbec7f..ff7fa7f9 100644 --- a/t/pt-query-digest/samples/issue_1196-output-5.7.txt +++ b/t/pt-query-digest/samples/issue_1196-output-5.7.txt @@ -32,7 +32,7 @@ # Tables # SHOW TABLE STATUS FROM `issue_1196` LIKE 't'\G # SHOW CREATE TABLE `issue_1196`.`t`\G -# EXPLAIN /*!50100 PARTITIONS */ +# EXPLAIN /*!50100 PARTITIONS*/ select t.a, count(*) from t join t t2 using(a) group by 1 order by 2 desc limit 10\G # *************************** 1. row *************************** # id: 1 diff --git a/t/pt-query-digest/samples/slow007_explain_1-57.txt b/t/pt-query-digest/samples/slow007_explain_1-57.txt index 20af54a6..f02b486e 100644 --- a/t/pt-query-digest/samples/slow007_explain_1-57.txt +++ b/t/pt-query-digest/samples/slow007_explain_1-57.txt @@ -28,7 +28,7 @@ # Tables # SHOW TABLE STATUS FROM `food` LIKE 'trees'\G # SHOW CREATE TABLE `food`.`trees`\G -# EXPLAIN /*!50100 PARTITIONS */ +# EXPLAIN /*!50100 PARTITIONS*/ SELECT fruit FROM trees\G # *************************** 1. row *************************** # id: 1 diff --git a/t/pt-query-digest/samples/slow007_explain_2-57.txt b/t/pt-query-digest/samples/slow007_explain_2-57.txt index f2493242..eee6d10d 100644 --- a/t/pt-query-digest/samples/slow007_explain_2-57.txt +++ b/t/pt-query-digest/samples/slow007_explain_2-57.txt @@ -28,7 +28,7 @@ # Tables # SHOW TABLE STATUS FROM `food` LIKE 'trees'\G # SHOW CREATE TABLE `food`.`trees`\G -# EXPLAIN /*!50100 PARTITIONS */ +# EXPLAIN /*!50100 PARTITIONS*/ SELECT fruit FROM trees\G # *************************** 1. row *************************** # id: 1 From 12b0d019a2aa27e41ced0acb55d3897d14b6daa8 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Tue, 27 Mar 2018 14:26:21 -0300 Subject: [PATCH 14/18] PT-243 Updated changelog --- Changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog b/Changelog index eac3a72e..2513238f 100644 --- a/Changelog +++ b/Changelog @@ -4,6 +4,7 @@ v3.0.9 * Feature PT-1509 : Only set binlog_format when necessary (Thanks Moritz Lenz) * Feature PT-1508 : Adding --read-only-interval flag, and read-only check on wake-up (Thanks Shlomi Noach) + * Feature PT-243 : Adding --max-hostname-length and --max-line-length to pt-query-digest v3.0.8 released 2018-03-13 From 2bb4f481dca1c486295b44f9f5aceacbef19e52d Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Wed, 28 Mar 2018 14:24:57 -0300 Subject: [PATCH 15/18] Merged PR #308 and updated changelog --- Changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog b/Changelog index 2513238f..208dd0e2 100644 --- a/Changelog +++ b/Changelog @@ -4,6 +4,7 @@ v3.0.9 * Feature PT-1509 : Only set binlog_format when necessary (Thanks Moritz Lenz) * Feature PT-1508 : Adding --read-only-interval flag, and read-only check on wake-up (Thanks Shlomi Noach) + * Improvement PT-1507 : pt-summary does not reliably read in the transparent huge pages setting (Thanks Nick Veenhof) * Feature PT-243 : Adding --max-hostname-length and --max-line-length to pt-query-digest v3.0.8 released 2018-03-13 From c6a8f843a7288812db8c51448f20d425e40a2142 Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Fri, 30 Mar 2018 16:35:08 -0300 Subject: [PATCH 16/18] PT-1488 Added MySQL 8 roles support to pt-show-grants --- bin/pt-show-grants | 55 +++++++++++++++++++- lib/Sandbox.pm | 2 +- t/pt-show-grants/mysql_8_roles.t | 89 ++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 t/pt-show-grants/mysql_8_roles.t diff --git a/bin/pt-show-grants b/bin/pt-show-grants index c5e53f92..fa5881fd 100755 --- a/bin/pt-show-grants +++ b/bin/pt-show-grants @@ -1922,9 +1922,25 @@ sub main { my $ignore_users = $o->get('ignore'); my $exit_status = 0; + if (my $roles = get_roles($dbh)) { + print "-- Roles\n"; + my $count=0; + + for my $role (@$roles) { + next if ($o->get("skip-unused-roles") && $role->{active} == 0); + $count++; + printf('CREATE ROLE IF NOT EXISTS `%s`;'."\n", $role->{name}); + } + + if ($count == 0) { + print "No active roles found\n"; + } + print "-- End of roles listing\n"; + } + USER: foreach my $u ( @$users ) { - my $user_host = "'$u->{User}'\@'$u->{Host}'"; + my $user_host = "`$u->{User}`\@`$u->{Host}`"; if ( $ignore_users && $ignore_users->{$user_host} ) { PTDEBUG && _d('Ignoring user', $user_host); next USER; @@ -2084,6 +2100,39 @@ sub parse_user { return ( $user, $host ); } +sub get_roles { + my $dbh = shift; + my $query = <<__EOQ; + SELECT DISTINCT user.user AS name, IF(from_user IS NULL,0, 1) AS active + FROM mysql.user + LEFT JOIN mysql.role_edges ON role_edges.from_user=user.user + WHERE `account_locked`='Y' + AND `password_expired`='Y' + AND `authentication_string`='' +__EOQ + PTDEBUG && _d("Getting roles"); + PTDEBUG && _d($query); + my $roles; + eval { $roles = $dbh->selectall_arrayref($query, { Slice => {} }) }; + if ($EVAL_ERROR) { + warn "Cannot list roles: $EVAL_ERROR"; + PTDEBUG && _d("Cannot list roles: $EVAL_ERROR"); + } + return $roles; +} + +sub is_role { + my ($users, $grant) = @_; + foreach my $u ( @$users ) { + my $user_host = "`$u->{User}`\@`$u->{Host}`"; + warn "> user_host: $user_host"; + if ($grant eq $user_host) { + return 1; + } + } + return 0; +} + sub split_grants { my ($grants) = @_; return unless $grants; @@ -2346,6 +2395,10 @@ example, specifying C<--set-vars wait_timeout=500> overrides the defaultvalue of The tool prints a warning and continues if a variable cannot be set. +=item --skip-unused-roles + +When dumping MySQL 8+ roles, skip unused roles. + =item --socket short form: -S; type: string diff --git a/lib/Sandbox.pm b/lib/Sandbox.pm index 474077e7..6ecd069e 100644 --- a/lib/Sandbox.pm +++ b/lib/Sandbox.pm @@ -391,7 +391,7 @@ sub verify_test_data { my @diffs; foreach my $c ( @checksums ) { next unless $c->{checksum}; - if ( $c->{checksum} ne $ref->{$c->{table}}->{checksum} ) { + if ( $ref->{$c->{table}} && $c->{checksum} ne $ref->{$c->{table}}->{checksum} ) { push @diffs, $c->{table}; } } diff --git a/t/pt-show-grants/mysql_8_roles.t b/t/pt-show-grants/mysql_8_roles.t new file mode 100644 index 00000000..5f787628 --- /dev/null +++ b/t/pt-show-grants/mysql_8_roles.t @@ -0,0 +1,89 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More; + +use PerconaTest; +use Sandbox; +use SqlModes; +require "$trunk/bin/pt-show-grants"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $dbh = $sb->get_dbh_for('master'); + +if ( !$dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} elsif ($sandbox_version < '8.0') { + plan skip_all => "There are no roles in this MySQL version. Need MySQL 8.0+"; +} else { + plan tests => 4; +} + +$sb->wipe_clean($dbh); + +my $setup_queries = [ + "CREATE ROLE IF NOT EXISTS 'app_developer', 'app_read', 'app_write', 'tester';", + "GRANT ALL ON test.* TO 'app_developer';", + "GRANT SELECT ON test.* TO 'app_read';", + "GRANT SELECT ON test.* TO 'tester';", + "CREATE USER 'zapp'\@'localhost' IDENTIFIED WITH 'mysql_native_password' BY 'test1234';", + "GRANT 'app_read','app_write' TO 'zapp'\@'localhost';", + "GRANT 'tester' TO 'zapp'\@'localhost' WITH ADMIN OPTION;", + "FLUSH PRIVILEGES", +]; + +my $cleanup_queries = [ + "DROP USER IF EXISTS 'zapp'\@'localhost'", + "DROP ROLE IF EXISTS 'app_developer', 'app_read', 'app_write', 'tester'", + "FLUSH PRIVILEGES", +]; +for my $query(@$setup_queries) { + $sb->do_as_root('master', $query); +} + +my $output; +my $cnf = '/tmp/12345/my.sandbox.cnf'; + +eval { + $output = output( + sub { pt_show_grants::main('-F', $cnf, '--skip-unused-roles'); } + ); +}; +is( + $EVAL_ERROR, + '', + 'Does not die on anonymous user (issue 445)', +); + +like( + $output, + qr/CREATE ROLE IF NOT EXISTS `app_read`;/, + 'Roles has been created' +) or diag($output); + +unlike( + $output, + qr/CREATE ROLE IF NOT EXISTS `app_developer`;/, + 'Unused roles has been skipped' + +); + +for my $query(@$cleanup_queries) { + $sb->do_as_root('master', $query); +} + +# ############################################################################# +# Done. +# ############################################################################# +$sb->wipe_clean($dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +exit; From cec7e7d0527f976985ac4e36eb1f6708f330dc0c Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Fri, 30 Mar 2018 19:31:32 -0300 Subject: [PATCH 17/18] PT-1488 Added a test --- t/pt-show-grants/mysql_8_roles.t | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/t/pt-show-grants/mysql_8_roles.t b/t/pt-show-grants/mysql_8_roles.t index 5f787628..5bb25e59 100644 --- a/t/pt-show-grants/mysql_8_roles.t +++ b/t/pt-show-grants/mysql_8_roles.t @@ -25,7 +25,7 @@ if ( !$dbh ) { } elsif ($sandbox_version < '8.0') { plan skip_all => "There are no roles in this MySQL version. Need MySQL 8.0+"; } else { - plan tests => 4; + plan tests => 49; } $sb->wipe_clean($dbh); @@ -77,6 +77,26 @@ unlike( ); +## Do a cleanup and try to run all the queries from the output +for my $query(@$cleanup_queries) { + $sb->do_as_root('master', $query); +} + +my @lines = split(/\n/, $output); +my $count=0; + +for my $query(@lines) { + next if $query =~ m/^-- /; + $count++; + eval { $sb->do_as_root('master', $query) }; + is( + $EVAL_ERROR, + '', + "Ran query $count from the output of pt-show grants", + ) or diag("Cannot execute query from the output: $query -> $EVAL_ERROR"); +} + +## Cleanup to leave sandbox ready for next test file. for my $query(@$cleanup_queries) { $sb->do_as_root('master', $query); } From cc99db8ddd9a7c0fa6e47d4537aa3291f10c297d Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Fri, 30 Mar 2018 20:08:04 -0300 Subject: [PATCH 18/18] PT-1448 Updated test for pt-show-grant MySQL 8 support --- bin/pt-show-grants | 6 +++--- t/pt-show-grants/basics.t | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/pt-show-grants b/bin/pt-show-grants index fa5881fd..38d78914 100755 --- a/bin/pt-show-grants +++ b/bin/pt-show-grants @@ -1922,7 +1922,8 @@ sub main { my $ignore_users = $o->get('ignore'); my $exit_status = 0; - if (my $roles = get_roles($dbh)) { + my $roles = get_roles($dbh); + if ($roles && scalar @$roles > 0) { print "-- Roles\n"; my $count=0; @@ -1940,7 +1941,7 @@ sub main { USER: foreach my $u ( @$users ) { - my $user_host = "`$u->{User}`\@`$u->{Host}`"; + my $user_host = "'$u->{User}'\@'$u->{Host}'"; if ( $ignore_users && $ignore_users->{$user_host} ) { PTDEBUG && _d('Ignoring user', $user_host); next USER; @@ -2115,7 +2116,6 @@ __EOQ my $roles; eval { $roles = $dbh->selectall_arrayref($query, { Slice => {} }) }; if ($EVAL_ERROR) { - warn "Cannot list roles: $EVAL_ERROR"; PTDEBUG && _d("Cannot list roles: $EVAL_ERROR"); } return $roles; diff --git a/t/pt-show-grants/basics.t b/t/pt-show-grants/basics.t index 21befe41..15c26e33 100644 --- a/t/pt-show-grants/basics.t +++ b/t/pt-show-grants/basics.t @@ -82,8 +82,9 @@ unlike( 'It has no timestamp', ); +# 'mysql.infoschema'@'localhost' user only exists on MySQL 8.0+ $output = output( - sub { pt_show_grants::main('-F', $cnf, '--ignore', 'baron,msandbox,root,root@localhost,user,mysql.session@localhost,mysql.sys@localhost,sys'); } + sub { pt_show_grants::main('-F', $cnf, '--ignore', 'baron,msandbox,root,root@localhost,user,mysql.session@localhost,mysql.sys@localhost,sys,mysql.infoschema@localhost'); } ); unlike( $output, @@ -94,7 +95,7 @@ like( $output, qr/\d\d:\d\d:\d\d\n\z/, 'No output when all users skipped' -); +) or diag($output); # ############################################################################# # pt-show-grant doesn't support column-level grants # https://bugs.launchpad.net/percona-toolkit/+bug/866075