diff --git a/bin/pt-query-digest b/bin/pt-query-digest index 0a2bc528..0b1b71b5 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -2082,24 +2082,26 @@ use Time::Local qw(timegm timelocal); use Digest::MD5 qw(md5_hex); use B qw(); -require Exporter; -our @ISA = qw(Exporter); -our %EXPORT_TAGS = (); -our @EXPORT = (); -our @EXPORT_OK = qw( - micro_t - percentage_of - secs_to_time - time_to_secs - shorten - ts - parse_timestamp - unix_timestamp - any_unix_timestamp - make_checksum - crc32 - encode_json -); +BEGIN { + require Exporter; + our @ISA = qw(Exporter); + our %EXPORT_TAGS = (); + our @EXPORT = (); + our @EXPORT_OK = qw( + micro_t + percentage_of + secs_to_time + time_to_secs + shorten + ts + parse_timestamp + unix_timestamp + any_unix_timestamp + make_checksum + crc32 + encode_json + ); +} our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/; our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/; @@ -7977,10 +7979,50 @@ package JSONReportFormatter; use Mo; use JSON; +use Transformers qw(make_checksum); + use constant PTDEBUG => $ENV{PTDEBUG} || 0; extends qw(QueryReportFormatter); +has history_metrics => ( + is => 'ro', + isa => 'HashRef', +); + +sub BUILDARGS { + my $class = shift; + my %orig_args = @_; + my $args = $class->SUPER::BUILDARGS(@_); + + my $o = $orig_args{OptionParser}; + + my $sql = $o->read_para_after( + __FILE__, qr/MAGIC_create_review_history/); + + my $pat = $o->read_para_after(__FILE__, qr/MAGIC_history_cols/); + $pat = qr/\ {3}(\w+?)_($pat)\s+/; + + my %metrics; + foreach my $sql_line (split /\n/, $sql) { + my ( $attr, $metric ) = $sql_line =~ $pat; + next unless $attr && $metric; + + $attr = ucfirst $attr if $attr =~ m/_/; + $attr = 'Filesort' if $attr eq 'filesort'; + + $attr =~ s/^Qc_hit/QC_Hit/; # Qc_hit is really QC_Hit + $attr =~ s/^Innodb/InnoDB/g; # Innodb is really InnoDB + $attr =~ s/_io_/_IO_/g; # io is really IO + + $metrics{$attr}{$metric} = 1; + } + + $args->{history_metrics} = \%metrics; + + return $args; +} + override [qw(rusage date hostname files header profile prepared)] => sub { return; }; @@ -7995,59 +8037,41 @@ override query_report => sub { foreach my $arg ( qw(ea worst orderby groupby) ) { die "I need a $arg argument" unless defined $arg; } - my $ea = $args{ea}; - my $groupby = $args{groupby}; - my $worst = $args{worst}; - my $q = $self->Quoter; - my $qv = $self->{QueryReview}; - my $qr = $self->{QueryRewriter}; + my $ea = $args{ea}; + my $worst = $args{worst}; - my $query_report_vals = $self->query_report_values(%args); + my $history_metrics = $self->history_metrics; + my @attribs = grep { $history_metrics->{$_} } @{$ea->get_attributes()}; - my $attribs = $self->sort_attribs( - ($args{select} ? $args{select} : $ea->get_attributes()), - $ea, - ); + my @queries; + foreach my $worst_info ( @$worst ) { + my $item = $worst_info->[0]; + my $stats = $ea->results->{classes}->{$item}; + my $sample = $ea->results->{samples}->{$item}; - ITEM: - foreach my $vals ( @$query_report_vals ) { - my $item = $vals->{item}; - my $samp_query = $vals->{samp_query}; - $vals->{event_report} = $self->event_report( - %args, - item => $item, - sample => $ea->results->{samples}->{$item}, - rank => $vals->{rank}, - reason => $vals->{reason}, - attribs => $attribs, - db => $vals->{default_db}, - ); + my %metrics; + foreach my $attrib ( @attribs ) { + $metrics{$attrib} = $ea->metrics( + attrib => $attrib, + where => $item, + ); - if ( $groupby eq 'fingerprint' ) { - if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { - if ( $item !~ m/^(?:insert|replace)/ ) { # No EXPLAIN - $vals->{for_explain} = "EXPLAIN /*!50100 PARTITIONS*/\n$samp_query\\G\n"; - $vals->{explain_report} = $self->explain_report($samp_query, $vals->{default_db}); - } - } - else { - my $converted = $qr->convert_to_select($samp_query); - if ( $converted - && $converted =~ m/^[\(\s]*select/i ) { - $vals->{for_explain} = "EXPLAIN /*!50100 PARTITIONS*/\n$converted\\G\n"; - } - } - } - else { - if ( $groupby eq 'tables' ) { - my ( $db, $tbl ) = $q->split_unquote($item); - $vals->{tables_report} = $self->tables_report([$db, $tbl]); + my $needed_metrics = $history_metrics->{$attrib}; + for my $key ( keys %{$metrics{$attrib}} ) { + delete $metrics{$attrib}{$key} + unless $needed_metrics->{$key}; } } + + push @queries, { + sample => $sample, + checksum => make_checksum($item), + %metrics + }; } - return encode_json($query_report_vals) . "\n"; + return encode_json(\@queries) . "\n"; }; 1; @@ -11776,8 +11800,6 @@ use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; -use Fcntl qw(:seek); - sub new { my ( $class, %args ) = @_; my $self = { @@ -11820,9 +11842,6 @@ sub get_file_itr { } open my $fh, '<', $fn or warn "Cannot open $fn: $OS_ERROR"; if ( $fh ) { - if ( my $pos = $self->{resume}->{$fn} ) { - seek $fh, $pos, SEEK_SET; - } return ( $fh, $fn, -s $fn ); } } diff --git a/lib/JSONReportFormatter.pm b/lib/JSONReportFormatter.pm index af3e2546..bac1194c 100644 --- a/lib/JSONReportFormatter.pm +++ b/lib/JSONReportFormatter.pm @@ -3,10 +3,50 @@ package JSONReportFormatter; use Mo; use JSON; +use Transformers qw(make_checksum); + use constant PTDEBUG => $ENV{PTDEBUG} || 0; extends qw(QueryReportFormatter); +has history_metrics => ( + is => 'ro', + isa => 'HashRef', +); + +sub BUILDARGS { + my $class = shift; + my %orig_args = @_; + my $args = $class->SUPER::BUILDARGS(@_); + + my $o = $orig_args{OptionParser}; + + my $sql = $o->read_para_after( + __FILE__, qr/MAGIC_create_review_history/); + + my $pat = $o->read_para_after(__FILE__, qr/MAGIC_history_cols/); + $pat = qr/\ {3}(\w+?)_($pat)\s+/; + + my %metrics; + foreach my $sql_line (split /\n/, $sql) { + my ( $attr, $metric ) = $sql_line =~ $pat; + next unless $attr && $metric; + + $attr = ucfirst $attr if $attr =~ m/_/; + $attr = 'Filesort' if $attr eq 'filesort'; + + $attr =~ s/^Qc_hit/QC_Hit/; # Qc_hit is really QC_Hit + $attr =~ s/^Innodb/InnoDB/g; # Innodb is really InnoDB + $attr =~ s/_io_/_IO_/g; # io is really IO + + $metrics{$attr}{$metric} = 1; + } + + $args->{history_metrics} = \%metrics; + + return $args; +} + override [qw(rusage date hostname files header profile prepared)] => sub { return; }; @@ -21,63 +61,41 @@ override query_report => sub { foreach my $arg ( qw(ea worst orderby groupby) ) { die "I need a $arg argument" unless defined $arg; } - my $ea = $args{ea}; - my $groupby = $args{groupby}; - my $worst = $args{worst}; - my $q = $self->Quoter; - my $qv = $self->{QueryReview}; - my $qr = $self->{QueryRewriter}; + my $ea = $args{ea}; + my $worst = $args{worst}; - my $query_report_vals = $self->query_report_values(%args); + my $history_metrics = $self->history_metrics; + my @attribs = grep { $history_metrics->{$_} } @{$ea->get_attributes()}; - # Sort the attributes, removing any hidden attributes. - my $attribs = $self->sort_attribs( - ($args{select} ? $args{select} : $ea->get_attributes()), - $ea, - ); + my @queries; + foreach my $worst_info ( @$worst ) { + my $item = $worst_info->[0]; + my $stats = $ea->results->{classes}->{$item}; + my $sample = $ea->results->{samples}->{$item}; - ITEM: - foreach my $vals ( @$query_report_vals ) { - my $item = $vals->{item}; - my $samp_query = $vals->{samp_query}; - # ############################################################### - # Print the standard query analysis report. - # ############################################################### - $vals->{event_report} = $self->event_report( - %args, - item => $item, - sample => $ea->results->{samples}->{$item}, - rank => $vals->{rank}, - reason => $vals->{reason}, - attribs => $attribs, - db => $vals->{default_db}, - ); + my %metrics; + foreach my $attrib ( @attribs ) { + $metrics{$attrib} = $ea->metrics( + attrib => $attrib, + where => $item, + ); - if ( $groupby eq 'fingerprint' ) { - if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { - if ( $item !~ m/^(?:insert|replace)/ ) { # No EXPLAIN - $vals->{for_explain} = "EXPLAIN /*!50100 PARTITIONS*/\n$samp_query\\G\n"; - $vals->{explain_report} = $self->explain_report($samp_query, $vals->{default_db}); - } - } - else { - my $converted = $qr->convert_to_select($samp_query); - if ( $converted - && $converted =~ m/^[\(\s]*select/i ) { - $vals->{for_explain} = "EXPLAIN /*!50100 PARTITIONS*/\n$converted\\G\n"; - } - } - } - else { - if ( $groupby eq 'tables' ) { - my ( $db, $tbl ) = $q->split_unquote($item); - $vals->{tables_report} = $self->tables_report([$db, $tbl]); + my $needed_metrics = $history_metrics->{$attrib}; + for my $key ( keys %{$metrics{$attrib}} ) { + delete $metrics{$attrib}{$key} + unless $needed_metrics->{$key}; } } + + push @queries, { + sample => $sample, + checksum => make_checksum($item), + %metrics + }; } - return encode_json($query_report_vals) . "\n"; + return encode_json(\@queries) . "\n"; }; 1;