percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-12-21 02:00:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update pqd so it works, sends actual new data struct.

Browse Source
This commit is contained in:
Daniel Nichter
2013-06-06 13:37:16 -05:00
parent 912d648922
commit f583ec6113
3 changed files with 68 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
bin/pt-query-digest
View File

@@ -7708,6 +7708,8 @@ override query_report => sub {
my $global_data = { my $global_data = {
metrics => {}, metrics => {},
files => $args{files},
($args{resume} && scalar keys %{$args{resume}} ? (resume => $args{resume}) : ()),
}; };
my $global_cnt = $results->{globals}->{$orderby}->{cnt} || 0; my $global_cnt = $results->{globals}->{$orderby}->{cnt} || 0;
@@ -7739,6 +7741,7 @@ override query_report => sub {
ts => 1, ts => 1,
bytes => 1, bytes => 1,
); );
foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) { foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) {
my $type = $ea->type_for($attrib) || 'string'; my $type = $ea->type_for($attrib) || 'string';
next if $type eq 'string'; next if $type eq 'string';
@@ -7751,43 +7754,42 @@ override query_report => sub {
if ( $type eq 'num' ) { if ( $type eq 'num' ) {
foreach my $m ( qw(sum min max) ) { foreach my $m ( qw(sum min max) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%d', $store->{$m} || 0); = sprintf('%d', $store->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%.6f', $store->{$m} || 0); = sprintf('%.6f', $store->{$m} || 0);
} }
} }
foreach my $m ( qw(pct_95 stddev median) ) { foreach my $m ( qw(pct_95 stddev median) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%d', $metrics->{$m} || 0); = sprintf('%d', $metrics->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%.6f', $metrics->{$m} || 0); = sprintf('%.6f', $metrics->{$m} || 0);
} }
} }
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib . "_avg"} $global_data->{metrics}->{$attrib}->{avg}
= sprintf('%d', $store->{sum} / $store->{cnt}); = sprintf('%d', $store->{sum} / $store->{cnt});
} }
else { else {
$global_data->{metrics}->{$attrib . "_avg"} $global_data->{metrics}->{$attrib}->{avg}
= sprintf('%.6f', $store->{sum} / $store->{cnt}); = sprintf('%.6f', $store->{sum} / $store->{cnt});
} }
} }
elsif ( $type eq 'bool' ) { elsif ( $type eq 'bool' ) {
my $store = $results->{globals}->{$attrib}; my $store = $results->{globals}->{$attrib};
$global_data->{metrics}->{$attrib . "_cnt"} $global_data->{metrics}->{$attrib}->{cnt}
= sprintf('%d', $store->{sum}); = sprintf('%d', $store->{sum});
} }
} }
my @queries; my @classes;
foreach my $worst_info ( @$worst ) { foreach my $worst_info ( @$worst ) {
my $item = $worst_info->[0]; my $item = $worst_info->[0];
my $stats = $ea->results->{classes}->{$item}; my $stats = $ea->results->{classes}->{$item};
@@ -7799,17 +7801,18 @@ override query_report => sub {
my $distill = $groupby eq 'fingerprint' ? $qr->distill($sample->{arg}) my $distill = $groupby eq 'fingerprint' ? $qr->distill($sample->{arg})
: undef; : undef;
my %class = ( my $checksum = make_checksum($item);
attribute => $groupby, my $class = {
checksum => $checksum,
fingerprint => $item, fingerprint => $item,
checksum => make_checksum($item),
distillate => $distill, distillate => $distill,
attribute => $groupby,
query_count => $times_seen, query_count => $times_seen,
example => { example => {
query => $sample->{arg}, query => $sample->{arg},
ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
}, },
); };
my %metrics; my %metrics;
foreach my $attrib ( @attribs ) { foreach my $attrib ( @attribs ) {
@@ -7833,8 +7836,8 @@ override query_report => sub {
next unless defined $ts && defined $ts->{$thing}; next unless defined $ts && defined $ts->{$thing};
$ts->{$thing} = parse_timestamp($ts->{$thing}); $ts->{$thing} = parse_timestamp($ts->{$thing});
} }
$class{ts_min} = $ts->{min}; $class->{ts_min} = $ts->{min};
$class{ts_max} = $ts->{max}; $class->{ts_max} = $ts->{max};
} }
else { else {
my $type = $ea->type_for($attrib) || 'string'; my $type = $ea->type_for($attrib) || 'string';
@@ -7860,7 +7863,7 @@ override query_report => sub {
} }
} }
my $copy_paste; my @tables;
if ( $groupby eq 'fingerprint' ) { if ( $groupby eq 'fingerprint' ) {
my $default_db = $sample->{db} ? $sample->{db} my $default_db = $sample->{db} ? $sample->{db}
: $stats->{db}->{unq} ? keys %{$stats->{db}->{unq}} : $stats->{db}->{unq} ? keys %{$stats->{db}->{unq}}
@@ -7870,7 +7873,6 @@ override query_report => sub {
default_db => $default_db, default_db => $default_db,
Quoter => $q, Quoter => $q,
); );
my @tables;
foreach my $db_tbl ( @table_names ) { foreach my $db_tbl ( @table_names ) {
my ( $db, $tbl ) = @$db_tbl; my ( $db, $tbl ) = @$db_tbl;
my $status my $status
@@ -7883,9 +7885,6 @@ override query_report => sub {
. "\\G"; . "\\G";
push @tables, { status => $status, create => $create }; push @tables, { status => $status, create => $create };
} }
if ( @tables ) {
$copy_paste->{tables} = \@tables;
}
if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
if ( $item =~ m/^(?:insert|replace)/ ) { if ( $item =~ m/^(?:insert|replace)/ ) {
@@ -7899,24 +7898,22 @@ override query_report => sub {
$sample->{arg} || '', $sample->{arg} || '',
); );
if ( $converted && $converted =~ m/^[\(\s]*select/i ) { if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
$copy_paste->{explain} = $converted; $class->{example}->{as_select} = $converted;
} }
} }
} }
push @queries, { $class->{metrics} = \%metrics;
class => \%class, if ( @tables ) {
attributes => \%metrics, $class->{tables} = \@tables;
($copy_paste ? (copy_paste => $copy_paste) : ()), }
push @classes, $class;
}
my $data = {
global => $global_data,
classes => \@classes,
}; };
}
# XXX
# my $data = {
# global => $global_data,
# classes => \@queries,
# };
my $data = \@queries;
my $json = $self->encode_json($data); my $json = $self->encode_json($data);
$json .= "\n" unless $json =~ /\n\Z/; $json .= "\n" unless $json =~ /\n\Z/;
return $json; return $json;
@@ -12940,7 +12937,7 @@ sub main {
if ( $fh ) { if ( $fh ) {
PTDEBUG && _d('Reading', $filename); PTDEBUG && _d('Reading', $filename);
PTDEBUG && _d('File size:', $filesize); PTDEBUG && _d('File size:', $filesize);
push @read_files, $filename || "STDIN"; push @read_files, { name => ($filename || "STDIN"), size => $filesize };
# Read the file offset for --resume. # Read the file offset for --resume.
if ( ($resume_file = $o->get('resume')) && $filename ) { if ( ($resume_file = $o->get('resume')) && $filename ) {
@@ -13419,6 +13416,10 @@ sub main {
if ( $report ) { if ( $report ) {
PTDEBUG && _d("Iteration", $args->{iter}, "stopped at",ts(time)); PTDEBUG && _d("Iteration", $args->{iter}, "stopped at",ts(time));
save_resume_offset(
last_event_offset => $parsers[0]->{last_event_offset},
);
# Get this before calling print_reports() because that sub # Get this before calling print_reports() because that sub
# resets each ea and we may need this later for stats. # resets each ea and we may need this later for stats.
my $n_events_aggregated = $ea[0]->events_processed(); my $n_events_aggregated = $ea[0]->events_processed();
@@ -14042,7 +14043,8 @@ sub print_reports {
files => $args{files}, files => $args{files},
log_type => $o->get('type')->[0], log_type => $o->get('type')->[0],
variations => $o->get('variations'), variations => $o->get('variations'),
group => { map { $_=>1 } qw(rusage date hostname files header) } group => { map { $_=>1 } qw(rusage date hostname files header) },
resume => $resume,
); );
} }

59
lib/JSONReportFormatter.pm
View File

@@ -104,6 +104,8 @@ override query_report => sub {
# ######################################################################## # ########################################################################
my $global_data = { my $global_data = {
metrics => {}, metrics => {},
files => $args{files},
($args{resume} && scalar keys %{$args{resume}} ? (resume => $args{resume}) : ()),
}; };
# Get global count # Get global count
@@ -137,6 +139,7 @@ override query_report => sub {
ts => 1, ts => 1,
bytes => 1, bytes => 1,
); );
foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) { foreach my $attrib ( grep { !$hidden_attrib{$_} } @attribs ) {
my $type = $ea->type_for($attrib) || 'string'; my $type = $ea->type_for($attrib) || 'string';
next if $type eq 'string'; next if $type eq 'string';
@@ -149,36 +152,36 @@ override query_report => sub {
if ( $type eq 'num' ) { if ( $type eq 'num' ) {
foreach my $m ( qw(sum min max) ) { foreach my $m ( qw(sum min max) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%d', $store->{$m} || 0); = sprintf('%d', $store->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%.6f', $store->{$m} || 0); = sprintf('%.6f', $store->{$m} || 0);
} }
} }
foreach my $m ( qw(pct_95 stddev median) ) { foreach my $m ( qw(pct_95 stddev median) ) {
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%d', $metrics->{$m} || 0); = sprintf('%d', $metrics->{$m} || 0);
} }
else { # microsecond else { # microsecond
$global_data->{metrics}->{$attrib . "_$m"} $global_data->{metrics}->{$attrib}->{$m}
= sprintf('%.6f', $metrics->{$m} || 0); = sprintf('%.6f', $metrics->{$m} || 0);
} }
} }
if ( $int ) { if ( $int ) {
$global_data->{metrics}->{$attrib . "_avg"} $global_data->{metrics}->{$attrib}->{avg}
= sprintf('%d', $store->{sum} / $store->{cnt}); = sprintf('%d', $store->{sum} / $store->{cnt});
} }
else { else {
$global_data->{metrics}->{$attrib . "_avg"} $global_data->{metrics}->{$attrib}->{avg}
= sprintf('%.6f', $store->{sum} / $store->{cnt}); = sprintf('%.6f', $store->{sum} / $store->{cnt});
} }
} }
elsif ( $type eq 'bool' ) { elsif ( $type eq 'bool' ) {
my $store = $results->{globals}->{$attrib}; my $store = $results->{globals}->{$attrib};
$global_data->{metrics}->{$attrib . "_cnt"} $global_data->{metrics}->{$attrib}->{cnt}
= sprintf('%d', $store->{sum}); = sprintf('%d', $store->{sum});
} }
} }
@@ -187,8 +190,7 @@ override query_report => sub {
# Query class data # Query class data
# ######################################################################## # ########################################################################
my @queries; my @classes;
foreach my $worst_info ( @$worst ) { foreach my $worst_info ( @$worst ) {
my $item = $worst_info->[0]; my $item = $worst_info->[0];
my $stats = $ea->results->{classes}->{$item}; my $stats = $ea->results->{classes}->{$item};
@@ -201,17 +203,18 @@ override query_report => sub {
my $distill = $groupby eq 'fingerprint' ? $qr->distill($sample->{arg}) my $distill = $groupby eq 'fingerprint' ? $qr->distill($sample->{arg})
: undef; : undef;
my %class = ( my $checksum = make_checksum($item);
attribute => $groupby, my $class = {
checksum => $checksum,
fingerprint => $item, fingerprint => $item,
checksum => make_checksum($item),
distillate => $distill, distillate => $distill,
attribute => $groupby,
query_count => $times_seen, query_count => $times_seen,
example => { example => {
query => $sample->{arg}, query => $sample->{arg},
ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef, ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
}, },
); };
my %metrics; my %metrics;
foreach my $attrib ( @attribs ) { foreach my $attrib ( @attribs ) {
@@ -235,8 +238,8 @@ override query_report => sub {
next unless defined $ts && defined $ts->{$thing}; next unless defined $ts && defined $ts->{$thing};
$ts->{$thing} = parse_timestamp($ts->{$thing}); $ts->{$thing} = parse_timestamp($ts->{$thing});
} }
$class{ts_min} = $ts->{min}; $class->{ts_min} = $ts->{min};
$class{ts_max} = $ts->{max}; $class->{ts_max} = $ts->{max};
} }
else { else {
my $type = $ea->type_for($attrib) || 'string'; my $type = $ea->type_for($attrib) || 'string';
@@ -282,17 +285,17 @@ override query_report => sub {
# #
# The formatting isn't included, just the useful data, like: # The formatting isn't included, just the useful data, like:
# #
# $copy_paste = { # $tables = [
# tables => { # {
# create => "SHOW CREATE TABLE db.foo", # create => "SHOW CREATE TABLE db.foo",
# status => "SHOW TABLE STATUS FROM db LIKE foo", # status => "SHOW TABLE STATUS FROM db LIKE foo",
# }, # },
# explain => "select ..." # explain => "select ..."
# } # ]
# #
# This is called "copy-paste" because users can copy-paste these # This is called "copy-paste" because users can copy-paste these
# ready-made lines into MySQL. # ready-made lines into MySQL.
my $copy_paste; my @tables;
if ( $groupby eq 'fingerprint' ) { if ( $groupby eq 'fingerprint' ) {
# Get SHOW CREATE TABLE and SHOW TABLE STATUS. # Get SHOW CREATE TABLE and SHOW TABLE STATUS.
my $default_db = $sample->{db} ? $sample->{db} my $default_db = $sample->{db} ? $sample->{db}
@@ -303,7 +306,6 @@ override query_report => sub {
default_db => $default_db, default_db => $default_db,
Quoter => $q, Quoter => $q,
); );
my @tables;
foreach my $db_tbl ( @table_names ) { foreach my $db_tbl ( @table_names ) {
my ( $db, $tbl ) = @$db_tbl; my ( $db, $tbl ) = @$db_tbl;
my $status my $status
@@ -316,9 +318,6 @@ override query_report => sub {
. "\\G"; . "\\G";
push @tables, { status => $status, create => $create }; push @tables, { status => $status, create => $create };
} }
if ( @tables ) {
$copy_paste->{tables} = \@tables;
}
# Convert possible non-SELECTs for EXPLAIN. # Convert possible non-SELECTs for EXPLAIN.
if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) { if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
@@ -339,16 +338,16 @@ override query_report => sub {
$sample->{arg} || '', $sample->{arg} || '',
); );
if ( $converted && $converted =~ m/^[\(\s]*select/i ) { if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
$copy_paste->{explain} = $converted; $class->{example}->{as_select} = $converted;
} }
} }
} }
push @queries, { $class->{metrics} = \%metrics;
class => \%class, if ( @tables ) {
attributes => \%metrics, $class->{tables} = \@tables;
($copy_paste ? (copy_paste => $copy_paste) : ()), }
}; push @classes, $class;
} }
# ######################################################################## # ########################################################################
@@ -356,7 +355,7 @@ override query_report => sub {
# ######################################################################## # ########################################################################
my $data = { my $data = {
global => $global_data, global => $global_data,
classes => \@queries, classes => \@classes,
}; };
my $json = $self->encode_json($data); my $json = $self->encode_json($data);
$json .= "\n" unless $json =~ /\n\Z/; $json .= "\n" unless $json =~ /\n\Z/;

2
lib/QueryReportFormatter.pm
View File

@@ -208,7 +208,7 @@ sub hostname {
sub files { sub files {
my ( $self, %args ) = @_; my ( $self, %args ) = @_;
if ( $args{files} ) { if ( $args{files} ) {
return "# Files: " . join(', ', @{$args{files}}) . "\n"; return "# Files: " . join(', ', map { $_->{name} } @{$args{files}}) . "\n";
} }
return; return;
} }