mirror of
https://github.com/percona/percona-toolkit.git
synced 2025-09-10 05:00:45 +00:00
Merged raw-log-parser-percona-22371
This commit is contained in:
Brian Fraser fraserb@gmail.com
@@ -9756,6 +9756,86 @@ sub _d {
|
||||
# End GeneralLogParser package
|
||||
# ###########################################################################
|
||||
|
||||
# ###########################################################################
|
||||
# RawLogParser package
|
||||
# This package is a copy without comments from the original. The original
|
||||
# with comments and its test file can be found in the Bazaar repository at,
|
||||
# lib/RawLogParser.pm
|
||||
# t/lib/RawLogParser.t
|
||||
# See https://launchpad.net/percona-toolkit for more information.
|
||||
# ###########################################################################
|
||||
{
|
||||
package RawLogParser;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
|
||||
|
||||
use Data::Dumper;
|
||||
$Data::Dumper::Indent = 1;
|
||||
$Data::Dumper::Sortkeys = 1;
|
||||
$Data::Dumper::Quotekeys = 0;
|
||||
|
||||
sub new {
|
||||
my ( $class ) = @_;
|
||||
my $self = {
|
||||
};
|
||||
return bless $self, $class;
|
||||
}
|
||||
|
||||
sub parse_event {
|
||||
my ( $self, %args ) = @_;
|
||||
my @required_args = qw(next_event tell);
|
||||
foreach my $arg ( @required_args ) {
|
||||
die "I need a $arg argument" unless $args{$arg};
|
||||
}
|
||||
my ($next_event, $tell) = @args{@required_args};
|
||||
|
||||
my $line;
|
||||
my $pos_in_log = $tell->();
|
||||
LINE:
|
||||
while ( defined($line = $next_event->()) ) {
|
||||
PTDEBUG && _d($line);
|
||||
chomp($line);
|
||||
my @properties = (
|
||||
'pos_in_log', $pos_in_log,
|
||||
'cmd', 'Query',
|
||||
'bytes', length($line),
|
||||
'Query_time', 0,
|
||||
'arg', $line,
|
||||
);
|
||||
|
||||
$pos_in_log = $tell->();
|
||||
|
||||
PTDEBUG && _d('Properties of event:', Dumper(\@properties));
|
||||
my $event = { @properties };
|
||||
if ( $args{stats} ) {
|
||||
$args{stats}->{events_read}++;
|
||||
$args{stats}->{events_parsed}++;
|
||||
}
|
||||
|
||||
return $event;
|
||||
}
|
||||
|
||||
$args{oktorun}->(0) if $args{oktorun};
|
||||
return;
|
||||
}
|
||||
|
||||
sub _d {
|
||||
my ($package, undef, $line) = caller 0;
|
||||
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
|
||||
map { defined $_ ? $_ : 'undef' }
|
||||
@_;
|
||||
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
|
||||
}
|
||||
|
||||
1;
|
||||
}
|
||||
# ###########################################################################
|
||||
# End RawLogParser package
|
||||
# ###########################################################################
|
||||
|
||||
# ###########################################################################
|
||||
# ProtocolParser package
|
||||
# This package is a copy without comments from the original. The original
|
||||
@@ -13227,7 +13307,7 @@ sub main {
|
||||
my $review_dsn = $o->get('review');
|
||||
my @groupby = @{$o->get('group-by')};
|
||||
my @orderby;
|
||||
if ( (grep { $_ eq 'genlog' || $_ eq 'GeneralLogParser' } @{$o->get('type')})
|
||||
if ( (grep { $_ =~ m/genlog|GeneralLogParser|rawlog|RawLogParser/ } @{$o->get('type')})
|
||||
&& !$o->got('order-by') ) {
|
||||
@orderby = 'Query_time:cnt';
|
||||
}
|
||||
@@ -13668,6 +13748,7 @@ sub main {
|
||||
'MemcachedEvent'],
|
||||
http => ['TcpdumpParser','HTTPProtocolParser'],
|
||||
pglog => ['PgLogParser'],
|
||||
rawlog => ['RawLogParser'],
|
||||
);
|
||||
my $type = $o->get('type');
|
||||
$type = $alias_for{$type->[0]} if $alias_for{$type->[0]};
|
||||
|
||||
95
lib/RawLogParser.pm
Normal file
95
lib/RawLogParser.pm
Normal file
@@ -0,0 +1,95 @@
|
||||
# This program is copyright 2012 Percona Inc.
|
||||
# Feedback and improvements are welcome.
|
||||
#
|
||||
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
|
||||
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
|
||||
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it under
|
||||
# the terms of the GNU General Public License as published by the Free Software
|
||||
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
|
||||
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
|
||||
# licenses.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
|
||||
# Place, Suite 330, Boston, MA 02111-1307 USA.
|
||||
# ###########################################################################
|
||||
# RawLogParser package
|
||||
# ###########################################################################
|
||||
{
|
||||
# Package: RawLogParser
|
||||
# RawLogParser parses logs with nothing but one query per line.
|
||||
package RawLogParser;
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
|
||||
|
||||
use Data::Dumper;
|
||||
$Data::Dumper::Indent = 1;
|
||||
$Data::Dumper::Sortkeys = 1;
|
||||
$Data::Dumper::Quotekeys = 0;
|
||||
|
||||
sub new {
|
||||
my ( $class ) = @_;
|
||||
my $self = {
|
||||
};
|
||||
return bless $self, $class;
|
||||
}
|
||||
|
||||
sub parse_event {
|
||||
my ( $self, %args ) = @_;
|
||||
my @required_args = qw(next_event tell);
|
||||
foreach my $arg ( @required_args ) {
|
||||
die "I need a $arg argument" unless $args{$arg};
|
||||
}
|
||||
my ($next_event, $tell) = @args{@required_args};
|
||||
|
||||
my $line;
|
||||
my $pos_in_log = $tell->();
|
||||
LINE:
|
||||
while ( defined($line = $next_event->()) ) {
|
||||
PTDEBUG && _d($line);
|
||||
chomp($line);
|
||||
my @properties = (
|
||||
'pos_in_log', $pos_in_log,
|
||||
'cmd', 'Query',
|
||||
'bytes', length($line),
|
||||
'Query_time', 0,
|
||||
'arg', $line,
|
||||
);
|
||||
|
||||
$pos_in_log = $tell->();
|
||||
|
||||
# Don't dump $event; want to see full dump of all properties,
|
||||
# and after it's been cast into a hash, duplicated keys will
|
||||
# be gone.
|
||||
PTDEBUG && _d('Properties of event:', Dumper(\@properties));
|
||||
my $event = { @properties };
|
||||
if ( $args{stats} ) {
|
||||
$args{stats}->{events_read}++;
|
||||
$args{stats}->{events_parsed}++;
|
||||
}
|
||||
|
||||
return $event;
|
||||
}
|
||||
|
||||
$args{oktorun}->(0) if $args{oktorun};
|
||||
return;
|
||||
}
|
||||
|
||||
sub _d {
|
||||
my ($package, undef, $line) = caller 0;
|
||||
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
|
||||
map { defined $_ ? $_ : 'undef' }
|
||||
@_;
|
||||
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
|
||||
}
|
||||
|
||||
1;
|
||||
}
|
||||
# ###########################################################################
|
||||
# End RawLogParser package
|
||||
# ###########################################################################
|
||||
52
t/lib/RawLogParser.t
Normal file
52
t/lib/RawLogParser.t
Normal file
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
BEGIN {
|
||||
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
|
||||
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
|
||||
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
|
||||
};
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use Test::More tests => 3;
|
||||
|
||||
use RawLogParser;
|
||||
use PerconaTest;
|
||||
|
||||
my $p = new RawLogParser();
|
||||
|
||||
my $oktorun = 1;
|
||||
my $sample = "t/lib/samples/rawlogs/";
|
||||
|
||||
test_log_parser(
|
||||
parser => $p,
|
||||
file => $sample.'rawlog001.txt',
|
||||
oktorun => sub { $oktorun = $_[0]; },
|
||||
result => [
|
||||
{ pos_in_log => 0,
|
||||
args => 'SELECT c FROM t WHERE id=1',
|
||||
bytes => 26,
|
||||
cmd => 'Query',
|
||||
Query_time => 0,
|
||||
},
|
||||
{ pos_in_log => 27,
|
||||
args => '/* Hello, world! */ SELECT * FROM t2 LIMIT 1',
|
||||
bytes => 44,
|
||||
cmd => 'Query',
|
||||
Query_time => 0,
|
||||
}
|
||||
]
|
||||
);
|
||||
|
||||
is(
|
||||
$oktorun,
|
||||
0,
|
||||
'Sets oktorun'
|
||||
);
|
||||
$oktorun = 1;
|
||||
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
exit;
|
||||
2
t/lib/samples/rawlogs/rawlog001.txt
Normal file
2
t/lib/samples/rawlogs/rawlog001.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
SELECT c FROM t WHERE id=1
|
||||
/* Hello, world! */ SELECT * FROM t2 LIMIT 1
|
||||
48
t/pt-query-digest/rawlog_analyses.t
Normal file
48
t/pt-query-digest/rawlog_analyses.t
Normal file
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env perl
|
||||
|
||||
BEGIN {
|
||||
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
|
||||
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
|
||||
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
|
||||
};
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use English qw(-no_match_vars);
|
||||
use Test::More tests => 2;
|
||||
|
||||
use PerconaTest;
|
||||
|
||||
# See 101_slowlog_analyses.t or http://code.google.com/p/maatkit/wiki/Testing
|
||||
shift @INC; # our unshift (above)
|
||||
shift @INC; # PerconaTest's unshift
|
||||
|
||||
require "$trunk/bin/pt-query-digest";
|
||||
|
||||
# #############################################################################
|
||||
# Issue 172: Make mk-query-digest able to read raweral logs
|
||||
# #############################################################################
|
||||
|
||||
my @args = ('--report-format', 'header,query_report,profile', '--type', 'rawlog');
|
||||
my $sample = "$trunk/t/lib/samples/rawlogs/";
|
||||
|
||||
# --help exists so don't run mqd as a module else --help's exit will
|
||||
# exit this test script.
|
||||
like(
|
||||
`$trunk/bin/pt-query-digest --type rawlog rawlog001.txt --help`,
|
||||
qr/--order-by\s+Query_time:cnt/,
|
||||
'--order-by defaults to Query_time:cnt for --type rawlog',
|
||||
);
|
||||
|
||||
ok(
|
||||
no_diff(
|
||||
sub { pt_query_digest::main(@args, $sample.'rawlog001.txt') },
|
||||
"t/pt-query-digest/samples/rawlog001.txt"
|
||||
),
|
||||
'Analysis for rawlog001',
|
||||
);
|
||||
|
||||
# #############################################################################
|
||||
# Done.
|
||||
# #############################################################################
|
||||
exit;
|
||||
60
t/pt-query-digest/samples/rawlog001.txt
Normal file
60
t/pt-query-digest/samples/rawlog001.txt
Normal file
@@ -0,0 +1,60 @@
|
||||
|
||||
# Overall: 2 total, 2 unique, 0 QPS, 0x concurrency ______________________
|
||||
# Attribute total min max avg 95% stddev median
|
||||
# ============ ======= ======= ======= ======= ======= ======= =======
|
||||
# Exec time 0 0 0 0 0 0 0
|
||||
# Query size 70 26 44 35 44 12.73 35
|
||||
|
||||
# Query 1: 0 QPS, 0x concurrency, ID 0xCB5621E548E5497F at byte 0 ________
|
||||
# This item is included in the report because it matches --limit.
|
||||
# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00
|
||||
# Query_time sparkline: | |
|
||||
# Attribute pct total min max avg 95% stddev median
|
||||
# ============ === ======= ======= ======= ======= ======= ======= =======
|
||||
# Count 50 1
|
||||
# Exec time 0 0 0 0 0 0 0 0
|
||||
# Query size 37 26 26 26 26 26 0 26
|
||||
# Query_time distribution
|
||||
# 1us
|
||||
# 10us
|
||||
# 100us
|
||||
# 1ms
|
||||
# 10ms
|
||||
# 100ms
|
||||
# 1s
|
||||
# 10s+
|
||||
# Tables
|
||||
# SHOW TABLE STATUS LIKE 't'\G
|
||||
# SHOW CREATE TABLE `t`\G
|
||||
# EXPLAIN /*!50100 PARTITIONS*/
|
||||
SELECT c FROM t WHERE id=1\G
|
||||
|
||||
# Query 2: 0 QPS, 0x concurrency, ID 0x774B2B0B59EBAC2C at byte 27 _______
|
||||
# This item is included in the report because it matches --limit.
|
||||
# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00
|
||||
# Query_time sparkline: | |
|
||||
# Attribute pct total min max avg 95% stddev median
|
||||
# ============ === ======= ======= ======= ======= ======= ======= =======
|
||||
# Count 50 1
|
||||
# Exec time 0 0 0 0 0 0 0 0
|
||||
# Query size 62 44 44 44 44 44 0 44
|
||||
# Query_time distribution
|
||||
# 1us
|
||||
# 10us
|
||||
# 100us
|
||||
# 1ms
|
||||
# 10ms
|
||||
# 100ms
|
||||
# 1s
|
||||
# 10s+
|
||||
# Tables
|
||||
# SHOW TABLE STATUS LIKE 't2'\G
|
||||
# SHOW CREATE TABLE `t2`\G
|
||||
# EXPLAIN /*!50100 PARTITIONS*/
|
||||
/* Hello, world! */ SELECT * FROM t2 LIMIT 1\G
|
||||
|
||||
# Profile
|
||||
# Rank Query ID Response time Calls R/Call Apdx V/M Item
|
||||
# ==== ================== ============= ===== ====== ==== ===== =========
|
||||
# 1 0xCB5621E548E5497F 0.0000 0.0% 1 0.0000 1.00 0.00 SELECT t
|
||||
# 2 0x774B2B0B59EBAC2C 0.0000 0.0% 1 0.0000 1.00 0.00 SELECT t?
|
||||
Reference in New Issue
Block a user