diff --git a/bin/pt-query-digest b/bin/pt-query-digest index 86b9be96..29d2a737 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -9756,6 +9756,86 @@ sub _d { # End GeneralLogParser package # ########################################################################### +# ########################################################################### +# RawLogParser package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/RawLogParser.pm +# t/lib/RawLogParser.t +# See https://launchpad.net/percona-toolkit for more information. +# ########################################################################### +{ +package RawLogParser; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +use Data::Dumper; +$Data::Dumper::Indent = 1; +$Data::Dumper::Sortkeys = 1; +$Data::Dumper::Quotekeys = 0; + +sub new { + my ( $class ) = @_; + my $self = { + }; + return bless $self, $class; +} + +sub parse_event { + my ( $self, %args ) = @_; + my @required_args = qw(next_event tell); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($next_event, $tell) = @args{@required_args}; + + my $line; + my $pos_in_log = $tell->(); + LINE: + while ( defined($line = $next_event->()) ) { + PTDEBUG && _d($line); + chomp($line); + my @properties = ( + 'pos_in_log', $pos_in_log, + 'cmd', 'Query', + 'bytes', length($line), + 'Query_time', 0, + 'arg', $line, + ); + + $pos_in_log = $tell->(); + + PTDEBUG && _d('Properties of event:', Dumper(\@properties)); + my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } + + return $event; + } + + $args{oktorun}->(0) if $args{oktorun}; + return; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +1; +} +# ########################################################################### +# End RawLogParser package +# ########################################################################### + # ########################################################################### # ProtocolParser package # This package is a copy without comments from the original. The original @@ -13227,7 +13307,7 @@ sub main { my $review_dsn = $o->get('review'); my @groupby = @{$o->get('group-by')}; my @orderby; - if ( (grep { $_ eq 'genlog' || $_ eq 'GeneralLogParser' } @{$o->get('type')}) + if ( (grep { $_ =~ m/genlog|GeneralLogParser|rawlog|RawLogParser/ } @{$o->get('type')}) && !$o->got('order-by') ) { @orderby = 'Query_time:cnt'; } @@ -13668,6 +13748,7 @@ sub main { 'MemcachedEvent'], http => ['TcpdumpParser','HTTPProtocolParser'], pglog => ['PgLogParser'], + rawlog => ['RawLogParser'], ); my $type = $o->get('type'); $type = $alias_for{$type->[0]} if $alias_for{$type->[0]}; diff --git a/lib/RawLogParser.pm b/lib/RawLogParser.pm new file mode 100644 index 00000000..1bd6426a --- /dev/null +++ b/lib/RawLogParser.pm @@ -0,0 +1,95 @@ +# This program is copyright 2012 Percona Inc. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# RawLogParser package +# ########################################################################### +{ +# Package: RawLogParser +# RawLogParser parses logs with nothing but one query per line. +package RawLogParser; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +use Data::Dumper; +$Data::Dumper::Indent = 1; +$Data::Dumper::Sortkeys = 1; +$Data::Dumper::Quotekeys = 0; + +sub new { + my ( $class ) = @_; + my $self = { + }; + return bless $self, $class; +} + +sub parse_event { + my ( $self, %args ) = @_; + my @required_args = qw(next_event tell); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($next_event, $tell) = @args{@required_args}; + + my $line; + my $pos_in_log = $tell->(); + LINE: + while ( defined($line = $next_event->()) ) { + PTDEBUG && _d($line); + chomp($line); + my @properties = ( + 'pos_in_log', $pos_in_log, + 'cmd', 'Query', + 'bytes', length($line), + 'Query_time', 0, + 'arg', $line, + ); + + $pos_in_log = $tell->(); + + # Don't dump $event; want to see full dump of all properties, + # and after it's been cast into a hash, duplicated keys will + # be gone. + PTDEBUG && _d('Properties of event:', Dumper(\@properties)); + my $event = { @properties }; + if ( $args{stats} ) { + $args{stats}->{events_read}++; + $args{stats}->{events_parsed}++; + } + + return $event; + } + + $args{oktorun}->(0) if $args{oktorun}; + return; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +1; +} +# ########################################################################### +# End RawLogParser package +# ########################################################################### diff --git a/t/lib/RawLogParser.t b/t/lib/RawLogParser.t new file mode 100644 index 00000000..e533041c --- /dev/null +++ b/t/lib/RawLogParser.t @@ -0,0 +1,52 @@ +#!/usr/bin/perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More tests => 3; + +use RawLogParser; +use PerconaTest; + +my $p = new RawLogParser(); + +my $oktorun = 1; +my $sample = "t/lib/samples/rawlogs/"; + +test_log_parser( + parser => $p, + file => $sample.'rawlog001.txt', + oktorun => sub { $oktorun = $_[0]; }, + result => [ + { pos_in_log => 0, + args => 'SELECT c FROM t WHERE id=1', + bytes => 26, + cmd => 'Query', + Query_time => 0, + }, + { pos_in_log => 27, + args => '/* Hello, world! */ SELECT * FROM t2 LIMIT 1', + bytes => 44, + cmd => 'Query', + Query_time => 0, + } + ] +); + +is( + $oktorun, + 0, + 'Sets oktorun' +); +$oktorun = 1; + +# ############################################################################# +# Done. +# ############################################################################# +exit; diff --git a/t/lib/samples/rawlogs/rawlog001.txt b/t/lib/samples/rawlogs/rawlog001.txt new file mode 100644 index 00000000..0c89b2f5 --- /dev/null +++ b/t/lib/samples/rawlogs/rawlog001.txt @@ -0,0 +1,2 @@ +SELECT c FROM t WHERE id=1 +/* Hello, world! */ SELECT * FROM t2 LIMIT 1 diff --git a/t/pt-query-digest/rawlog_analyses.t b/t/pt-query-digest/rawlog_analyses.t new file mode 100644 index 00000000..7bc30acb --- /dev/null +++ b/t/pt-query-digest/rawlog_analyses.t @@ -0,0 +1,48 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More tests => 2; + +use PerconaTest; + +# See 101_slowlog_analyses.t or http://code.google.com/p/maatkit/wiki/Testing +shift @INC; # our unshift (above) +shift @INC; # PerconaTest's unshift + +require "$trunk/bin/pt-query-digest"; + +# ############################################################################# +# Issue 172: Make mk-query-digest able to read raweral logs +# ############################################################################# + +my @args = ('--report-format', 'header,query_report,profile', '--type', 'rawlog'); +my $sample = "$trunk/t/lib/samples/rawlogs/"; + +# --help exists so don't run mqd as a module else --help's exit will +# exit this test script. +like( + `$trunk/bin/pt-query-digest --type rawlog rawlog001.txt --help`, + qr/--order-by\s+Query_time:cnt/, + '--order-by defaults to Query_time:cnt for --type rawlog', +); + +ok( + no_diff( + sub { pt_query_digest::main(@args, $sample.'rawlog001.txt') }, + "t/pt-query-digest/samples/rawlog001.txt" + ), + 'Analysis for rawlog001', +); + +# ############################################################################# +# Done. +# ############################################################################# +exit; diff --git a/t/pt-query-digest/samples/rawlog001.txt b/t/pt-query-digest/samples/rawlog001.txt new file mode 100644 index 00000000..3f298ac0 --- /dev/null +++ b/t/pt-query-digest/samples/rawlog001.txt @@ -0,0 +1,60 @@ + +# Overall: 2 total, 2 unique, 0 QPS, 0x concurrency ______________________ +# Attribute total min max avg 95% stddev median +# ============ ======= ======= ======= ======= ======= ======= ======= +# Exec time 0 0 0 0 0 0 0 +# Query size 70 26 44 35 44 12.73 35 + +# Query 1: 0 QPS, 0x concurrency, ID 0xCB5621E548E5497F at byte 0 ________ +# This item is included in the report because it matches --limit. +# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00 +# Query_time sparkline: | | +# Attribute pct total min max avg 95% stddev median +# ============ === ======= ======= ======= ======= ======= ======= ======= +# Count 50 1 +# Exec time 0 0 0 0 0 0 0 0 +# Query size 37 26 26 26 26 26 0 26 +# Query_time distribution +# 1us +# 10us +# 100us +# 1ms +# 10ms +# 100ms +# 1s +# 10s+ +# Tables +# SHOW TABLE STATUS LIKE 't'\G +# SHOW CREATE TABLE `t`\G +# EXPLAIN /*!50100 PARTITIONS*/ +SELECT c FROM t WHERE id=1\G + +# Query 2: 0 QPS, 0x concurrency, ID 0x774B2B0B59EBAC2C at byte 27 _______ +# This item is included in the report because it matches --limit. +# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00 +# Query_time sparkline: | | +# Attribute pct total min max avg 95% stddev median +# ============ === ======= ======= ======= ======= ======= ======= ======= +# Count 50 1 +# Exec time 0 0 0 0 0 0 0 0 +# Query size 62 44 44 44 44 44 0 44 +# Query_time distribution +# 1us +# 10us +# 100us +# 1ms +# 10ms +# 100ms +# 1s +# 10s+ +# Tables +# SHOW TABLE STATUS LIKE 't2'\G +# SHOW CREATE TABLE `t2`\G +# EXPLAIN /*!50100 PARTITIONS*/ +/* Hello, world! */ SELECT * FROM t2 LIMIT 1\G + +# Profile +# Rank Query ID Response time Calls R/Call Apdx V/M Item +# ==== ================== ============= ===== ====== ==== ===== ========= +# 1 0xCB5621E548E5497F 0.0000 0.0% 1 0.0000 1.00 0.00 SELECT t +# 2 0x774B2B0B59EBAC2C 0.0000 0.0% 1 0.0000 1.00 0.00 SELECT t?