From 5530e7ab17cebe70737cc228033a0398d53c4285 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Sat, 9 Feb 2013 14:20:38 -0700 Subject: [PATCH] Stripping down pt-upgrade; work in progress. --- bin/pt-upgrade | 250 ++-------------------------------- lib/QueryIterator.pm | 139 +++++++++++++++++++ lib/UpgradeReportFormatter.pm | 223 ------------------------------ lib/UpgradeResults.pm | 41 ++++++ 4 files changed, 190 insertions(+), 463 deletions(-) create mode 100644 lib/QueryIterator.pm delete mode 100644 lib/UpgradeReportFormatter.pm create mode 100644 lib/UpgradeResults.pm diff --git a/bin/pt-upgrade b/bin/pt-upgrade index 715bd686..df972ecd 100755 --- a/bin/pt-upgrade +++ b/bin/pt-upgrade @@ -3797,244 +3797,6 @@ sub main { $host->{name} = $name || 'unknown host'; } - # ######################################################################## - # Make some common modules. - # ######################################################################## - my $q = new Quoter(); - my $qp = new QueryParser(); - my $qr = new QueryRewriter(); - my $rr = new Retry(); - my $tp = new TableParser(Quoter => $q); - my $chunker = new TableChunker(Quoter => $q, TableParser => $tp ); - my $nibbler = new TableNibbler(Quoter => $q, TableParser => $tp ); - my $checksum = new TableChecksum(Quoter => $q); - my $syncer = new TableSyncer( - MasterSlave => 1, # I don't think we need this. - Quoter => $q, - TableChecksum => $checksum, - Retry => $rr, - ); - my %common_modules = ( - DSNParser => $dp, - OptionParser => $o, - QueryParser => $qp, - QueryRewriter => $qr, - TableParser => $tp, - Quoter => $q, - TableChunker => $chunker, - TableNibbler => $nibbler, - TableChecksum => $checksum, - TableSyncer => $syncer, - ); - - # ######################################################################## - # Make compare modules in order. - # ######################################################################## - my $compare = $o->get('compare'); - my @compare_modules; - if ( $compare->{results} ) { - my $method = lc $o->get('compare-results-method'); - PTDEBUG && _d('Compare results method:', $method); - - my $plugins = []; - if ( $method eq 'rows' ) { - push @$plugins, new TableSyncChunk(%common_modules); - push @$plugins, new TableSyncNibble(%common_modules); - push @$plugins, new TableSyncGroupBy(%common_modules); - } - - push @compare_modules, new CompareResults( - method => $method, - plugins => $plugins, - get_id => sub { return make_checksum(@_); }, - 'base-dir' => $o->get('base-dir'), - %common_modules, - ); - } - - if ( $compare->{query_times} ) { - push @compare_modules, new CompareQueryTimes( - get_id => sub { return make_checksum(@_); }, - %common_modules, - ); - } - - if ( $compare->{warnings} ) { - push @compare_modules, new CompareWarnings( - 'clear-warnings' => $o->get('clear-warnings'), - 'clear-warnings-table' => $o->get('clear-warnings-table'), - get_id => sub { return make_checksum(@_); }, - %common_modules, - ); - } - - # ######################################################################## - # Make an EventAggregator for each host and a "meta-aggregator" for all. - # ######################################################################## - my $groupby = 'fingerprint'; - map { - my $ea = new EventAggregator( - groupby => 'fingerprint', - worst => 'Query_time', - ); - $_->{ea} = $ea; - } @$hosts; - - my $meta_ea = new EventAggregator( - groupby => 'fingerprint', - worst => 'differences' - ); - - # ######################################################################## - # Create the event pipeline. - # ######################################################################## - my @callbacks; - my $stats = {}; - my $errors = {}; - my $current_db; - my $tmp_db = $o->get('temp-database'); - my $tmp_tbl = $o->get('temp-table'); - - if ( my $query = $o->get('query') ) { - push @callbacks, sub { - my ( $event, %args ) = @_; - PTDEBUG && _d('callback: query:', $query); - $args{oktorun}->(0) if $args{oktorun}; - return { - cmd => 'Query', - arg => $query, - pos_in_log => 0, # for compatibility - }; - }; - } - else { - my $parser = new SlowLogParser(); - push @callbacks, sub { - my ( $event, %args ) = @_; - return $parser->parse_event(%args); - }; - } - - push @callbacks, sub { - my ( $event ) = @_; - PTDEBUG && _d('callback: check cmd and arg'); - $stats->{events}++; - if ( ($event->{cmd} || '') ne 'Query' ) { - PTDEBUG && _d('Skipping non-Query cmd'); - $stats->{not_query}++; - return; - } - if ( !$event->{arg} ) { - PTDEBUG && _d('Skipping empty arg'); - $stats->{empty_query}++; - return; - } - return $event; - }; - - # User-defined filter. - if ( $o->get('filter') ) { - my $filter = $o->get('filter'); - if ( -f $filter && -r $filter ) { - PTDEBUG && _d('Reading file', $filter, 'for --filter code'); - open my $fh, "<", $filter or die "Cannot open $filter: $OS_ERROR"; - $filter = do { local $/ = undef; <$fh> }; - close $fh; - } - else { - $filter = "( $filter )"; # issue 565 - } - my $code = "sub { PTDEBUG && _d('callback: filter'); my(\$event) = shift; $filter && return \$event; };"; - PTDEBUG && _d('--filter code:', $code); - my $sub = eval $code - or die "Error compiling --filter code: $code\n$EVAL_ERROR"; - push @callbacks, $sub; - } - - if ( $o->get('convert-to-select') ) { - push @callbacks, sub { - my ( $event ) = @_; - PTDEBUG && _d('callback: convert to select'); - return $event if $event->{arg} =~ m/(?:^SELECT|(?:\*\/\s*SELECT))/i; - my $new_arg = $qr->convert_to_select($event->{arg}); - if ( $new_arg =~ m/^SELECT/i ) { - $stats->{convert_to_select_ok}++; - $event->{original_arg} = $event->{arg}; - $event->{arg} = $new_arg; - return $event; - } - PTDEBUG && _d('Convert to SELECT failed:', $event->{arg}); - $stats->{convert_to_select_failed}++; - return; - }; - } - else { - push @callbacks, sub { - my ( $event ) = @_; - PTDEBUG && _d('callback: filter non-select'); - if ( $event->{arg} !~ m/(?:^SELECT|(?:\*\/\s*SELECT))/i ) { - PTDEBUG && _d('Skipping non-SELECT query'); - $stats->{not_select}++; - return; - } - return $event; - }; - } - - # Do this callback before the fingerprint and sampleno attribs are added. - my %allowed_attribs = qw(arg 1 db 1 sampleno 1 pos_in_log 1 original_arg 1); - push @callbacks, sub { - my ( $event ) = @_; - PTDEBUG && _d('callback: remove not-allowed attributes'); - # Events will have a lot of attribs from the log that we want - # to remove because 1) we don't need them and 2) we want to avoid - # attrib conflicts (e.g. there's probably a Query_time from the - # log but the compare modules add a Query_time attrib, too). - map { delete $event->{$_} unless $allowed_attribs{$_} } keys %$event; - return $event; - }; - - push @callbacks, sub { - my ( $event ) = @_; - PTDEBUG && _d('callback: fingerprint'); - $event->{fingerprint} = $qr->fingerprint($event->{arg}); - return $event; - }; - - my %samples; - my %samplenos; - push @callbacks, sub { - my ( $event ) = @_; - PTDEBUG && _d('callback: sampleno'); - $event->{sampleno} = ++$samplenos{$event->{$groupby}}; - PTDEBUG && _d('Event sampleno', $event->{sampleno}); - return $event; - }; - - # Keep the default database update-to-date. This helps when queries - # don't use db-qualified tables. - push @callbacks, sub { - my ( $event ) = @_; - PTDEBUG && _d('callback: current db'); - my $db = $event->{db} || $event->{Schema} || $hosts->[0]->{dsn}->{D}; - if ( $db && (!$current_db || $db ne $current_db) ) { - my $sql = "USE `$db`"; - PTDEBUG && _d($sql); - eval { - map { $_->{dbh}->do($sql); } @$hosts; - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d('Error:', $EVAL_ERROR); - $EVAL_ERROR =~ m/Unknown database/ ? $stats->{unknown_database}++ - : $stats->{use_database_error}++; - return; - } - $event->{db} = $current_db = $db; - } - $stats->{no_database}++ unless $current_db; - return $event; - }; - # ######################################################################## # Do the version-check # ######################################################################## @@ -4111,6 +3873,14 @@ sub compare_host_to_host { } sub save_host_results { + while ( + # oktorun + # run-time + my $query = $query_iter->next() ) { + # Exec query on host + # Save results to disk + # Report progress + } } sub compare_results_to_host { @@ -4282,7 +4052,7 @@ pt-upgrade - Verify that queries produce identical results on different servers. =head1 SYNOPSIS -Usage: pt-upgrade [OPTIONS] LOG DSN DSN +Usage: pt-upgrade [OPTIONS] LOG|RESULTS DSN [DSN] pt-upgrade executes the queries in C on each C, compares the results, and reports any significant differences. C can be @@ -4296,7 +4066,7 @@ Save results for host1, then compare host2 to them: pt-upgrade slow.log h=host1 --save-results host1_results/ - pt-upgrade slow.log host1_results1/ h=host2 + pt-upgrade host1_results1/ h=host2 =head1 RISKS diff --git a/lib/QueryIterator.pm b/lib/QueryIterator.pm new file mode 100644 index 00000000..4ccf85b0 --- /dev/null +++ b/lib/QueryIterator.pm @@ -0,0 +1,139 @@ +# This program is copyright 2013 Percona Ireland Ltd. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# QueryIterator package +# ########################################################################### +{ +package QueryIterator; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +use Mo; + +has 'parser' => ( + is => 'ro', + isa => 'Object', + required => 1, +); + +has 'oktorun' => ( + is => 'ro', + isa => 'CodeRef', + required => 1, +); + +has 'database' => ( + is => 'rw', + isa => 'Maybe[Str]', + required => 0, +); + +has 'filter' => ( + is => 'ro', + isa => 'CodeRef', + required => 0, +); + +has 'read_only' => ( + is => 'ro', + isa => 'Bool', + required => 0, + default => 0, +); + +sub BUILDARGS { + + my $filter_code; + if ( my $filter = $args{filter} ) { + if ( -f $filter && -r $filter ) { + PTDEBUG && _d('Reading file', $filter, 'for --filter code'); + open my $fh, "<", $filter or die "Cannot open $filter: $OS_ERROR"; + $filter = do { local $/ = undef; <$fh> }; + close $fh; + } + else { + $filter = "( $filter )"; # issue 565 + } + my $code = "sub { PTDEBUG && _d('callback: filter'); my(\$event) = shift; $filter && return \$event; };"; + PTDEBUG && _d('--filter code:', $code); + $filter_code = eval $code + or die "Error compiling --filter code: $code\n$EVAL_ERROR"; + } + else { + $filter_code = sub { return 1 }; + } + +} + +sub next { + my ($self) = @_; + + EVENT: + while ( + $self->oktorun() + && (my $event = $parser->parse_event(%args)) + ) { + + $self->stats->{events}++; + + if ( ($event->{cmd} || '') ne 'Query' ) { + PTDEBUG && _d('Skipping non-Query cmd'); + $stats->{not_query}++; + next EVENT; + } + + if ( !$event->{arg} ) { + PTDEBUG && _d('Skipping empty arg'); + $stats->{empty_query}++; + next EVENT; + } + + next EVENT unless $self->filter->(); + + if ( $self->read_only ) { + if ( $event->{arg} !~ m/(?:^SELECT|(?:\*\/\s*SELECT))/i ) { + PTDEBUG && _d('Skipping non-SELECT query'); + $stats->{not_select}++; + next EVENT; + } + } + + $event->{fingerprint} = $qr->fingerprint($event->{arg}); + + my $db = $event->{db} || $event->{Schema} || $hosts->[0]->{dsn}->{D}; + if ( $db && (!$current_db || $db ne $current_db) ) { + $self->database($db); + } + else { + $self->database(undef); + } + + return $event; + } # EVENT + + return; +} + +no Mo; +1; +} +# ########################################################################### +# End QueryIterator package +# ########################################################################### diff --git a/lib/UpgradeReportFormatter.pm b/lib/UpgradeReportFormatter.pm deleted file mode 100644 index 46dd80e7..00000000 --- a/lib/UpgradeReportFormatter.pm +++ /dev/null @@ -1,223 +0,0 @@ -# This program is copyright 2009-2012 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. -# ########################################################################### -# UpgradeReportFormatter package -# ########################################################################### -{ -# Package: UpgradeReportFormatter -# UpgradeReportFormatter formats the output of pt-upgrade. -package UpgradeReportFormatter; - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); - -use constant PTDEBUG => $ENV{PTDEBUG}; -use constant LINE_LENGTH => 74; -use constant MAX_STRING_LENGTH => 10; - -Transformers->import(qw(make_checksum percentage_of shorten micro_t)); - -# Special formatting functions -my %formatting_function = ( - ts => sub { - my ( $stats ) = @_; - my $min = parse_timestamp($stats->{min} || ''); - my $max = parse_timestamp($stats->{max} || ''); - return $min && $max ? "$min to $max" : ''; - }, -); - -my $bool_format = '# %3s%% %-6s %s'; - -sub new { - my ( $class, %args ) = @_; - return bless { }, $class; -} - -sub event_report { - my ( $self, %args ) = @_; - my @required_args = qw(where rank worst meta_ea hosts); - foreach my $arg ( @required_args ) { - die "I need a $arg argument" unless $args{$arg}; - } - my ($where, $rank, $worst, $meta_ea, $hosts) = @args{@required_args}; - my $meta_stats = $meta_ea->results; - my @result; - - - # First line - my $line = sprintf( - '# Query %d: ID 0x%s at byte %d ', - $rank || 0, - make_checksum($where) || '0x0', - 0, # $sample->{pos_in_log} || 0 - ); - $line .= ('_' x (LINE_LENGTH - length($line))); - push @result, $line; - - # Second line: full host names - # https://bugs.launchpad.net/percona-toolkit/+bug/980318 - my $hostno = 0; - foreach my $host ( @$hosts ) { - $hostno++; - push @result, "# host$hostno: " . ($host->{name} || '?') - } - - # Differences report. This relies on a sampleno attrib in each class - # since all other attributes (except maybe Query_time) are optional. - my $class = $meta_stats->{classes}->{$where}; - push @result, - '# Found ' . ($class->{differences}->{sum} || 0) - . ' differences in ' . $class->{sampleno}->{cnt} . " samples:\n"; - - my $fmt = "# %-17s %d\n"; - my @diffs = grep { $_ =~ m/^different_/ } keys %$class; - foreach my $diff ( sort @diffs ) { - push @result, - sprintf $fmt, ' ' . (make_label($diff) || ''), ($class->{$diff}->{sum} || 0); - } - - # Side-by-side hosts report. - my $report = new ReportFormatter( - underline_header => 0, - strip_whitespace => 0, - ); - $hostno = 0; - $report->set_columns( - { name => '' }, - map { $hostno++; { name => "host$hostno", right_justify => 1 } } @$hosts, - ); - # Bool values. - foreach my $thing ( qw(Errors Warnings) ) { - my @vals = $thing; - foreach my $host ( @$hosts ) { - my $ea = $host->{ea}; - my $stats = $ea->results->{classes}->{$where}; - if ( $stats && $stats->{$thing} ) { - push @vals, shorten($stats->{$thing}->{sum}, d=>1_000, p=>0) - } - else { - push @vals, 0; - } - } - $report->add_line(@vals); - } - # Fully aggregated numeric values. - foreach my $thing ( qw(Query_time row_count) ) { - my @vals; - - foreach my $host ( @$hosts ) { - my $ea = $host->{ea}; - my $stats = $ea->results->{classes}->{$where}; - if ( $stats && $stats->{$thing} ) { - my $vals = $stats->{$thing}; - my $func = $thing =~ m/time$/ ? \µ_t : \&shorten; - my $metrics = $host->{ea}->metrics(attrib=>$thing, where=>$where); - my @n = ( - @{$vals}{qw(sum min max)}, - ($vals->{sum} || 0) / ($vals->{cnt} || 1), - @{$metrics}{qw(pct_95 stddev median)}, - ); - @n = map { defined $_ ? $func->($_) : '' } @n; - push @vals, \@n; - } - else { - push @vals, undef; - } - } - - if ( scalar @vals && grep { defined } @vals ) { - $report->add_line($thing, map { '' } @$hosts); - my @metrics = qw(sum min max avg pct_95 stddev median); - for my $i ( 0..$#metrics ) { - my @n = ' ' . $metrics[$i]; - push @n, map { $_ && defined $_->[$i] ? $_->[$i] : '' } @vals; - $report->add_line(@n); - } - } - } - - push @result, $report->get_report(); - - return join("\n", map { s/\s+$//; $_ } @result) . "\n"; -} - -# Convert attribute names into labels -sub make_label { - my ( $val ) = @_; - - $val =~ s/^different_//; - $val =~ s/_/ /g; - - return $val; -} - -# Does pretty-printing for lists of strings like users, hosts, db. -sub format_string_list { - my ( $stats ) = @_; - if ( exists $stats->{unq} ) { - # Only class stats have unq. - my $cnt_for = $stats->{unq}; - if ( 1 == keys %$cnt_for ) { - my ($str) = keys %$cnt_for; - # - 30 for label, spacing etc. - $str = substr($str, 0, LINE_LENGTH - 30) . '...' - if length $str > LINE_LENGTH - 30; - return (1, $str); - } - my $line = ''; - my @top = sort { $cnt_for->{$b} <=> $cnt_for->{$a} || $a cmp $b } - keys %$cnt_for; - my $i = 0; - foreach my $str ( @top ) { - my $print_str; - if ( length $str > MAX_STRING_LENGTH ) { - $print_str = substr($str, 0, MAX_STRING_LENGTH) . '...'; - } - else { - $print_str = $str; - } - last if (length $line) + (length $print_str) > LINE_LENGTH - 27; - $line .= "$print_str ($cnt_for->{$str}), "; - $i++; - } - $line =~ s/, $//; - if ( $i < @top ) { - $line .= "... " . (@top - $i) . " more"; - } - return (scalar keys %$cnt_for, $line); - } - else { - # Global stats don't have unq. - return ($stats->{cnt}); - } -} - -sub _d { - my ($package, undef, $line) = caller 0; - @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } - map { defined $_ ? $_ : 'undef' } - @_; - print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; -} - -1; -} -# ########################################################################### -# End UpgradeReportFormatter package -# ########################################################################### diff --git a/lib/UpgradeResults.pm b/lib/UpgradeResults.pm new file mode 100644 index 00000000..d0f34fb3 --- /dev/null +++ b/lib/UpgradeResults.pm @@ -0,0 +1,41 @@ +# This program is copyright 2013 Percona Ireland Ltd. +# Feedback and improvements are welcome. +# +# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +# systems, you can issue `man perlgpl' or `man perlartistic' to read these +# licenses. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# ########################################################################### +# UpgradeResults package +# ########################################################################### +{ +package UpgradeResults; + +use Mo; +use Scalar::Util qw(blessed); +use English qw(-no_match_vars); +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +has 'query_classes' => ( + is => 'rw', + isa => 'HashRef', + required => 0, + default => sub { return {} }, +); + + +no Mo; +1; +} +# ########################################################################### +# End UpgradeResults package +# ###########################################################################