Stripping down pt-upgrade; work in progress.

This commit is contained in:
Daniel Nichter
2013-02-09 14:20:38 -07:00
parent 0c229fce51
commit 5530e7ab17
4 changed files with 190 additions and 463 deletions

View File

@@ -3797,244 +3797,6 @@ sub main {
$host->{name} = $name || 'unknown host';
}
# ########################################################################
# Make some common modules.
# ########################################################################
my $q = new Quoter();
my $qp = new QueryParser();
my $qr = new QueryRewriter();
my $rr = new Retry();
my $tp = new TableParser(Quoter => $q);
my $chunker = new TableChunker(Quoter => $q, TableParser => $tp );
my $nibbler = new TableNibbler(Quoter => $q, TableParser => $tp );
my $checksum = new TableChecksum(Quoter => $q);
my $syncer = new TableSyncer(
MasterSlave => 1, # I don't think we need this.
Quoter => $q,
TableChecksum => $checksum,
Retry => $rr,
);
my %common_modules = (
DSNParser => $dp,
OptionParser => $o,
QueryParser => $qp,
QueryRewriter => $qr,
TableParser => $tp,
Quoter => $q,
TableChunker => $chunker,
TableNibbler => $nibbler,
TableChecksum => $checksum,
TableSyncer => $syncer,
);
# ########################################################################
# Make compare modules in order.
# ########################################################################
my $compare = $o->get('compare');
my @compare_modules;
if ( $compare->{results} ) {
my $method = lc $o->get('compare-results-method');
PTDEBUG && _d('Compare results method:', $method);
my $plugins = [];
if ( $method eq 'rows' ) {
push @$plugins, new TableSyncChunk(%common_modules);
push @$plugins, new TableSyncNibble(%common_modules);
push @$plugins, new TableSyncGroupBy(%common_modules);
}
push @compare_modules, new CompareResults(
method => $method,
plugins => $plugins,
get_id => sub { return make_checksum(@_); },
'base-dir' => $o->get('base-dir'),
%common_modules,
);
}
if ( $compare->{query_times} ) {
push @compare_modules, new CompareQueryTimes(
get_id => sub { return make_checksum(@_); },
%common_modules,
);
}
if ( $compare->{warnings} ) {
push @compare_modules, new CompareWarnings(
'clear-warnings' => $o->get('clear-warnings'),
'clear-warnings-table' => $o->get('clear-warnings-table'),
get_id => sub { return make_checksum(@_); },
%common_modules,
);
}
# ########################################################################
# Make an EventAggregator for each host and a "meta-aggregator" for all.
# ########################################################################
my $groupby = 'fingerprint';
map {
my $ea = new EventAggregator(
groupby => 'fingerprint',
worst => 'Query_time',
);
$_->{ea} = $ea;
} @$hosts;
my $meta_ea = new EventAggregator(
groupby => 'fingerprint',
worst => 'differences'
);
# ########################################################################
# Create the event pipeline.
# ########################################################################
my @callbacks;
my $stats = {};
my $errors = {};
my $current_db;
my $tmp_db = $o->get('temp-database');
my $tmp_tbl = $o->get('temp-table');
if ( my $query = $o->get('query') ) {
push @callbacks, sub {
my ( $event, %args ) = @_;
PTDEBUG && _d('callback: query:', $query);
$args{oktorun}->(0) if $args{oktorun};
return {
cmd => 'Query',
arg => $query,
pos_in_log => 0, # for compatibility
};
};
}
else {
my $parser = new SlowLogParser();
push @callbacks, sub {
my ( $event, %args ) = @_;
return $parser->parse_event(%args);
};
}
push @callbacks, sub {
my ( $event ) = @_;
PTDEBUG && _d('callback: check cmd and arg');
$stats->{events}++;
if ( ($event->{cmd} || '') ne 'Query' ) {
PTDEBUG && _d('Skipping non-Query cmd');
$stats->{not_query}++;
return;
}
if ( !$event->{arg} ) {
PTDEBUG && _d('Skipping empty arg');
$stats->{empty_query}++;
return;
}
return $event;
};
# User-defined filter.
if ( $o->get('filter') ) {
my $filter = $o->get('filter');
if ( -f $filter && -r $filter ) {
PTDEBUG && _d('Reading file', $filter, 'for --filter code');
open my $fh, "<", $filter or die "Cannot open $filter: $OS_ERROR";
$filter = do { local $/ = undef; <$fh> };
close $fh;
}
else {
$filter = "( $filter )"; # issue 565
}
my $code = "sub { PTDEBUG && _d('callback: filter'); my(\$event) = shift; $filter && return \$event; };";
PTDEBUG && _d('--filter code:', $code);
my $sub = eval $code
or die "Error compiling --filter code: $code\n$EVAL_ERROR";
push @callbacks, $sub;
}
if ( $o->get('convert-to-select') ) {
push @callbacks, sub {
my ( $event ) = @_;
PTDEBUG && _d('callback: convert to select');
return $event if $event->{arg} =~ m/(?:^SELECT|(?:\*\/\s*SELECT))/i;
my $new_arg = $qr->convert_to_select($event->{arg});
if ( $new_arg =~ m/^SELECT/i ) {
$stats->{convert_to_select_ok}++;
$event->{original_arg} = $event->{arg};
$event->{arg} = $new_arg;
return $event;
}
PTDEBUG && _d('Convert to SELECT failed:', $event->{arg});
$stats->{convert_to_select_failed}++;
return;
};
}
else {
push @callbacks, sub {
my ( $event ) = @_;
PTDEBUG && _d('callback: filter non-select');
if ( $event->{arg} !~ m/(?:^SELECT|(?:\*\/\s*SELECT))/i ) {
PTDEBUG && _d('Skipping non-SELECT query');
$stats->{not_select}++;
return;
}
return $event;
};
}
# Do this callback before the fingerprint and sampleno attribs are added.
my %allowed_attribs = qw(arg 1 db 1 sampleno 1 pos_in_log 1 original_arg 1);
push @callbacks, sub {
my ( $event ) = @_;
PTDEBUG && _d('callback: remove not-allowed attributes');
# Events will have a lot of attribs from the log that we want
# to remove because 1) we don't need them and 2) we want to avoid
# attrib conflicts (e.g. there's probably a Query_time from the
# log but the compare modules add a Query_time attrib, too).
map { delete $event->{$_} unless $allowed_attribs{$_} } keys %$event;
return $event;
};
push @callbacks, sub {
my ( $event ) = @_;
PTDEBUG && _d('callback: fingerprint');
$event->{fingerprint} = $qr->fingerprint($event->{arg});
return $event;
};
my %samples;
my %samplenos;
push @callbacks, sub {
my ( $event ) = @_;
PTDEBUG && _d('callback: sampleno');
$event->{sampleno} = ++$samplenos{$event->{$groupby}};
PTDEBUG && _d('Event sampleno', $event->{sampleno});
return $event;
};
# Keep the default database update-to-date. This helps when queries
# don't use db-qualified tables.
push @callbacks, sub {
my ( $event ) = @_;
PTDEBUG && _d('callback: current db');
my $db = $event->{db} || $event->{Schema} || $hosts->[0]->{dsn}->{D};
if ( $db && (!$current_db || $db ne $current_db) ) {
my $sql = "USE `$db`";
PTDEBUG && _d($sql);
eval {
map { $_->{dbh}->do($sql); } @$hosts;
};
if ( $EVAL_ERROR ) {
PTDEBUG && _d('Error:', $EVAL_ERROR);
$EVAL_ERROR =~ m/Unknown database/ ? $stats->{unknown_database}++
: $stats->{use_database_error}++;
return;
}
$event->{db} = $current_db = $db;
}
$stats->{no_database}++ unless $current_db;
return $event;
};
# ########################################################################
# Do the version-check
# ########################################################################
@@ -4111,6 +3873,14 @@ sub compare_host_to_host {
}
sub save_host_results {
while (
# oktorun
# run-time
my $query = $query_iter->next() ) {
# Exec query on host
# Save results to disk
# Report progress
}
}
sub compare_results_to_host {
@@ -4282,7 +4052,7 @@ pt-upgrade - Verify that queries produce identical results on different servers.
=head1 SYNOPSIS
Usage: pt-upgrade [OPTIONS] LOG DSN DSN
Usage: pt-upgrade [OPTIONS] LOG|RESULTS DSN [DSN]
pt-upgrade executes the queries in C<LOG> on each C<DSN>, compares
the results, and reports any significant differences. C<LOG> can be
@@ -4296,7 +4066,7 @@ Save results for host1, then compare host2 to them:
pt-upgrade slow.log h=host1 --save-results host1_results/
pt-upgrade slow.log host1_results1/ h=host2
pt-upgrade host1_results1/ h=host2
=head1 RISKS

139
lib/QueryIterator.pm Normal file
View File

@@ -0,0 +1,139 @@
# This program is copyright 2013 Percona Ireland Ltd.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# QueryIterator package
# ###########################################################################
{
package QueryIterator;
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use Mo;
has 'parser' => (
is => 'ro',
isa => 'Object',
required => 1,
);
has 'oktorun' => (
is => 'ro',
isa => 'CodeRef',
required => 1,
);
has 'database' => (
is => 'rw',
isa => 'Maybe[Str]',
required => 0,
);
has 'filter' => (
is => 'ro',
isa => 'CodeRef',
required => 0,
);
has 'read_only' => (
is => 'ro',
isa => 'Bool',
required => 0,
default => 0,
);
sub BUILDARGS {
my $filter_code;
if ( my $filter = $args{filter} ) {
if ( -f $filter && -r $filter ) {
PTDEBUG && _d('Reading file', $filter, 'for --filter code');
open my $fh, "<", $filter or die "Cannot open $filter: $OS_ERROR";
$filter = do { local $/ = undef; <$fh> };
close $fh;
}
else {
$filter = "( $filter )"; # issue 565
}
my $code = "sub { PTDEBUG && _d('callback: filter'); my(\$event) = shift; $filter && return \$event; };";
PTDEBUG && _d('--filter code:', $code);
$filter_code = eval $code
or die "Error compiling --filter code: $code\n$EVAL_ERROR";
}
else {
$filter_code = sub { return 1 };
}
}
sub next {
my ($self) = @_;
EVENT:
while (
$self->oktorun()
&& (my $event = $parser->parse_event(%args))
) {
$self->stats->{events}++;
if ( ($event->{cmd} || '') ne 'Query' ) {
PTDEBUG && _d('Skipping non-Query cmd');
$stats->{not_query}++;
next EVENT;
}
if ( !$event->{arg} ) {
PTDEBUG && _d('Skipping empty arg');
$stats->{empty_query}++;
next EVENT;
}
next EVENT unless $self->filter->();
if ( $self->read_only ) {
if ( $event->{arg} !~ m/(?:^SELECT|(?:\*\/\s*SELECT))/i ) {
PTDEBUG && _d('Skipping non-SELECT query');
$stats->{not_select}++;
next EVENT;
}
}
$event->{fingerprint} = $qr->fingerprint($event->{arg});
my $db = $event->{db} || $event->{Schema} || $hosts->[0]->{dsn}->{D};
if ( $db && (!$current_db || $db ne $current_db) ) {
$self->database($db);
}
else {
$self->database(undef);
}
return $event;
} # EVENT
return;
}
no Mo;
1;
}
# ###########################################################################
# End QueryIterator package
# ###########################################################################

View File

@@ -1,223 +0,0 @@
# This program is copyright 2009-2012 Percona Inc.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# UpgradeReportFormatter package
# ###########################################################################
{
# Package: UpgradeReportFormatter
# UpgradeReportFormatter formats the output of pt-upgrade.
package UpgradeReportFormatter;
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG};
use constant LINE_LENGTH => 74;
use constant MAX_STRING_LENGTH => 10;
Transformers->import(qw(make_checksum percentage_of shorten micro_t));
# Special formatting functions
my %formatting_function = (
ts => sub {
my ( $stats ) = @_;
my $min = parse_timestamp($stats->{min} || '');
my $max = parse_timestamp($stats->{max} || '');
return $min && $max ? "$min to $max" : '';
},
);
my $bool_format = '# %3s%% %-6s %s';
sub new {
my ( $class, %args ) = @_;
return bless { }, $class;
}
sub event_report {
my ( $self, %args ) = @_;
my @required_args = qw(where rank worst meta_ea hosts);
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
}
my ($where, $rank, $worst, $meta_ea, $hosts) = @args{@required_args};
my $meta_stats = $meta_ea->results;
my @result;
# First line
my $line = sprintf(
'# Query %d: ID 0x%s at byte %d ',
$rank || 0,
make_checksum($where) || '0x0',
0, # $sample->{pos_in_log} || 0
);
$line .= ('_' x (LINE_LENGTH - length($line)));
push @result, $line;
# Second line: full host names
# https://bugs.launchpad.net/percona-toolkit/+bug/980318
my $hostno = 0;
foreach my $host ( @$hosts ) {
$hostno++;
push @result, "# host$hostno: " . ($host->{name} || '?')
}
# Differences report. This relies on a sampleno attrib in each class
# since all other attributes (except maybe Query_time) are optional.
my $class = $meta_stats->{classes}->{$where};
push @result,
'# Found ' . ($class->{differences}->{sum} || 0)
. ' differences in ' . $class->{sampleno}->{cnt} . " samples:\n";
my $fmt = "# %-17s %d\n";
my @diffs = grep { $_ =~ m/^different_/ } keys %$class;
foreach my $diff ( sort @diffs ) {
push @result,
sprintf $fmt, ' ' . (make_label($diff) || ''), ($class->{$diff}->{sum} || 0);
}
# Side-by-side hosts report.
my $report = new ReportFormatter(
underline_header => 0,
strip_whitespace => 0,
);
$hostno = 0;
$report->set_columns(
{ name => '' },
map { $hostno++; { name => "host$hostno", right_justify => 1 } } @$hosts,
);
# Bool values.
foreach my $thing ( qw(Errors Warnings) ) {
my @vals = $thing;
foreach my $host ( @$hosts ) {
my $ea = $host->{ea};
my $stats = $ea->results->{classes}->{$where};
if ( $stats && $stats->{$thing} ) {
push @vals, shorten($stats->{$thing}->{sum}, d=>1_000, p=>0)
}
else {
push @vals, 0;
}
}
$report->add_line(@vals);
}
# Fully aggregated numeric values.
foreach my $thing ( qw(Query_time row_count) ) {
my @vals;
foreach my $host ( @$hosts ) {
my $ea = $host->{ea};
my $stats = $ea->results->{classes}->{$where};
if ( $stats && $stats->{$thing} ) {
my $vals = $stats->{$thing};
my $func = $thing =~ m/time$/ ? \&micro_t : \&shorten;
my $metrics = $host->{ea}->metrics(attrib=>$thing, where=>$where);
my @n = (
@{$vals}{qw(sum min max)},
($vals->{sum} || 0) / ($vals->{cnt} || 1),
@{$metrics}{qw(pct_95 stddev median)},
);
@n = map { defined $_ ? $func->($_) : '' } @n;
push @vals, \@n;
}
else {
push @vals, undef;
}
}
if ( scalar @vals && grep { defined } @vals ) {
$report->add_line($thing, map { '' } @$hosts);
my @metrics = qw(sum min max avg pct_95 stddev median);
for my $i ( 0..$#metrics ) {
my @n = ' ' . $metrics[$i];
push @n, map { $_ && defined $_->[$i] ? $_->[$i] : '' } @vals;
$report->add_line(@n);
}
}
}
push @result, $report->get_report();
return join("\n", map { s/\s+$//; $_ } @result) . "\n";
}
# Convert attribute names into labels
sub make_label {
my ( $val ) = @_;
$val =~ s/^different_//;
$val =~ s/_/ /g;
return $val;
}
# Does pretty-printing for lists of strings like users, hosts, db.
sub format_string_list {
my ( $stats ) = @_;
if ( exists $stats->{unq} ) {
# Only class stats have unq.
my $cnt_for = $stats->{unq};
if ( 1 == keys %$cnt_for ) {
my ($str) = keys %$cnt_for;
# - 30 for label, spacing etc.
$str = substr($str, 0, LINE_LENGTH - 30) . '...'
if length $str > LINE_LENGTH - 30;
return (1, $str);
}
my $line = '';
my @top = sort { $cnt_for->{$b} <=> $cnt_for->{$a} || $a cmp $b }
keys %$cnt_for;
my $i = 0;
foreach my $str ( @top ) {
my $print_str;
if ( length $str > MAX_STRING_LENGTH ) {
$print_str = substr($str, 0, MAX_STRING_LENGTH) . '...';
}
else {
$print_str = $str;
}
last if (length $line) + (length $print_str) > LINE_LENGTH - 27;
$line .= "$print_str ($cnt_for->{$str}), ";
$i++;
}
$line =~ s/, $//;
if ( $i < @top ) {
$line .= "... " . (@top - $i) . " more";
}
return (scalar keys %$cnt_for, $line);
}
else {
# Global stats don't have unq.
return ($stats->{cnt});
}
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
map { defined $_ ? $_ : 'undef' }
@_;
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
}
1;
}
# ###########################################################################
# End UpgradeReportFormatter package
# ###########################################################################

41
lib/UpgradeResults.pm Normal file
View File

@@ -0,0 +1,41 @@
# This program is copyright 2013 Percona Ireland Ltd.
# Feedback and improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA.
# ###########################################################################
# UpgradeResults package
# ###########################################################################
{
package UpgradeResults;
use Mo;
use Scalar::Util qw(blessed);
use English qw(-no_match_vars);
use constant PTDEBUG => $ENV{PTDEBUG} || 0;
has 'query_classes' => (
is => 'rw',
isa => 'HashRef',
required => 0,
default => sub { return {} },
);
no Mo;
1;
}
# ###########################################################################
# End UpgradeResults package
# ###########################################################################