Use new Schema and SchemaIterator in pt-duplicate-key-checker. Remove unused Transformers package from tool.

This commit is contained in:
Daniel Nichter
2011-07-13 10:22:26 -06:00
parent 9621291f89
commit 1a869239bc
+251 -324
View File
@@ -2772,262 +2772,6 @@ sub _d {
# End DuplicateKeyFinder package
# ###########################################################################
# ###########################################################################
# Transformers package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/Transformers.pm
# t/lib/Transformers.t
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
{
package Transformers;
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
use Time::Local qw(timegm timelocal);
use Digest::MD5 qw(md5_hex);
require Exporter;
our @ISA = qw(Exporter);
our %EXPORT_TAGS = ();
our @EXPORT = ();
our @EXPORT_OK = qw(
micro_t
percentage_of
secs_to_time
time_to_secs
shorten
ts
parse_timestamp
unix_timestamp
any_unix_timestamp
make_checksum
crc32
);
our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/;
our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/;
our $n_ts = qr/(\d{1,5})([shmd]?)/; # Limit \d{1,5} because \d{6} looks
sub micro_t {
my ( $t, %args ) = @_;
my $p_ms = defined $args{p_ms} ? $args{p_ms} : 0; # precision for ms vals
my $p_s = defined $args{p_s} ? $args{p_s} : 0; # precision for s vals
my $f;
$t = 0 if $t < 0;
$t = sprintf('%.17f', $t) if $t =~ /e/;
$t =~ s/\.(\d{1,6})\d*/\.$1/;
if ($t > 0 && $t <= 0.000999) {
$f = ($t * 1000000) . 'us';
}
elsif ($t >= 0.001000 && $t <= 0.999999) {
$f = sprintf("%.${p_ms}f", $t * 1000);
$f = ($f * 1) . 'ms'; # * 1 to remove insignificant zeros
}
elsif ($t >= 1) {
$f = sprintf("%.${p_s}f", $t);
$f = ($f * 1) . 's'; # * 1 to remove insignificant zeros
}
else {
$f = 0; # $t should = 0 at this point
}
return $f;
}
sub percentage_of {
my ( $is, $of, %args ) = @_;
my $p = $args{p} || 0; # float precision
my $fmt = $p ? "%.${p}f" : "%d";
return sprintf $fmt, ($is * 100) / ($of ||= 1);
}
sub secs_to_time {
my ( $secs, $fmt ) = @_;
$secs ||= 0;
return '00:00' unless $secs;
$fmt ||= $secs >= 86_400 ? 'd'
: $secs >= 3_600 ? 'h'
: 'm';
return
$fmt eq 'd' ? sprintf(
"%d+%02d:%02d:%02d",
int($secs / 86_400),
int(($secs % 86_400) / 3_600),
int(($secs % 3_600) / 60),
$secs % 60)
: $fmt eq 'h' ? sprintf(
"%02d:%02d:%02d",
int(($secs % 86_400) / 3_600),
int(($secs % 3_600) / 60),
$secs % 60)
: sprintf(
"%02d:%02d",
int(($secs % 3_600) / 60),
$secs % 60);
}
sub time_to_secs {
my ( $val, $default_suffix ) = @_;
die "I need a val argument" unless defined $val;
my $t = 0;
my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/;
$suffix = $suffix || $default_suffix || 's';
if ( $suffix =~ m/[smhd]/ ) {
$t = $suffix eq 's' ? $num * 1 # Seconds
: $suffix eq 'm' ? $num * 60 # Minutes
: $suffix eq 'h' ? $num * 3600 # Hours
: $num * 86400; # Days
$t *= -1 if $prefix && $prefix eq '-';
}
else {
die "Invalid suffix for $val: $suffix";
}
return $t;
}
sub shorten {
my ( $num, %args ) = @_;
my $p = defined $args{p} ? $args{p} : 2; # float precision
my $d = defined $args{d} ? $args{d} : 1_024; # divisor
my $n = 0;
my @units = ('', qw(k M G T P E Z Y));
while ( $num >= $d && $n < @units - 1 ) {
$num /= $d;
++$n;
}
return sprintf(
$num =~ m/\./ || $n
? "%.${p}f%s"
: '%d',
$num, $units[$n]);
}
sub ts {
my ( $time, $gmt ) = @_;
my ( $sec, $min, $hour, $mday, $mon, $year )
= $gmt ? gmtime($time) : localtime($time);
$mon += 1;
$year += 1900;
my $val = sprintf("%d-%02d-%02dT%02d:%02d:%02d",
$year, $mon, $mday, $hour, $min, $sec);
if ( my ($us) = $time =~ m/(\.\d+)$/ ) {
$us = sprintf("%.6f", $us);
$us =~ s/^0\././;
$val .= $us;
}
return $val;
}
sub parse_timestamp {
my ( $val ) = @_;
if ( my($y, $m, $d, $h, $i, $s, $f)
= $val =~ m/^$mysql_ts$/ )
{
return sprintf "%d-%02d-%02d %02d:%02d:"
. (defined $f ? '%09.6f' : '%02d'),
$y + 2000, $m, $d, $h, $i, (defined $f ? $s + $f : $s);
}
return $val;
}
sub unix_timestamp {
my ( $val, $gmt ) = @_;
if ( my($y, $m, $d, $h, $i, $s, $us) = $val =~ m/^$proper_ts$/ ) {
$val = $gmt
? timegm($s, $i, $h, $d, $m - 1, $y)
: timelocal($s, $i, $h, $d, $m - 1, $y);
if ( defined $us ) {
$us = sprintf('%.6f', $us);
$us =~ s/^0\././;
$val .= $us;
}
}
return $val;
}
sub any_unix_timestamp {
my ( $val, $callback ) = @_;
if ( my ($n, $suffix) = $val =~ m/^$n_ts$/ ) {
$n = $suffix eq 's' ? $n # Seconds
: $suffix eq 'm' ? $n * 60 # Minutes
: $suffix eq 'h' ? $n * 3600 # Hours
: $suffix eq 'd' ? $n * 86400 # Days
: $n; # default: Seconds
MKDEBUG && _d('ts is now - N[shmd]:', $n);
return time - $n;
}
elsif ( $val =~ m/^\d{9,}/ ) {
MKDEBUG && _d('ts is already a unix timestamp');
return $val;
}
elsif ( my ($ymd, $hms) = $val =~ m/^(\d{6})(?:\s+(\d+:\d+:\d+))?/ ) {
MKDEBUG && _d('ts is MySQL slow log timestamp');
$val .= ' 00:00:00' unless $hms;
return unix_timestamp(parse_timestamp($val));
}
elsif ( ($ymd, $hms) = $val =~ m/^(\d{4}-\d\d-\d\d)(?:[T ](\d+:\d+:\d+))?/) {
MKDEBUG && _d('ts is properly formatted timestamp');
$val .= ' 00:00:00' unless $hms;
return unix_timestamp($val);
}
else {
MKDEBUG && _d('ts is MySQL expression');
return $callback->($val) if $callback && ref $callback eq 'CODE';
}
MKDEBUG && _d('Unknown ts type:', $val);
return;
}
sub make_checksum {
my ( $val ) = @_;
my $checksum = uc substr(md5_hex($val), -16);
MKDEBUG && _d($checksum, 'checksum for', $val);
return $checksum;
}
sub crc32 {
my ( $string ) = @_;
return unless $string;
my $poly = 0xEDB88320;
my $crc = 0xFFFFFFFF;
foreach my $char ( split(//, $string) ) {
my $comp = ($crc ^ ord($char)) & 0xFF;
for ( 1 .. 8 ) {
$comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1;
}
$crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp;
}
return $crc ^ 0xFFFFFFFF;
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
map { defined $_ ? $_ : 'undef' }
@_;
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
}
1;
}
# ###########################################################################
# End Transformers package
# ###########################################################################
# ###########################################################################
# Daemon package
# This package is a copy without comments from the original. The original
@@ -3214,6 +2958,202 @@ sub _d {
# End Daemon package
# ###########################################################################
# ###########################################################################
# Schema package
# This package is a copy without comments from the original. The original
# with comments and its test file can be found in the Bazaar repository at,
# lib/Schema.pm
# t/lib/Schema.t
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################
{
package Schema;
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
sub new {
my ( $class, %args ) = @_;
my @required_args = qw();
foreach my $arg ( @required_args ) {
die "I need a $arg argument" unless $args{$arg};
}
my $self = {
%args,
schema => {}, # keyed on db->tbl
};
return bless $self, $class;
}
sub get_schema {
my ( $self ) = @_;
return $self->{schema};
}
sub get_table {
my ( $self, $db_name, $tbl_name ) = @_;
if ( exists $self->{schema}->{$db_name}
&& exists $self->{schema}->{$db_name}->{$tbl_name} ) {
return $self->{schema}->{$db_name}->{$tbl_name};
}
return;
}
sub add_schema_object {
my ( $self, $schema_object ) = @_;
die "I need a schema_object argument" unless $schema_object;
my ($db, $tbl) = @{$schema_object}{qw(db tbl)};
if ( !$db || !$tbl ) {
warn "No database or table for schema object";
return;
}
my $tbl_struct = $schema_object->{tbl_struct};
if ( !$tbl_struct ) {
warn "No table structure for $db.$tbl";
return;
}
$self->{schema}->{lc $db}->{lc $tbl} = $schema_object;
return;
}
sub find_column {
my ( $self, %args ) = @_;
my $ignore = $args{ignore};
my $schema = $self->{schema};
my ($col, $tbl, $db);
if ( my $col_name = $args{col_name} ) {
($col, $tbl, $db) = reverse map { s/`//g; $_ } split /[.]/, $col_name;
MKDEBUG && _d('Column', $col_name, 'has db', $db, 'tbl', $tbl,
'col', $col);
}
else {
($col, $tbl, $db) = @args{qw(col tbl db)};
}
$db = lc $db;
$tbl = lc $tbl;
$col = lc $col;
if ( !$col ) {
MKDEBUG && _d('No column specified or parsed');
return;
}
MKDEBUG && _d('Finding column', $col, 'in', $db, $tbl);
if ( $db && !$schema->{$db} ) {
MKDEBUG && _d('Database', $db, 'does not exist');
return;
}
if ( $db && $tbl && !$schema->{$db}->{$tbl} ) {
MKDEBUG && _d('Table', $tbl, 'does not exist in database', $db);
return;
}
my @tbls;
my @search_dbs = $db ? ($db) : keys %$schema;
DATABASE:
foreach my $search_db ( @search_dbs ) {
my @search_tbls = $tbl ? ($tbl) : keys %{$schema->{$search_db}};
TABLE:
foreach my $search_tbl ( @search_tbls ) {
next DATABASE unless exists $schema->{$search_db}->{$search_tbl};
if ( $ignore
&& grep { $_->{db} eq $search_db && $_->{tbl} eq $search_tbl }
@$ignore ) {
MKDEBUG && _d('Ignoring', $search_db, $search_tbl, $col);
next TABLE;
}
my $tbl = $schema->{$search_db}->{$search_tbl};
if ( $tbl->{tbl_struct}->{is_col}->{$col} ) {
MKDEBUG && _d('Column', $col, 'exists in', $tbl->{db}, $tbl->{tbl});
push @tbls, $tbl;
}
}
}
return \@tbls;
}
sub find_table {
my ( $self, %args ) = @_;
my $ignore = $args{ignore};
my $schema = $self->{schema};
my ($tbl, $db);
if ( my $tbl_name = $args{tbl_name} ) {
($tbl, $db) = reverse map { s/`//g; $_ } split /[.]/, $tbl_name;
MKDEBUG && _d('Table', $tbl_name, 'has db', $db, 'tbl', $tbl);
}
else {
($tbl, $db) = @args{qw(tbl db)};
}
$db = lc $db;
$tbl = lc $tbl;
if ( !$tbl ) {
MKDEBUG && _d('No table specified or parsed');
return;
}
MKDEBUG && _d('Finding table', $tbl, 'in', $db);
if ( $db && !$schema->{$db} ) {
MKDEBUG && _d('Database', $db, 'does not exist');
return;
}
if ( $db && $tbl && !$schema->{$db}->{$tbl} ) {
MKDEBUG && _d('Table', $tbl, 'does not exist in database', $db);
return;
}
my @dbs;
my @search_dbs = $db ? ($db) : keys %$schema;
DATABASE:
foreach my $search_db ( @search_dbs ) {
if ( $ignore && grep { $_->{db} eq $search_db } @$ignore ) {
MKDEBUG && _d('Ignoring', $search_db);
next DATABASE;
}
if ( exists $schema->{$search_db}->{$tbl} ) {
MKDEBUG && _d('Table', $tbl, 'exists in', $search_db);
push @dbs, $search_db;
}
}
return \@dbs;
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
map { defined $_ ? $_ : 'undef' }
@_;
print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
}
1;
}
# ###########################################################################
# End Schema package
# ###########################################################################
# ###########################################################################
# SchemaIterator package
# This package is a copy without comments from the original. The original
@@ -3611,14 +3551,13 @@ sub _d {
# #############################################################################
package pt_duplicate_key_checker;
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Getopt::Long;
use List::Util qw(max);
Transformers->import(qw(shorten));
use constant MKDEBUG => $ENV{MKDEBUG} || 0;
use List::Util qw(max);
$OUTPUT_AUTOFLUSH = 1;
my $max_width = 74;
@@ -3659,12 +3598,8 @@ sub main {
}
# #######################################################################
# Get ready to do the main work.
# Connect to MySQL.
# #######################################################################
my $get_keys = $o->get('key-types') =~ m/k/ ? 1 : 0;
my $get_fks = $o->get('key-types') =~ m/f/ ? 1 : 0;
# Connect to the database
if ( $o->got('ask-pass') ) {
$o->set('password', OptionParser::prompt_noecho("Enter password: "));
}
@@ -3677,6 +3612,9 @@ sub main {
my $vp = new VersionParser();
my $version = $vp->parse($dbh->selectrow_array('SELECT VERSION()'));
# #######################################################################
# Do the main work.
# #######################################################################
my $ks = $o->get('summary') ? new KeySize(q=>$q) : undef;
my $dk = new DuplicateKeyFinder();
my $du = new MySQLDump();
@@ -3687,58 +3625,46 @@ sub main {
clustered => $o->get('clustered'),
);
# #######################################################################
# Do the main work.
# #######################################################################
my $get_keys = $o->get('key-types') =~ m/k/ ? 1 : 0;
my $get_fks = $o->get('key-types') =~ m/f/ ? 1 : 0;
my $si = new SchemaIterator(
Quoter => $q,
my $schema = new Schema();
my $schema_itr = new SchemaIterator(
dbh => $dbh,
OptionParser => $o,
Quoter => $q,
MySQLDump => $du,
TableParser => $tp,
Schema => $schema,
keep_ddl => 1,
);
$si->set_filter($si->make_filter($o));
my $next_db = $si->get_db_itr(dbh => $dbh);
DATABASE:
while ( my $database = $next_db->() ) {
MKDEBUG && _d('Getting tables from', $database);
my $next_tbl = $si->get_tbl_itr(
dbh => $dbh,
db => $database,
views => 0,
);
TABLE:
while ( my $table = $next_tbl->() ) {
MKDEBUG && _d('Got table', $table);
TABLE:
while ( my $tbl = $schema_itr->next_schema_object() ) {
$tbl->{engine} = $tp->get_engine($tbl->{ddl});
# If get_create_table() fails, it will throw a warning and return
# undef. So we can just move on to the next table.
my $ddl = $du->get_create_table($dbh, $q, $database, $table);
next TABLE unless $ddl;
$ddl = $ddl->[1]; # retval is an arrayref: [table|view, SHOW CREATE]
my ($keys, $clustered_key, $fks);
if ( $get_keys ) {
($keys, $clustered_key)
= $tp->get_keys($tbl->{ddl}, {version => $version});
}
if ( $get_fks ) {
$fks = $tp->get_fks($tbl->{ddl}, {database => $tbl->{db}});
}
my $engine = $tp->get_engine($ddl) || next TABLE;
my $tbl_info = {
db => $database,
tbl => $table,
engine => $engine,
ddl => $ddl,
};
next TABLE unless %$keys || %$fks;
my ($keys, $clustered_key)
= $tp->get_keys($ddl, {version => $version }) if $get_keys;
my $fks = $tp->get_fks($ddl, {database => $database}) if $get_fks;
next TABLE unless %$keys || %$fks;
if ( $o->got('verbose') ) {
print_all_keys($keys, $tbl_info, \%seen_tbl) if $keys;
print_all_keys($fks, $tbl_info, \%seen_tbl) if $fks;
}
else {
MKDEBUG && _d('Getting duplicate keys on', $database, $table);
eval {
if ( $o->got('verbose') ) {
print_all_keys($keys, $tbl, \%seen_tbl) if $keys;
print_all_keys($fks, $tbl, \%seen_tbl) if $fks;
}
else {
MKDEBUG && _d('Getting duplicate keys on', $tbl->{db}, $tbl->{tbl});
eval {
if ( $keys ) {
$dk->get_duplicate_keys(
$keys,
clustered_key => $clustered_key,
tbl_info => $tbl_info,
tbl_info => $tbl,
callback => \&print_duplicate_key,
%tp_opts,
# get_duplicate_keys() ignores these args but passes them
@@ -3751,11 +3677,12 @@ sub main {
q => $q,
seen_tbl => \%seen_tbl,
summary => \%summary,
) if $keys;
);
}
if ( $fks ) {
$dk->get_duplicate_fks(
$fks,
tbl_info => $tbl_info,
tbl_info => $tbl,
callback => \&print_duplicate_key,
%tp_opts,
# get_duplicate_fks() ignores these args but passes them
@@ -3768,20 +3695,20 @@ sub main {
q => $q,
seen_tbl => \%seen_tbl,
summary => \%summary,
) if $fks;
};
if ( $EVAL_ERROR ) {
warn "Error checking `$database`.`$table` for duplicate keys: "
. $EVAL_ERROR;
next TABLE;
);
}
};
if ( $EVAL_ERROR ) {
warn "Error checking `$tbl->{db}`.`$tbl->{tbl}` for duplicate keys: "
. $EVAL_ERROR;
next TABLE;
}
}
# Always count Total Keys so print_key_summary won't die
# because %summary is empty.
$summary{'Total Indexes'} += (scalar keys %$keys) + (scalar keys %$fks)
} # TABLE
} # DATABASE
# Always count Total Keys so print_key_summary won't die
# because %summary is empty.
$summary{'Total Indexes'} += (scalar keys %$keys) + (scalar keys %$fks)
} # TABLE
print_key_summary(%summary) if $o->get('summary');