diff --git a/bin/pt-duplicate-key-checker b/bin/pt-duplicate-key-checker index d5128421..1955e886 100755 --- a/bin/pt-duplicate-key-checker +++ b/bin/pt-duplicate-key-checker @@ -2772,262 +2772,6 @@ sub _d { # End DuplicateKeyFinder package # ########################################################################### -# ########################################################################### -# Transformers package -# This package is a copy without comments from the original. The original -# with comments and its test file can be found in the Bazaar repository at, -# lib/Transformers.pm -# t/lib/Transformers.t -# See https://launchpad.net/percona-toolkit for more information. -# ########################################################################### -{ -package Transformers; - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); -use constant MKDEBUG => $ENV{MKDEBUG} || 0; - -use Time::Local qw(timegm timelocal); -use Digest::MD5 qw(md5_hex); - -require Exporter; -our @ISA = qw(Exporter); -our %EXPORT_TAGS = (); -our @EXPORT = (); -our @EXPORT_OK = qw( - micro_t - percentage_of - secs_to_time - time_to_secs - shorten - ts - parse_timestamp - unix_timestamp - any_unix_timestamp - make_checksum - crc32 -); - -our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/; -our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/; -our $n_ts = qr/(\d{1,5})([shmd]?)/; # Limit \d{1,5} because \d{6} looks - -sub micro_t { - my ( $t, %args ) = @_; - my $p_ms = defined $args{p_ms} ? $args{p_ms} : 0; # precision for ms vals - my $p_s = defined $args{p_s} ? $args{p_s} : 0; # precision for s vals - my $f; - - $t = 0 if $t < 0; - - $t = sprintf('%.17f', $t) if $t =~ /e/; - - $t =~ s/\.(\d{1,6})\d*/\.$1/; - - if ($t > 0 && $t <= 0.000999) { - $f = ($t * 1000000) . 'us'; - } - elsif ($t >= 0.001000 && $t <= 0.999999) { - $f = sprintf("%.${p_ms}f", $t * 1000); - $f = ($f * 1) . 'ms'; # * 1 to remove insignificant zeros - } - elsif ($t >= 1) { - $f = sprintf("%.${p_s}f", $t); - $f = ($f * 1) . 's'; # * 1 to remove insignificant zeros - } - else { - $f = 0; # $t should = 0 at this point - } - - return $f; -} - -sub percentage_of { - my ( $is, $of, %args ) = @_; - my $p = $args{p} || 0; # float precision - my $fmt = $p ? "%.${p}f" : "%d"; - return sprintf $fmt, ($is * 100) / ($of ||= 1); -} - -sub secs_to_time { - my ( $secs, $fmt ) = @_; - $secs ||= 0; - return '00:00' unless $secs; - - $fmt ||= $secs >= 86_400 ? 'd' - : $secs >= 3_600 ? 'h' - : 'm'; - - return - $fmt eq 'd' ? sprintf( - "%d+%02d:%02d:%02d", - int($secs / 86_400), - int(($secs % 86_400) / 3_600), - int(($secs % 3_600) / 60), - $secs % 60) - : $fmt eq 'h' ? sprintf( - "%02d:%02d:%02d", - int(($secs % 86_400) / 3_600), - int(($secs % 3_600) / 60), - $secs % 60) - : sprintf( - "%02d:%02d", - int(($secs % 3_600) / 60), - $secs % 60); -} - -sub time_to_secs { - my ( $val, $default_suffix ) = @_; - die "I need a val argument" unless defined $val; - my $t = 0; - my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/; - $suffix = $suffix || $default_suffix || 's'; - if ( $suffix =~ m/[smhd]/ ) { - $t = $suffix eq 's' ? $num * 1 # Seconds - : $suffix eq 'm' ? $num * 60 # Minutes - : $suffix eq 'h' ? $num * 3600 # Hours - : $num * 86400; # Days - - $t *= -1 if $prefix && $prefix eq '-'; - } - else { - die "Invalid suffix for $val: $suffix"; - } - return $t; -} - -sub shorten { - my ( $num, %args ) = @_; - my $p = defined $args{p} ? $args{p} : 2; # float precision - my $d = defined $args{d} ? $args{d} : 1_024; # divisor - my $n = 0; - my @units = ('', qw(k M G T P E Z Y)); - while ( $num >= $d && $n < @units - 1 ) { - $num /= $d; - ++$n; - } - return sprintf( - $num =~ m/\./ || $n - ? "%.${p}f%s" - : '%d', - $num, $units[$n]); -} - -sub ts { - my ( $time, $gmt ) = @_; - my ( $sec, $min, $hour, $mday, $mon, $year ) - = $gmt ? gmtime($time) : localtime($time); - $mon += 1; - $year += 1900; - my $val = sprintf("%d-%02d-%02dT%02d:%02d:%02d", - $year, $mon, $mday, $hour, $min, $sec); - if ( my ($us) = $time =~ m/(\.\d+)$/ ) { - $us = sprintf("%.6f", $us); - $us =~ s/^0\././; - $val .= $us; - } - return $val; -} - -sub parse_timestamp { - my ( $val ) = @_; - if ( my($y, $m, $d, $h, $i, $s, $f) - = $val =~ m/^$mysql_ts$/ ) - { - return sprintf "%d-%02d-%02d %02d:%02d:" - . (defined $f ? '%09.6f' : '%02d'), - $y + 2000, $m, $d, $h, $i, (defined $f ? $s + $f : $s); - } - return $val; -} - -sub unix_timestamp { - my ( $val, $gmt ) = @_; - if ( my($y, $m, $d, $h, $i, $s, $us) = $val =~ m/^$proper_ts$/ ) { - $val = $gmt - ? timegm($s, $i, $h, $d, $m - 1, $y) - : timelocal($s, $i, $h, $d, $m - 1, $y); - if ( defined $us ) { - $us = sprintf('%.6f', $us); - $us =~ s/^0\././; - $val .= $us; - } - } - return $val; -} - -sub any_unix_timestamp { - my ( $val, $callback ) = @_; - - if ( my ($n, $suffix) = $val =~ m/^$n_ts$/ ) { - $n = $suffix eq 's' ? $n # Seconds - : $suffix eq 'm' ? $n * 60 # Minutes - : $suffix eq 'h' ? $n * 3600 # Hours - : $suffix eq 'd' ? $n * 86400 # Days - : $n; # default: Seconds - MKDEBUG && _d('ts is now - N[shmd]:', $n); - return time - $n; - } - elsif ( $val =~ m/^\d{9,}/ ) { - MKDEBUG && _d('ts is already a unix timestamp'); - return $val; - } - elsif ( my ($ymd, $hms) = $val =~ m/^(\d{6})(?:\s+(\d+:\d+:\d+))?/ ) { - MKDEBUG && _d('ts is MySQL slow log timestamp'); - $val .= ' 00:00:00' unless $hms; - return unix_timestamp(parse_timestamp($val)); - } - elsif ( ($ymd, $hms) = $val =~ m/^(\d{4}-\d\d-\d\d)(?:[T ](\d+:\d+:\d+))?/) { - MKDEBUG && _d('ts is properly formatted timestamp'); - $val .= ' 00:00:00' unless $hms; - return unix_timestamp($val); - } - else { - MKDEBUG && _d('ts is MySQL expression'); - return $callback->($val) if $callback && ref $callback eq 'CODE'; - } - - MKDEBUG && _d('Unknown ts type:', $val); - return; -} - -sub make_checksum { - my ( $val ) = @_; - my $checksum = uc substr(md5_hex($val), -16); - MKDEBUG && _d($checksum, 'checksum for', $val); - return $checksum; -} - -sub crc32 { - my ( $string ) = @_; - return unless $string; - my $poly = 0xEDB88320; - my $crc = 0xFFFFFFFF; - foreach my $char ( split(//, $string) ) { - my $comp = ($crc ^ ord($char)) & 0xFF; - for ( 1 .. 8 ) { - $comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1; - } - $crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp; - } - return $crc ^ 0xFFFFFFFF; -} - -sub _d { - my ($package, undef, $line) = caller 0; - @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } - map { defined $_ ? $_ : 'undef' } - @_; - print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; -} - -1; -} -# ########################################################################### -# End Transformers package -# ########################################################################### - # ########################################################################### # Daemon package # This package is a copy without comments from the original. The original @@ -3214,6 +2958,202 @@ sub _d { # End Daemon package # ########################################################################### +# ########################################################################### +# Schema package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/Schema.pm +# t/lib/Schema.t +# See https://launchpad.net/percona-toolkit for more information. +# ########################################################################### +{ +package Schema; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use constant MKDEBUG => $ENV{MKDEBUG} || 0; + +sub new { + my ( $class, %args ) = @_; + my @required_args = qw(); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + + my $self = { + %args, + schema => {}, # keyed on db->tbl + }; + return bless $self, $class; +} + +sub get_schema { + my ( $self ) = @_; + return $self->{schema}; +} + +sub get_table { + my ( $self, $db_name, $tbl_name ) = @_; + if ( exists $self->{schema}->{$db_name} + && exists $self->{schema}->{$db_name}->{$tbl_name} ) { + return $self->{schema}->{$db_name}->{$tbl_name}; + } + return; +} + + + +sub add_schema_object { + my ( $self, $schema_object ) = @_; + die "I need a schema_object argument" unless $schema_object; + + my ($db, $tbl) = @{$schema_object}{qw(db tbl)}; + if ( !$db || !$tbl ) { + warn "No database or table for schema object"; + return; + } + + my $tbl_struct = $schema_object->{tbl_struct}; + if ( !$tbl_struct ) { + warn "No table structure for $db.$tbl"; + return; + } + + $self->{schema}->{lc $db}->{lc $tbl} = $schema_object; + + + return; +} + +sub find_column { + my ( $self, %args ) = @_; + my $ignore = $args{ignore}; + my $schema = $self->{schema}; + + my ($col, $tbl, $db); + if ( my $col_name = $args{col_name} ) { + ($col, $tbl, $db) = reverse map { s/`//g; $_ } split /[.]/, $col_name; + MKDEBUG && _d('Column', $col_name, 'has db', $db, 'tbl', $tbl, + 'col', $col); + } + else { + ($col, $tbl, $db) = @args{qw(col tbl db)}; + } + + $db = lc $db; + $tbl = lc $tbl; + $col = lc $col; + + if ( !$col ) { + MKDEBUG && _d('No column specified or parsed'); + return; + } + MKDEBUG && _d('Finding column', $col, 'in', $db, $tbl); + + if ( $db && !$schema->{$db} ) { + MKDEBUG && _d('Database', $db, 'does not exist'); + return; + } + + if ( $db && $tbl && !$schema->{$db}->{$tbl} ) { + MKDEBUG && _d('Table', $tbl, 'does not exist in database', $db); + return; + } + + my @tbls; + my @search_dbs = $db ? ($db) : keys %$schema; + DATABASE: + foreach my $search_db ( @search_dbs ) { + my @search_tbls = $tbl ? ($tbl) : keys %{$schema->{$search_db}}; + + TABLE: + foreach my $search_tbl ( @search_tbls ) { + next DATABASE unless exists $schema->{$search_db}->{$search_tbl}; + + if ( $ignore + && grep { $_->{db} eq $search_db && $_->{tbl} eq $search_tbl } + @$ignore ) { + MKDEBUG && _d('Ignoring', $search_db, $search_tbl, $col); + next TABLE; + } + + my $tbl = $schema->{$search_db}->{$search_tbl}; + if ( $tbl->{tbl_struct}->{is_col}->{$col} ) { + MKDEBUG && _d('Column', $col, 'exists in', $tbl->{db}, $tbl->{tbl}); + push @tbls, $tbl; + } + } + } + + return \@tbls; +} + +sub find_table { + my ( $self, %args ) = @_; + my $ignore = $args{ignore}; + my $schema = $self->{schema}; + + my ($tbl, $db); + if ( my $tbl_name = $args{tbl_name} ) { + ($tbl, $db) = reverse map { s/`//g; $_ } split /[.]/, $tbl_name; + MKDEBUG && _d('Table', $tbl_name, 'has db', $db, 'tbl', $tbl); + } + else { + ($tbl, $db) = @args{qw(tbl db)}; + } + + $db = lc $db; + $tbl = lc $tbl; + + if ( !$tbl ) { + MKDEBUG && _d('No table specified or parsed'); + return; + } + MKDEBUG && _d('Finding table', $tbl, 'in', $db); + + if ( $db && !$schema->{$db} ) { + MKDEBUG && _d('Database', $db, 'does not exist'); + return; + } + + if ( $db && $tbl && !$schema->{$db}->{$tbl} ) { + MKDEBUG && _d('Table', $tbl, 'does not exist in database', $db); + return; + } + + my @dbs; + my @search_dbs = $db ? ($db) : keys %$schema; + DATABASE: + foreach my $search_db ( @search_dbs ) { + if ( $ignore && grep { $_->{db} eq $search_db } @$ignore ) { + MKDEBUG && _d('Ignoring', $search_db); + next DATABASE; + } + + if ( exists $schema->{$search_db}->{$tbl} ) { + MKDEBUG && _d('Table', $tbl, 'exists in', $search_db); + push @dbs, $search_db; + } + } + + return \@dbs; +} + +sub _d { + my ($package, undef, $line) = caller 0; + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } + map { defined $_ ? $_ : 'undef' } + @_; + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; +} + +1; +} +# ########################################################################### +# End Schema package +# ########################################################################### + # ########################################################################### # SchemaIterator package # This package is a copy without comments from the original. The original @@ -3611,14 +3551,13 @@ sub _d { # ############################################################################# package pt_duplicate_key_checker; +use strict; +use warnings FATAL => 'all'; use English qw(-no_match_vars); -use Getopt::Long; -use List::Util qw(max); - -Transformers->import(qw(shorten)); - use constant MKDEBUG => $ENV{MKDEBUG} || 0; +use List::Util qw(max); + $OUTPUT_AUTOFLUSH = 1; my $max_width = 74; @@ -3659,12 +3598,8 @@ sub main { } # ####################################################################### - # Get ready to do the main work. + # Connect to MySQL. # ####################################################################### - my $get_keys = $o->get('key-types') =~ m/k/ ? 1 : 0; - my $get_fks = $o->get('key-types') =~ m/f/ ? 1 : 0; - - # Connect to the database if ( $o->got('ask-pass') ) { $o->set('password', OptionParser::prompt_noecho("Enter password: ")); } @@ -3677,6 +3612,9 @@ sub main { my $vp = new VersionParser(); my $version = $vp->parse($dbh->selectrow_array('SELECT VERSION()')); + # ####################################################################### + # Do the main work. + # ####################################################################### my $ks = $o->get('summary') ? new KeySize(q=>$q) : undef; my $dk = new DuplicateKeyFinder(); my $du = new MySQLDump(); @@ -3687,58 +3625,46 @@ sub main { clustered => $o->get('clustered'), ); - # ####################################################################### - # Do the main work. - # ####################################################################### + my $get_keys = $o->get('key-types') =~ m/k/ ? 1 : 0; + my $get_fks = $o->get('key-types') =~ m/f/ ? 1 : 0; - my $si = new SchemaIterator( - Quoter => $q, + my $schema = new Schema(); + my $schema_itr = new SchemaIterator( + dbh => $dbh, + OptionParser => $o, + Quoter => $q, + MySQLDump => $du, + TableParser => $tp, + Schema => $schema, + keep_ddl => 1, ); - $si->set_filter($si->make_filter($o)); - my $next_db = $si->get_db_itr(dbh => $dbh); - DATABASE: - while ( my $database = $next_db->() ) { - MKDEBUG && _d('Getting tables from', $database); - my $next_tbl = $si->get_tbl_itr( - dbh => $dbh, - db => $database, - views => 0, - ); - TABLE: - while ( my $table = $next_tbl->() ) { - MKDEBUG && _d('Got table', $table); + TABLE: + while ( my $tbl = $schema_itr->next_schema_object() ) { + $tbl->{engine} = $tp->get_engine($tbl->{ddl}); - # If get_create_table() fails, it will throw a warning and return - # undef. So we can just move on to the next table. - my $ddl = $du->get_create_table($dbh, $q, $database, $table); - next TABLE unless $ddl; - $ddl = $ddl->[1]; # retval is an arrayref: [table|view, SHOW CREATE] + my ($keys, $clustered_key, $fks); + if ( $get_keys ) { + ($keys, $clustered_key) + = $tp->get_keys($tbl->{ddl}, {version => $version}); + } + if ( $get_fks ) { + $fks = $tp->get_fks($tbl->{ddl}, {database => $tbl->{db}}); + } - my $engine = $tp->get_engine($ddl) || next TABLE; - my $tbl_info = { - db => $database, - tbl => $table, - engine => $engine, - ddl => $ddl, - }; + next TABLE unless %$keys || %$fks; - my ($keys, $clustered_key) - = $tp->get_keys($ddl, {version => $version }) if $get_keys; - my $fks = $tp->get_fks($ddl, {database => $database}) if $get_fks; - - next TABLE unless %$keys || %$fks; - - if ( $o->got('verbose') ) { - print_all_keys($keys, $tbl_info, \%seen_tbl) if $keys; - print_all_keys($fks, $tbl_info, \%seen_tbl) if $fks; - } - else { - MKDEBUG && _d('Getting duplicate keys on', $database, $table); - eval { + if ( $o->got('verbose') ) { + print_all_keys($keys, $tbl, \%seen_tbl) if $keys; + print_all_keys($fks, $tbl, \%seen_tbl) if $fks; + } + else { + MKDEBUG && _d('Getting duplicate keys on', $tbl->{db}, $tbl->{tbl}); + eval { + if ( $keys ) { $dk->get_duplicate_keys( $keys, clustered_key => $clustered_key, - tbl_info => $tbl_info, + tbl_info => $tbl, callback => \&print_duplicate_key, %tp_opts, # get_duplicate_keys() ignores these args but passes them @@ -3751,11 +3677,12 @@ sub main { q => $q, seen_tbl => \%seen_tbl, summary => \%summary, - ) if $keys; - + ); + } + if ( $fks ) { $dk->get_duplicate_fks( $fks, - tbl_info => $tbl_info, + tbl_info => $tbl, callback => \&print_duplicate_key, %tp_opts, # get_duplicate_fks() ignores these args but passes them @@ -3768,20 +3695,20 @@ sub main { q => $q, seen_tbl => \%seen_tbl, summary => \%summary, - ) if $fks; - }; - if ( $EVAL_ERROR ) { - warn "Error checking `$database`.`$table` for duplicate keys: " - . $EVAL_ERROR; - next TABLE; + ); } + }; + if ( $EVAL_ERROR ) { + warn "Error checking `$tbl->{db}`.`$tbl->{tbl}` for duplicate keys: " + . $EVAL_ERROR; + next TABLE; } + } - # Always count Total Keys so print_key_summary won't die - # because %summary is empty. - $summary{'Total Indexes'} += (scalar keys %$keys) + (scalar keys %$fks) - } # TABLE - } # DATABASE + # Always count Total Keys so print_key_summary won't die + # because %summary is empty. + $summary{'Total Indexes'} += (scalar keys %$keys) + (scalar keys %$fks) + } # TABLE print_key_summary(%summary) if $o->get('summary');