#!/usr/bin/env perl # This program is part of Percona Toolkit: http://www.percona.com/software/ # See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal # notices and disclaimers. use strict; use warnings FATAL => 'all'; use constant MKDEBUG => $ENV{MKDEBUG} || 0; # ########################################################################### # DSNParser package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/DSNParser.pm # t/lib/DSNParser.t # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { package DSNParser; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; use Data::Dumper; $Data::Dumper::Indent = 0; $Data::Dumper::Quotekeys = 0; eval { require DBI; }; my $have_dbi = $EVAL_ERROR ? 0 : 1; sub new { my ( $class, %args ) = @_; foreach my $arg ( qw(opts) ) { die "I need a $arg argument" unless $args{$arg}; } my $self = { opts => {} # h, P, u, etc. Should come from DSN OPTIONS section in POD. }; foreach my $opt ( @{$args{opts}} ) { if ( !$opt->{key} || !$opt->{desc} ) { die "Invalid DSN option: ", Dumper($opt); } MKDEBUG && _d('DSN option:', join(', ', map { "$_=" . (defined $opt->{$_} ? ($opt->{$_} || '') : 'undef') } keys %$opt ) ); $self->{opts}->{$opt->{key}} = { dsn => $opt->{dsn}, desc => $opt->{desc}, copy => $opt->{copy} || 0, }; } return bless $self, $class; } sub prop { my ( $self, $prop, $value ) = @_; if ( @_ > 2 ) { MKDEBUG && _d('Setting', $prop, 'property'); $self->{$prop} = $value; } return $self->{$prop}; } sub parse { my ( $self, $dsn, $prev, $defaults ) = @_; if ( !$dsn ) { MKDEBUG && _d('No DSN to parse'); return; } MKDEBUG && _d('Parsing', $dsn); $prev ||= {}; $defaults ||= {}; my %given_props; my %final_props; my $opts = $self->{opts}; foreach my $dsn_part ( split(/,/, $dsn) ) { if ( my ($prop_key, $prop_val) = $dsn_part =~ m/^(.)=(.*)$/ ) { $given_props{$prop_key} = $prop_val; } else { MKDEBUG && _d('Interpreting', $dsn_part, 'as h=', $dsn_part); $given_props{h} = $dsn_part; } } foreach my $key ( keys %$opts ) { MKDEBUG && _d('Finding value for', $key); $final_props{$key} = $given_props{$key}; if ( !defined $final_props{$key} && defined $prev->{$key} && $opts->{$key}->{copy} ) { $final_props{$key} = $prev->{$key}; MKDEBUG && _d('Copying value for', $key, 'from previous DSN'); } if ( !defined $final_props{$key} ) { $final_props{$key} = $defaults->{$key}; MKDEBUG && _d('Copying value for', $key, 'from defaults'); } } foreach my $key ( keys %given_props ) { die "Unknown DSN option '$key' in '$dsn'. For more details, " . "please use the --help option, or try 'perldoc $PROGRAM_NAME' " . "for complete documentation." unless exists $opts->{$key}; } if ( (my $required = $self->prop('required')) ) { foreach my $key ( keys %$required ) { die "Missing required DSN option '$key' in '$dsn'. For more details, " . "please use the --help option, or try 'perldoc $PROGRAM_NAME' " . "for complete documentation." unless $final_props{$key}; } } return \%final_props; } sub parse_options { my ( $self, $o ) = @_; die 'I need an OptionParser object' unless ref $o eq 'OptionParser'; my $dsn_string = join(',', map { "$_=".$o->get($_); } grep { $o->has($_) && $o->get($_) } keys %{$self->{opts}} ); MKDEBUG && _d('DSN string made from options:', $dsn_string); return $self->parse($dsn_string); } sub as_string { my ( $self, $dsn, $props ) = @_; return $dsn unless ref $dsn; my %allowed = $props ? map { $_=>1 } @$props : (); return join(',', map { "$_=" . ($_ eq 'p' ? '...' : $dsn->{$_}) } grep { defined $dsn->{$_} && $self->{opts}->{$_} } grep { !$props || $allowed{$_} } sort keys %$dsn ); } sub usage { my ( $self ) = @_; my $usage = "DSN syntax is key=value[,key=value...] Allowable DSN keys:\n\n" . " KEY COPY MEANING\n" . " === ==== =============================================\n"; my %opts = %{$self->{opts}}; foreach my $key ( sort keys %opts ) { $usage .= " $key " . ($opts{$key}->{copy} ? 'yes ' : 'no ') . ($opts{$key}->{desc} || '[No description]') . "\n"; } $usage .= "\n If the DSN is a bareword, the word is treated as the 'h' key.\n"; return $usage; } sub get_cxn_params { my ( $self, $info ) = @_; my $dsn; my %opts = %{$self->{opts}}; my $driver = $self->prop('dbidriver') || ''; if ( $driver eq 'Pg' ) { $dsn = 'DBI:Pg:dbname=' . ( $info->{D} || '' ) . ';' . join(';', map { "$opts{$_}->{dsn}=$info->{$_}" } grep { defined $info->{$_} } qw(h P)); } else { $dsn = 'DBI:mysql:' . ( $info->{D} || '' ) . ';' . join(';', map { "$opts{$_}->{dsn}=$info->{$_}" } grep { defined $info->{$_} } qw(F h P S A)) . ';mysql_read_default_group=client'; } MKDEBUG && _d($dsn); return ($dsn, $info->{u}, $info->{p}); } sub fill_in_dsn { my ( $self, $dbh, $dsn ) = @_; my $vars = $dbh->selectall_hashref('SHOW VARIABLES', 'Variable_name'); my ($user, $db) = $dbh->selectrow_array('SELECT USER(), DATABASE()'); $user =~ s/@.*//; $dsn->{h} ||= $vars->{hostname}->{Value}; $dsn->{S} ||= $vars->{'socket'}->{Value}; $dsn->{P} ||= $vars->{port}->{Value}; $dsn->{u} ||= $user; $dsn->{D} ||= $db; } sub get_dbh { my ( $self, $cxn_string, $user, $pass, $opts ) = @_; $opts ||= {}; my $defaults = { AutoCommit => 0, RaiseError => 1, PrintError => 0, ShowErrorStatement => 1, mysql_enable_utf8 => ($cxn_string =~ m/charset=utf8/i ? 1 : 0), }; @{$defaults}{ keys %$opts } = values %$opts; if ( $opts->{mysql_use_result} ) { $defaults->{mysql_use_result} = 1; } if ( !$have_dbi ) { die "Cannot connect to MySQL because the Perl DBI module is not " . "installed or not found. Run 'perl -MDBI' to see the directories " . "that Perl searches for DBI. If DBI is not installed, try:\n" . " Debian/Ubuntu apt-get install libdbi-perl\n" . " RHEL/CentOS yum install perl-DBI\n" . " OpenSolaris pgk install pkg:/SUNWpmdbi\n"; } my $dbh; my $tries = 2; while ( !$dbh && $tries-- ) { MKDEBUG && _d($cxn_string, ' ', $user, ' ', $pass, join(', ', map { "$_=>$defaults->{$_}" } keys %$defaults )); eval { $dbh = DBI->connect($cxn_string, $user, $pass, $defaults); if ( $cxn_string =~ m/mysql/i ) { my $sql; $sql = 'SELECT @@SQL_MODE'; MKDEBUG && _d($dbh, $sql); my ($sql_mode) = $dbh->selectrow_array($sql); $sql = 'SET @@SQL_QUOTE_SHOW_CREATE = 1' . '/*!40101, @@SQL_MODE=\'NO_AUTO_VALUE_ON_ZERO' . ($sql_mode ? ",$sql_mode" : '') . '\'*/'; MKDEBUG && _d($dbh, $sql); $dbh->do($sql); if ( my ($charset) = $cxn_string =~ m/charset=(\w+)/ ) { $sql = "/*!40101 SET NAMES $charset*/"; MKDEBUG && _d($dbh, ':', $sql); $dbh->do($sql); MKDEBUG && _d('Enabling charset for STDOUT'); if ( $charset eq 'utf8' ) { binmode(STDOUT, ':utf8') or die "Can't binmode(STDOUT, ':utf8'): $OS_ERROR"; } else { binmode(STDOUT) or die "Can't binmode(STDOUT): $OS_ERROR"; } } if ( $self->prop('set-vars') ) { $sql = "SET " . $self->prop('set-vars'); MKDEBUG && _d($dbh, ':', $sql); $dbh->do($sql); } } }; if ( !$dbh && $EVAL_ERROR ) { MKDEBUG && _d($EVAL_ERROR); if ( $EVAL_ERROR =~ m/not a compiled character set|character set utf8/ ) { MKDEBUG && _d('Going to try again without utf8 support'); delete $defaults->{mysql_enable_utf8}; } elsif ( $EVAL_ERROR =~ m/locate DBD\/mysql/i ) { die "Cannot connect to MySQL because the Perl DBD::mysql module is " . "not installed or not found. Run 'perl -MDBD::mysql' to see " . "the directories that Perl searches for DBD::mysql. If " . "DBD::mysql is not installed, try:\n" . " Debian/Ubuntu apt-get install libdbd-mysql-perl\n" . " RHEL/CentOS yum install perl-DBD-MySQL\n" . " OpenSolaris pgk install pkg:/SUNWapu13dbd-mysql\n"; } if ( !$tries ) { die $EVAL_ERROR; } } } MKDEBUG && _d('DBH info: ', $dbh, Dumper($dbh->selectrow_hashref( 'SELECT DATABASE(), CONNECTION_ID(), VERSION()/*!50038 , @@hostname*/')), 'Connection info:', $dbh->{mysql_hostinfo}, 'Character set info:', Dumper($dbh->selectall_arrayref( 'SHOW VARIABLES LIKE "character_set%"', { Slice => {}})), '$DBD::mysql::VERSION:', $DBD::mysql::VERSION, '$DBI::VERSION:', $DBI::VERSION, ); return $dbh; } sub get_hostname { my ( $self, $dbh ) = @_; if ( my ($host) = ($dbh->{mysql_hostinfo} || '') =~ m/^(\w+) via/ ) { return $host; } my ( $hostname, $one ) = $dbh->selectrow_array( 'SELECT /*!50038 @@hostname, */ 1'); return $hostname; } sub disconnect { my ( $self, $dbh ) = @_; MKDEBUG && $self->print_active_handles($dbh); $dbh->disconnect; } sub print_active_handles { my ( $self, $thing, $level ) = @_; $level ||= 0; printf("# Active %sh: %s %s %s\n", ($thing->{Type} || 'undef'), "\t" x $level, $thing, (($thing->{Type} || '') eq 'st' ? $thing->{Statement} || '' : '')) or die "Cannot print: $OS_ERROR"; foreach my $handle ( grep {defined} @{ $thing->{ChildHandles} } ) { $self->print_active_handles( $handle, $level + 1 ); } } sub copy { my ( $self, $dsn_1, $dsn_2, %args ) = @_; die 'I need a dsn_1 argument' unless $dsn_1; die 'I need a dsn_2 argument' unless $dsn_2; my %new_dsn = map { my $key = $_; my $val; if ( $args{overwrite} ) { $val = defined $dsn_1->{$key} ? $dsn_1->{$key} : $dsn_2->{$key}; } else { $val = defined $dsn_2->{$key} ? $dsn_2->{$key} : $dsn_1->{$key}; } $key => $val; } keys %{$self->{opts}}; return \%new_dsn; } sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } 1; } # ########################################################################### # End DSNParser package # ########################################################################### # ########################################################################### # OptionParser package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/OptionParser.pm # t/lib/OptionParser.t # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { package OptionParser; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; use List::Util qw(max); use Getopt::Long; my $POD_link_re = '[LC]<"?([^">]+)"?>'; sub new { my ( $class, %args ) = @_; my @required_args = qw(); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my ($program_name) = $PROGRAM_NAME =~ m/([.A-Za-z-]+)$/; $program_name ||= $PROGRAM_NAME; my $home = $ENV{HOME} || $ENV{HOMEPATH} || $ENV{USERPROFILE} || '.'; my %attributes = ( 'type' => 1, 'short form' => 1, 'group' => 1, 'default' => 1, 'cumulative' => 1, 'negatable' => 1, ); my $self = { head1 => 'OPTIONS', # These args are used internally skip_rules => 0, # to instantiate another Option- item => '--(.*)', # Parser obj that parses the attributes => \%attributes, # DSN OPTIONS section. Tools parse_attributes => \&_parse_attribs, # don't tinker with these args. %args, strict => 1, # disabled by a special rule program_name => $program_name, opts => {}, got_opts => 0, short_opts => {}, defaults => {}, groups => {}, allowed_groups => {}, errors => [], rules => [], # desc of rules for --help mutex => [], # rule: opts are mutually exclusive atleast1 => [], # rule: at least one opt is required disables => {}, # rule: opt disables other opts defaults_to => {}, # rule: opt defaults to value of other opt DSNParser => undef, default_files => [ "/etc/percona-toolkit/percona-toolkit.conf", "/etc/percona-toolkit/$program_name.conf", "$home/.percona-toolkit.conf", "$home/.$program_name.conf", ], types => { string => 's', # standard Getopt type int => 'i', # standard Getopt type float => 'f', # standard Getopt type Hash => 'H', # hash, formed from a comma-separated list hash => 'h', # hash as above, but only if a value is given Array => 'A', # array, similar to Hash array => 'a', # array, similar to hash DSN => 'd', # DSN size => 'z', # size with kMG suffix (powers of 2^10) time => 'm', # time, with an optional suffix of s/h/m/d }, }; return bless $self, $class; } sub get_specs { my ( $self, $file ) = @_; $file ||= $self->{file} || __FILE__; my @specs = $self->_pod_to_specs($file); $self->_parse_specs(@specs); open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR"; my $contents = do { local $/ = undef; <$fh> }; close $fh; if ( $contents =~ m/^=head1 DSN OPTIONS/m ) { MKDEBUG && _d('Parsing DSN OPTIONS'); my $dsn_attribs = { dsn => 1, copy => 1, }; my $parse_dsn_attribs = sub { my ( $self, $option, $attribs ) = @_; map { my $val = $attribs->{$_}; if ( $val ) { $val = $val eq 'yes' ? 1 : $val eq 'no' ? 0 : $val; $attribs->{$_} = $val; } } keys %$attribs; return { key => $option, %$attribs, }; }; my $dsn_o = new OptionParser( description => 'DSN OPTIONS', head1 => 'DSN OPTIONS', dsn => 0, # XXX don't infinitely recurse! item => '\* (.)', # key opts are a single character skip_rules => 1, # no rules before opts attributes => $dsn_attribs, parse_attributes => $parse_dsn_attribs, ); my @dsn_opts = map { my $opts = { key => $_->{spec}->{key}, dsn => $_->{spec}->{dsn}, copy => $_->{spec}->{copy}, desc => $_->{desc}, }; $opts; } $dsn_o->_pod_to_specs($file); $self->{DSNParser} = DSNParser->new(opts => \@dsn_opts); } if ( $contents =~ m/^=head1 VERSION\n\n^(.+)$/m ) { $self->{version} = $1; MKDEBUG && _d($self->{version}); } return; } sub DSNParser { my ( $self ) = @_; return $self->{DSNParser}; }; sub get_defaults_files { my ( $self ) = @_; return @{$self->{default_files}}; } sub _pod_to_specs { my ( $self, $file ) = @_; $file ||= $self->{file} || __FILE__; open my $fh, '<', $file or die "Cannot open $file: $OS_ERROR"; my @specs = (); my @rules = (); my $para; local $INPUT_RECORD_SEPARATOR = ''; while ( $para = <$fh> ) { next unless $para =~ m/^=head1 $self->{head1}/; last; } while ( $para = <$fh> ) { last if $para =~ m/^=over/; next if $self->{skip_rules}; chomp $para; $para =~ s/\s+/ /g; $para =~ s/$POD_link_re/$1/go; MKDEBUG && _d('Option rule:', $para); push @rules, $para; } die "POD has no $self->{head1} section" unless $para; do { if ( my ($option) = $para =~ m/^=item $self->{item}/ ) { chomp $para; MKDEBUG && _d($para); my %attribs; $para = <$fh>; # read next paragraph, possibly attributes if ( $para =~ m/: / ) { # attributes $para =~ s/\s+\Z//g; %attribs = map { my ( $attrib, $val) = split(/: /, $_); die "Unrecognized attribute for --$option: $attrib" unless $self->{attributes}->{$attrib}; ($attrib, $val); } split(/; /, $para); if ( $attribs{'short form'} ) { $attribs{'short form'} =~ s/-//; } $para = <$fh>; # read next paragraph, probably short help desc } else { MKDEBUG && _d('Option has no attributes'); } $para =~ s/\s+\Z//g; $para =~ s/\s+/ /g; $para =~ s/$POD_link_re/$1/go; $para =~ s/\.(?:\n.*| [A-Z].*|\Z)//s; MKDEBUG && _d('Short help:', $para); die "No description after option spec $option" if $para =~ m/^=item/; if ( my ($base_option) = $option =~ m/^\[no\](.*)/ ) { $option = $base_option; $attribs{'negatable'} = 1; } push @specs, { spec => $self->{parse_attributes}->($self, $option, \%attribs), desc => $para . (defined $attribs{default} ? " (default $attribs{default})" : ''), group => ($attribs{'group'} ? $attribs{'group'} : 'default'), }; } while ( $para = <$fh> ) { last unless $para; if ( $para =~ m/^=head1/ ) { $para = undef; # Can't 'last' out of a do {} block. last; } last if $para =~ m/^=item /; } } while ( $para ); die "No valid specs in $self->{head1}" unless @specs; close $fh; return @specs, @rules; } sub _parse_specs { my ( $self, @specs ) = @_; my %disables; # special rule that requires deferred checking foreach my $opt ( @specs ) { if ( ref $opt ) { # It's an option spec, not a rule. MKDEBUG && _d('Parsing opt spec:', map { ($_, '=>', $opt->{$_}) } keys %$opt); my ( $long, $short ) = $opt->{spec} =~ m/^([\w-]+)(?:\|([^!+=]*))?/; if ( !$long ) { die "Cannot parse long option from spec $opt->{spec}"; } $opt->{long} = $long; die "Duplicate long option --$long" if exists $self->{opts}->{$long}; $self->{opts}->{$long} = $opt; if ( length $long == 1 ) { MKDEBUG && _d('Long opt', $long, 'looks like short opt'); $self->{short_opts}->{$long} = $long; } if ( $short ) { die "Duplicate short option -$short" if exists $self->{short_opts}->{$short}; $self->{short_opts}->{$short} = $long; $opt->{short} = $short; } else { $opt->{short} = undef; } $opt->{is_negatable} = $opt->{spec} =~ m/!/ ? 1 : 0; $opt->{is_cumulative} = $opt->{spec} =~ m/\+/ ? 1 : 0; $opt->{is_required} = $opt->{desc} =~ m/required/ ? 1 : 0; $opt->{group} ||= 'default'; $self->{groups}->{ $opt->{group} }->{$long} = 1; $opt->{value} = undef; $opt->{got} = 0; my ( $type ) = $opt->{spec} =~ m/=(.)/; $opt->{type} = $type; MKDEBUG && _d($long, 'type:', $type); $opt->{spec} =~ s/=./=s/ if ( $type && $type =~ m/[HhAadzm]/ ); if ( (my ($def) = $opt->{desc} =~ m/default\b(?: ([^)]+))?/) ) { $self->{defaults}->{$long} = defined $def ? $def : 1; MKDEBUG && _d($long, 'default:', $def); } if ( $long eq 'config' ) { $self->{defaults}->{$long} = join(',', $self->get_defaults_files()); } if ( (my ($dis) = $opt->{desc} =~ m/(disables .*)/) ) { $disables{$long} = $dis; MKDEBUG && _d('Deferring check of disables rule for', $opt, $dis); } $self->{opts}->{$long} = $opt; } else { # It's an option rule, not a spec. MKDEBUG && _d('Parsing rule:', $opt); push @{$self->{rules}}, $opt; my @participants = $self->_get_participants($opt); my $rule_ok = 0; if ( $opt =~ m/mutually exclusive|one and only one/ ) { $rule_ok = 1; push @{$self->{mutex}}, \@participants; MKDEBUG && _d(@participants, 'are mutually exclusive'); } if ( $opt =~ m/at least one|one and only one/ ) { $rule_ok = 1; push @{$self->{atleast1}}, \@participants; MKDEBUG && _d(@participants, 'require at least one'); } if ( $opt =~ m/default to/ ) { $rule_ok = 1; $self->{defaults_to}->{$participants[0]} = $participants[1]; MKDEBUG && _d($participants[0], 'defaults to', $participants[1]); } if ( $opt =~ m/restricted to option groups/ ) { $rule_ok = 1; my ($groups) = $opt =~ m/groups ([\w\s\,]+)/; my @groups = split(',', $groups); %{$self->{allowed_groups}->{$participants[0]}} = map { s/\s+//; $_ => 1; } @groups; } if( $opt =~ m/accepts additional command-line arguments/ ) { $rule_ok = 1; $self->{strict} = 0; MKDEBUG && _d("Strict mode disabled by rule"); } die "Unrecognized option rule: $opt" unless $rule_ok; } } foreach my $long ( keys %disables ) { my @participants = $self->_get_participants($disables{$long}); $self->{disables}->{$long} = \@participants; MKDEBUG && _d('Option', $long, 'disables', @participants); } return; } sub _get_participants { my ( $self, $str ) = @_; my @participants; foreach my $long ( $str =~ m/--(?:\[no\])?([\w-]+)/g ) { die "Option --$long does not exist while processing rule $str" unless exists $self->{opts}->{$long}; push @participants, $long; } MKDEBUG && _d('Participants for', $str, ':', @participants); return @participants; } sub opts { my ( $self ) = @_; my %opts = %{$self->{opts}}; return %opts; } sub short_opts { my ( $self ) = @_; my %short_opts = %{$self->{short_opts}}; return %short_opts; } sub set_defaults { my ( $self, %defaults ) = @_; $self->{defaults} = {}; foreach my $long ( keys %defaults ) { die "Cannot set default for nonexistent option $long" unless exists $self->{opts}->{$long}; $self->{defaults}->{$long} = $defaults{$long}; MKDEBUG && _d('Default val for', $long, ':', $defaults{$long}); } return; } sub get_defaults { my ( $self ) = @_; return $self->{defaults}; } sub get_groups { my ( $self ) = @_; return $self->{groups}; } sub _set_option { my ( $self, $opt, $val ) = @_; my $long = exists $self->{opts}->{$opt} ? $opt : exists $self->{short_opts}->{$opt} ? $self->{short_opts}->{$opt} : die "Getopt::Long gave a nonexistent option: $opt"; $opt = $self->{opts}->{$long}; if ( $opt->{is_cumulative} ) { $opt->{value}++; } else { $opt->{value} = $val; } $opt->{got} = 1; MKDEBUG && _d('Got option', $long, '=', $val); } sub get_opts { my ( $self ) = @_; foreach my $long ( keys %{$self->{opts}} ) { $self->{opts}->{$long}->{got} = 0; $self->{opts}->{$long}->{value} = exists $self->{defaults}->{$long} ? $self->{defaults}->{$long} : $self->{opts}->{$long}->{is_cumulative} ? 0 : undef; } $self->{got_opts} = 0; $self->{errors} = []; if ( @ARGV && $ARGV[0] eq "--config" ) { shift @ARGV; $self->_set_option('config', shift @ARGV); } if ( $self->has('config') ) { my @extra_args; foreach my $filename ( split(',', $self->get('config')) ) { eval { push @extra_args, $self->_read_config_file($filename); }; if ( $EVAL_ERROR ) { if ( $self->got('config') ) { die $EVAL_ERROR; } elsif ( MKDEBUG ) { _d($EVAL_ERROR); } } } unshift @ARGV, @extra_args; } Getopt::Long::Configure('no_ignore_case', 'bundling'); GetOptions( map { $_->{spec} => sub { $self->_set_option(@_); } } grep { $_->{long} ne 'config' } # --config is handled specially above. values %{$self->{opts}} ) or $self->save_error('Error parsing options'); if ( exists $self->{opts}->{version} && $self->{opts}->{version}->{got} ) { if ( $self->{version} ) { print $self->{version}, "\n"; } else { print "Error parsing version. See the VERSION section of the tool's documentation.\n"; } exit 0; } if ( @ARGV && $self->{strict} ) { $self->save_error("Unrecognized command-line options @ARGV"); } foreach my $mutex ( @{$self->{mutex}} ) { my @set = grep { $self->{opts}->{$_}->{got} } @$mutex; if ( @set > 1 ) { my $err = join(', ', map { "--$self->{opts}->{$_}->{long}" } @{$mutex}[ 0 .. scalar(@$mutex) - 2] ) . ' and --'.$self->{opts}->{$mutex->[-1]}->{long} . ' are mutually exclusive.'; $self->save_error($err); } } foreach my $required ( @{$self->{atleast1}} ) { my @set = grep { $self->{opts}->{$_}->{got} } @$required; if ( @set == 0 ) { my $err = join(', ', map { "--$self->{opts}->{$_}->{long}" } @{$required}[ 0 .. scalar(@$required) - 2] ) .' or --'.$self->{opts}->{$required->[-1]}->{long}; $self->save_error("Specify at least one of $err"); } } $self->_check_opts( keys %{$self->{opts}} ); $self->{got_opts} = 1; return; } sub _check_opts { my ( $self, @long ) = @_; my $long_last = scalar @long; while ( @long ) { foreach my $i ( 0..$#long ) { my $long = $long[$i]; next unless $long; my $opt = $self->{opts}->{$long}; if ( $opt->{got} ) { if ( exists $self->{disables}->{$long} ) { my @disable_opts = @{$self->{disables}->{$long}}; map { $self->{opts}->{$_}->{value} = undef; } @disable_opts; MKDEBUG && _d('Unset options', @disable_opts, 'because', $long,'disables them'); } if ( exists $self->{allowed_groups}->{$long} ) { my @restricted_groups = grep { !exists $self->{allowed_groups}->{$long}->{$_} } keys %{$self->{groups}}; my @restricted_opts; foreach my $restricted_group ( @restricted_groups ) { RESTRICTED_OPT: foreach my $restricted_opt ( keys %{$self->{groups}->{$restricted_group}} ) { next RESTRICTED_OPT if $restricted_opt eq $long; push @restricted_opts, $restricted_opt if $self->{opts}->{$restricted_opt}->{got}; } } if ( @restricted_opts ) { my $err; if ( @restricted_opts == 1 ) { $err = "--$restricted_opts[0]"; } else { $err = join(', ', map { "--$self->{opts}->{$_}->{long}" } grep { $_ } @restricted_opts[0..scalar(@restricted_opts) - 2] ) . ' or --'.$self->{opts}->{$restricted_opts[-1]}->{long}; } $self->save_error("--$long is not allowed with $err"); } } } elsif ( $opt->{is_required} ) { $self->save_error("Required option --$long must be specified"); } $self->_validate_type($opt); if ( $opt->{parsed} ) { delete $long[$i]; } else { MKDEBUG && _d('Temporarily failed to parse', $long); } } die "Failed to parse options, possibly due to circular dependencies" if @long == $long_last; $long_last = @long; } return; } sub _validate_type { my ( $self, $opt ) = @_; return unless $opt; if ( !$opt->{type} ) { $opt->{parsed} = 1; return; } my $val = $opt->{value}; if ( $val && $opt->{type} eq 'm' ) { # type time MKDEBUG && _d('Parsing option', $opt->{long}, 'as a time value'); my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/; if ( !$suffix ) { my ( $s ) = $opt->{desc} =~ m/\(suffix (.)\)/; $suffix = $s || 's'; MKDEBUG && _d('No suffix given; using', $suffix, 'for', $opt->{long}, '(value:', $val, ')'); } if ( $suffix =~ m/[smhd]/ ) { $val = $suffix eq 's' ? $num # Seconds : $suffix eq 'm' ? $num * 60 # Minutes : $suffix eq 'h' ? $num * 3600 # Hours : $num * 86400; # Days $opt->{value} = ($prefix || '') . $val; MKDEBUG && _d('Setting option', $opt->{long}, 'to', $val); } else { $self->save_error("Invalid time suffix for --$opt->{long}"); } } elsif ( $val && $opt->{type} eq 'd' ) { # type DSN MKDEBUG && _d('Parsing option', $opt->{long}, 'as a DSN'); my $prev = {}; my $from_key = $self->{defaults_to}->{ $opt->{long} }; if ( $from_key ) { MKDEBUG && _d($opt->{long}, 'DSN copies from', $from_key, 'DSN'); if ( $self->{opts}->{$from_key}->{parsed} ) { $prev = $self->{opts}->{$from_key}->{value}; } else { MKDEBUG && _d('Cannot parse', $opt->{long}, 'until', $from_key, 'parsed'); return; } } my $defaults = $self->{DSNParser}->parse_options($self); $opt->{value} = $self->{DSNParser}->parse($val, $prev, $defaults); } elsif ( $val && $opt->{type} eq 'z' ) { # type size MKDEBUG && _d('Parsing option', $opt->{long}, 'as a size value'); $self->_parse_size($opt, $val); } elsif ( $opt->{type} eq 'H' || (defined $val && $opt->{type} eq 'h') ) { $opt->{value} = { map { $_ => 1 } split(/(?{type} eq 'A' || (defined $val && $opt->{type} eq 'a') ) { $opt->{value} = [ split(/(?{long}, 'type', $opt->{type}, 'value', $val); } $opt->{parsed} = 1; return; } sub get { my ( $self, $opt ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); die "Option $opt does not exist" unless $long && exists $self->{opts}->{$long}; return $self->{opts}->{$long}->{value}; } sub got { my ( $self, $opt ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); die "Option $opt does not exist" unless $long && exists $self->{opts}->{$long}; return $self->{opts}->{$long}->{got}; } sub has { my ( $self, $opt ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); return defined $long ? exists $self->{opts}->{$long} : 0; } sub set { my ( $self, $opt, $val ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); die "Option $opt does not exist" unless $long && exists $self->{opts}->{$long}; $self->{opts}->{$long}->{value} = $val; return; } sub save_error { my ( $self, $error ) = @_; push @{$self->{errors}}, $error; return; } sub errors { my ( $self ) = @_; return $self->{errors}; } sub usage { my ( $self ) = @_; warn "No usage string is set" unless $self->{usage}; # XXX return "Usage: " . ($self->{usage} || '') . "\n"; } sub descr { my ( $self ) = @_; warn "No description string is set" unless $self->{description}; # XXX my $descr = ($self->{description} || $self->{program_name} || '') . " For more details, please use the --help option, " . "or try 'perldoc $PROGRAM_NAME' " . "for complete documentation."; $descr = join("\n", $descr =~ m/(.{0,80})(?:\s+|$)/g) unless $ENV{DONT_BREAK_LINES}; $descr =~ s/ +$//mg; return $descr; } sub usage_or_errors { my ( $self, $file, $return ) = @_; $file ||= $self->{file} || __FILE__; if ( !$self->{description} || !$self->{usage} ) { MKDEBUG && _d("Getting description and usage from SYNOPSIS in", $file); my %synop = $self->_parse_synopsis($file); $self->{description} ||= $synop{description}; $self->{usage} ||= $synop{usage}; MKDEBUG && _d("Description:", $self->{description}, "\nUsage:", $self->{usage}); } if ( $self->{opts}->{help}->{got} ) { print $self->print_usage() or die "Cannot print usage: $OS_ERROR"; exit 0 unless $return; } elsif ( scalar @{$self->{errors}} ) { print $self->print_errors() or die "Cannot print errors: $OS_ERROR"; exit 0 unless $return; } return; } sub print_errors { my ( $self ) = @_; my $usage = $self->usage() . "\n"; if ( (my @errors = @{$self->{errors}}) ) { $usage .= join("\n * ", 'Errors in command-line arguments:', @errors) . "\n"; } return $usage . "\n" . $self->descr(); } sub print_usage { my ( $self ) = @_; die "Run get_opts() before print_usage()" unless $self->{got_opts}; my @opts = values %{$self->{opts}}; my $maxl = max( map { length($_->{long}) # option long name + ($_->{is_negatable} ? 4 : 0) # "[no]" if opt is negatable + ($_->{type} ? 2 : 0) # "=x" where x is the opt type } @opts); my $maxs = max(0, map { length($_) + ($self->{opts}->{$_}->{is_negatable} ? 4 : 0) + ($self->{opts}->{$_}->{type} ? 2 : 0) } values %{$self->{short_opts}}); my $lcol = max($maxl, ($maxs + 3)); my $rcol = 80 - $lcol - 6; my $rpad = ' ' x ( 80 - $rcol ); $maxs = max($lcol - 3, $maxs); my $usage = $self->descr() . "\n" . $self->usage(); my @groups = reverse sort grep { $_ ne 'default'; } keys %{$self->{groups}}; push @groups, 'default'; foreach my $group ( reverse @groups ) { $usage .= "\n".($group eq 'default' ? 'Options' : $group).":\n\n"; foreach my $opt ( sort { $a->{long} cmp $b->{long} } grep { $_->{group} eq $group } @opts ) { my $long = $opt->{is_negatable} ? "[no]$opt->{long}" : $opt->{long}; my $short = $opt->{short}; my $desc = $opt->{desc}; $long .= $opt->{type} ? "=$opt->{type}" : ""; if ( $opt->{type} && $opt->{type} eq 'm' ) { my ($s) = $desc =~ m/\(suffix (.)\)/; $s ||= 's'; $desc =~ s/\s+\(suffix .\)//; $desc .= ". Optional suffix s=seconds, m=minutes, h=hours, " . "d=days; if no suffix, $s is used."; } $desc = join("\n$rpad", grep { $_ } $desc =~ m/(.{0,$rcol})(?:\s+|$)/g); $desc =~ s/ +$//mg; if ( $short ) { $usage .= sprintf(" --%-${maxs}s -%s %s\n", $long, $short, $desc); } else { $usage .= sprintf(" --%-${lcol}s %s\n", $long, $desc); } } } $usage .= "\nOption types: s=string, i=integer, f=float, h/H/a/A=comma-separated list, d=DSN, z=size, m=time\n"; if ( (my @rules = @{$self->{rules}}) ) { $usage .= "\nRules:\n\n"; $usage .= join("\n", map { " $_" } @rules) . "\n"; } if ( $self->{DSNParser} ) { $usage .= "\n" . $self->{DSNParser}->usage(); } $usage .= "\nOptions and values after processing arguments:\n\n"; foreach my $opt ( sort { $a->{long} cmp $b->{long} } @opts ) { my $val = $opt->{value}; my $type = $opt->{type} || ''; my $bool = $opt->{spec} =~ m/^[\w-]+(?:\|[\w-])?!?$/; $val = $bool ? ( $val ? 'TRUE' : 'FALSE' ) : !defined $val ? '(No value)' : $type eq 'd' ? $self->{DSNParser}->as_string($val) : $type =~ m/H|h/ ? join(',', sort keys %$val) : $type =~ m/A|a/ ? join(',', @$val) : $val; $usage .= sprintf(" --%-${lcol}s %s\n", $opt->{long}, $val); } return $usage; } sub prompt_noecho { shift @_ if ref $_[0] eq __PACKAGE__; my ( $prompt ) = @_; local $OUTPUT_AUTOFLUSH = 1; print $prompt or die "Cannot print: $OS_ERROR"; my $response; eval { require Term::ReadKey; Term::ReadKey::ReadMode('noecho'); chomp($response = ); Term::ReadKey::ReadMode('normal'); print "\n" or die "Cannot print: $OS_ERROR"; }; if ( $EVAL_ERROR ) { die "Cannot read response; is Term::ReadKey installed? $EVAL_ERROR"; } return $response; } sub _read_config_file { my ( $self, $filename ) = @_; open my $fh, "<", $filename or die "Cannot open $filename: $OS_ERROR\n"; my @args; my $prefix = '--'; my $parse = 1; LINE: while ( my $line = <$fh> ) { chomp $line; next LINE if $line =~ m/^\s*(?:\#|\;|$)/; $line =~ s/\s+#.*$//g; $line =~ s/^\s+|\s+$//g; if ( $line eq '--' ) { $prefix = ''; $parse = 0; next LINE; } if ( $parse && (my($opt, $arg) = $line =~ m/^\s*([^=\s]+?)(?:\s*=\s*(.*?)\s*)?$/) ) { push @args, grep { defined $_ } ("$prefix$opt", $arg); } elsif ( $line =~ m/./ ) { push @args, $line; } else { die "Syntax error in file $filename at line $INPUT_LINE_NUMBER"; } } close $fh; return @args; } sub read_para_after { my ( $self, $file, $regex ) = @_; open my $fh, "<", $file or die "Can't open $file: $OS_ERROR"; local $INPUT_RECORD_SEPARATOR = ''; my $para; while ( $para = <$fh> ) { next unless $para =~ m/^=pod$/m; last; } while ( $para = <$fh> ) { next unless $para =~ m/$regex/; last; } $para = <$fh>; chomp($para); close $fh or die "Can't close $file: $OS_ERROR"; return $para; } sub clone { my ( $self ) = @_; my %clone = map { my $hashref = $self->{$_}; my $val_copy = {}; foreach my $key ( keys %$hashref ) { my $ref = ref $hashref->{$key}; $val_copy->{$key} = !$ref ? $hashref->{$key} : $ref eq 'HASH' ? { %{$hashref->{$key}} } : $ref eq 'ARRAY' ? [ @{$hashref->{$key}} ] : $hashref->{$key}; } $_ => $val_copy; } qw(opts short_opts defaults); foreach my $scalar ( qw(got_opts) ) { $clone{$scalar} = $self->{$scalar}; } return bless \%clone; } sub _parse_size { my ( $self, $opt, $val ) = @_; if ( lc($val || '') eq 'null' ) { MKDEBUG && _d('NULL size for', $opt->{long}); $opt->{value} = 'null'; return; } my %factor_for = (k => 1_024, M => 1_048_576, G => 1_073_741_824); my ($pre, $num, $factor) = $val =~ m/^([+-])?(\d+)([kMG])?$/; if ( defined $num ) { if ( $factor ) { $num *= $factor_for{$factor}; MKDEBUG && _d('Setting option', $opt->{y}, 'to num', $num, '* factor', $factor); } $opt->{value} = ($pre || '') . $num; } else { $self->save_error("Invalid size for --$opt->{long}"); } return; } sub _parse_attribs { my ( $self, $option, $attribs ) = @_; my $types = $self->{types}; return $option . ($attribs->{'short form'} ? '|' . $attribs->{'short form'} : '' ) . ($attribs->{'negatable'} ? '!' : '' ) . ($attribs->{'cumulative'} ? '+' : '' ) . ($attribs->{'type'} ? '=' . $types->{$attribs->{type}} : '' ); } sub _parse_synopsis { my ( $self, $file ) = @_; $file ||= $self->{file} || __FILE__; MKDEBUG && _d("Parsing SYNOPSIS in", $file); local $INPUT_RECORD_SEPARATOR = ''; # read paragraphs open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR"; my $para; 1 while defined($para = <$fh>) && $para !~ m/^=head1 SYNOPSIS/; die "$file does not contain a SYNOPSIS section" unless $para; my @synop; for ( 1..2 ) { # 1 for the usage, 2 for the description my $para = <$fh>; push @synop, $para; } close $fh; MKDEBUG && _d("Raw SYNOPSIS text:", @synop); my ($usage, $desc) = @synop; die "The SYNOPSIS section in $file is not formatted properly" unless $usage && $desc; $usage =~ s/^\s*Usage:\s+(.+)/$1/; chomp $usage; $desc =~ s/\n/ /g; $desc =~ s/\s{2,}/ /g; $desc =~ s/\. ([A-Z][a-z])/. $1/g; $desc =~ s/\s+$//; return ( description => $desc, usage => $usage, ); }; sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } if ( MKDEBUG ) { print '# ', $^X, ' ', $], "\n"; if ( my $uname = `uname -a` ) { $uname =~ s/\s+/ /g; print "# $uname\n"; } print '# Arguments: ', join(' ', map { my $a = "_[$_]_"; $a =~ s/\n/\n# /g; $a; } @ARGV), "\n"; } 1; } # ########################################################################### # End OptionParser package # ########################################################################### # ########################################################################### # Quoter package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/Quoter.pm # t/lib/Quoter.t # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { package Quoter; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; sub new { my ( $class, %args ) = @_; return bless {}, $class; } sub quote { my ( $self, @vals ) = @_; foreach my $val ( @vals ) { $val =~ s/`/``/g; } return join('.', map { '`' . $_ . '`' } @vals); } sub quote_val { my ( $self, $val ) = @_; return 'NULL' unless defined $val; # undef = NULL return "''" if $val eq ''; # blank string = '' return $val if $val =~ m/^0x[0-9a-fA-F]+$/; # hex data $val =~ s/(['\\])/\\$1/g; return "'$val'"; } sub split_unquote { my ( $self, $db_tbl, $default_db ) = @_; $db_tbl =~ s/`//g; my ( $db, $tbl ) = split(/[.]/, $db_tbl); if ( !$tbl ) { $tbl = $db; $db = $default_db; } return ($db, $tbl); } sub literal_like { my ( $self, $like ) = @_; return unless $like; $like =~ s/([%_])/\\$1/g; return "'$like'"; } sub join_quote { my ( $self, $default_db, $db_tbl ) = @_; return unless $db_tbl; my ($db, $tbl) = split(/[.]/, $db_tbl); if ( !$tbl ) { $tbl = $db; $db = $default_db; } $db = "`$db`" if $db && $db !~ m/^`/; $tbl = "`$tbl`" if $tbl && $tbl !~ m/^`/; return $db ? "$db.$tbl" : $tbl; } 1; } # ########################################################################### # End Quoter package # ########################################################################### # ########################################################################### # VersionParser package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/VersionParser.pm # t/lib/VersionParser.t # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { package VersionParser; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; sub new { my ( $class ) = @_; bless {}, $class; } sub parse { my ( $self, $str ) = @_; my $result = sprintf('%03d%03d%03d', $str =~ m/(\d+)/g); MKDEBUG && _d($str, 'parses to', $result); return $result; } sub version_ge { my ( $self, $dbh, $target ) = @_; if ( !$self->{$dbh} ) { $self->{$dbh} = $self->parse( $dbh->selectrow_array('SELECT VERSION()')); } my $result = $self->{$dbh} ge $self->parse($target) ? 1 : 0; MKDEBUG && _d($self->{$dbh}, 'ge', $target, ':', $result); return $result; } sub innodb_version { my ( $self, $dbh ) = @_; return unless $dbh; my $innodb_version = "NO"; my ($innodb) = grep { $_->{engine} =~ m/InnoDB/i } map { my %hash; @hash{ map { lc $_ } keys %$_ } = values %$_; \%hash; } @{ $dbh->selectall_arrayref("SHOW ENGINES", {Slice=>{}}) }; if ( $innodb ) { MKDEBUG && _d("InnoDB support:", $innodb->{support}); if ( $innodb->{support} =~ m/YES|DEFAULT/i ) { my $vars = $dbh->selectrow_hashref( "SHOW VARIABLES LIKE 'innodb_version'"); $innodb_version = !$vars ? "BUILTIN" : ($vars->{Value} || $vars->{value}); } else { $innodb_version = $innodb->{support}; # probably DISABLED or NO } } MKDEBUG && _d("InnoDB version:", $innodb_version); return $innodb_version; } sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } 1; } # ########################################################################### # End VersionParser package # ########################################################################### # ########################################################################### # TableParser package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/TableParser.pm # t/lib/TableParser.t # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { package TableParser; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; use Data::Dumper; $Data::Dumper::Indent = 1; $Data::Dumper::Sortkeys = 1; $Data::Dumper::Quotekeys = 0; sub new { my ( $class, %args ) = @_; my @required_args = qw(Quoter); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my $self = { %args }; return bless $self, $class; } sub parse { my ( $self, $ddl, $opts ) = @_; return unless $ddl; if ( ref $ddl eq 'ARRAY' ) { if ( lc $ddl->[0] eq 'table' ) { $ddl = $ddl->[1]; } else { return { engine => 'VIEW', }; } } if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) { die "Cannot parse table definition; is ANSI quoting " . "enabled or SQL_QUOTE_SHOW_CREATE disabled?"; } my ($name) = $ddl =~ m/CREATE (?:TEMPORARY )?TABLE\s+(`.+?`)/; (undef, $name) = $self->{Quoter}->split_unquote($name) if $name; $ddl =~ s/(`[^`]+`)/\L$1/g; my $engine = $self->get_engine($ddl); my @defs = $ddl =~ m/^(\s+`.*?),?$/gm; my @cols = map { $_ =~ m/`([^`]+)`/ } @defs; MKDEBUG && _d('Table cols:', join(', ', map { "`$_`" } @cols)); my %def_for; @def_for{@cols} = @defs; my (@nums, @null); my (%type_for, %is_nullable, %is_numeric, %is_autoinc); foreach my $col ( @cols ) { my $def = $def_for{$col}; my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/; die "Can't determine column type for $def" unless $type; $type_for{$col} = $type; if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) { push @nums, $col; $is_numeric{$col} = 1; } if ( $def !~ m/NOT NULL/ ) { push @null, $col; $is_nullable{$col} = 1; } $is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0; } my ($keys, $clustered_key) = $self->get_keys($ddl, $opts, \%is_nullable); my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/; return { name => $name, cols => \@cols, col_posn => { map { $cols[$_] => $_ } 0..$#cols }, is_col => { map { $_ => 1 } @cols }, null_cols => \@null, is_nullable => \%is_nullable, is_autoinc => \%is_autoinc, clustered_key => $clustered_key, keys => $keys, defs => \%def_for, numeric_cols => \@nums, is_numeric => \%is_numeric, engine => $engine, type_for => \%type_for, charset => $charset, }; } sub sort_indexes { my ( $self, $tbl ) = @_; my @indexes = sort { (($a ne 'PRIMARY') <=> ($b ne 'PRIMARY')) || ( !$tbl->{keys}->{$a}->{is_unique} <=> !$tbl->{keys}->{$b}->{is_unique} ) || ( $tbl->{keys}->{$a}->{is_nullable} <=> $tbl->{keys}->{$b}->{is_nullable} ) || ( scalar(@{$tbl->{keys}->{$a}->{cols}}) <=> scalar(@{$tbl->{keys}->{$b}->{cols}}) ) } grep { $tbl->{keys}->{$_}->{type} eq 'BTREE' } sort keys %{$tbl->{keys}}; MKDEBUG && _d('Indexes sorted best-first:', join(', ', @indexes)); return @indexes; } sub find_best_index { my ( $self, $tbl, $index ) = @_; my $best; if ( $index ) { ($best) = grep { uc $_ eq uc $index } keys %{$tbl->{keys}}; } if ( !$best ) { if ( $index ) { die "Index '$index' does not exist in table"; } else { ($best) = $self->sort_indexes($tbl); } } MKDEBUG && _d('Best index found is', $best); return $best; } sub find_possible_keys { my ( $self, $dbh, $database, $table, $quoter, $where ) = @_; return () unless $where; my $sql = 'EXPLAIN SELECT * FROM ' . $quoter->quote($database, $table) . ' WHERE ' . $where; MKDEBUG && _d($sql); my $expl = $dbh->selectrow_hashref($sql); $expl = { map { lc($_) => $expl->{$_} } keys %$expl }; if ( $expl->{possible_keys} ) { MKDEBUG && _d('possible_keys =', $expl->{possible_keys}); my @candidates = split(',', $expl->{possible_keys}); my %possible = map { $_ => 1 } @candidates; if ( $expl->{key} ) { MKDEBUG && _d('MySQL chose', $expl->{key}); unshift @candidates, grep { $possible{$_} } split(',', $expl->{key}); MKDEBUG && _d('Before deduping:', join(', ', @candidates)); my %seen; @candidates = grep { !$seen{$_}++ } @candidates; } MKDEBUG && _d('Final list:', join(', ', @candidates)); return @candidates; } else { MKDEBUG && _d('No keys in possible_keys'); return (); } } sub check_table { my ( $self, %args ) = @_; my @required_args = qw(dbh db tbl); foreach my $arg ( @required_args ) { die "I need a $arg argument" unless $args{$arg}; } my ($dbh, $db, $tbl) = @args{@required_args}; my $q = $self->{Quoter}; my $db_tbl = $q->quote($db, $tbl); MKDEBUG && _d('Checking', $db_tbl); my $sql = "SHOW TABLES FROM " . $q->quote($db) . ' LIKE ' . $q->literal_like($tbl); MKDEBUG && _d($sql); my $row; eval { $row = $dbh->selectrow_arrayref($sql); }; if ( $EVAL_ERROR ) { MKDEBUG && _d($EVAL_ERROR); return 0; } if ( !$row->[0] || $row->[0] ne $tbl ) { MKDEBUG && _d('Table does not exist'); return 0; } MKDEBUG && _d('Table exists; no privs to check'); return 1 unless $args{all_privs}; $sql = "SHOW FULL COLUMNS FROM $db_tbl"; MKDEBUG && _d($sql); eval { $row = $dbh->selectrow_hashref($sql); }; if ( $EVAL_ERROR ) { MKDEBUG && _d($EVAL_ERROR); return 0; } if ( !scalar keys %$row ) { MKDEBUG && _d('Table has no columns:', Dumper($row)); return 0; } my $privs = $row->{privileges} || $row->{Privileges}; $sql = "DELETE FROM $db_tbl LIMIT 0"; MKDEBUG && _d($sql); eval { $dbh->do($sql); }; my $can_delete = $EVAL_ERROR ? 0 : 1; MKDEBUG && _d('User privs on', $db_tbl, ':', $privs, ($can_delete ? 'delete' : '')); if ( !($privs =~ m/select/ && $privs =~ m/insert/ && $privs =~ m/update/ && $can_delete) ) { MKDEBUG && _d('User does not have all privs'); return 0; } MKDEBUG && _d('User has all privs'); return 1; } sub get_engine { my ( $self, $ddl, $opts ) = @_; my ( $engine ) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/; MKDEBUG && _d('Storage engine:', $engine); return $engine || undef; } sub get_keys { my ( $self, $ddl, $opts, $is_nullable ) = @_; my $engine = $self->get_engine($ddl); my $keys = {}; my $clustered_key = undef; KEY: foreach my $key ( $ddl =~ m/^ ((?:[A-Z]+ )?KEY .*)$/gm ) { next KEY if $key =~ m/FOREIGN/; my $key_ddl = $key; MKDEBUG && _d('Parsed key:', $key_ddl); if ( $engine !~ m/MEMORY|HEAP/ ) { $key =~ s/USING HASH/USING BTREE/; } my ( $type, $cols ) = $key =~ m/(?:USING (\w+))? \((.+)\)/; my ( $special ) = $key =~ m/(FULLTEXT|SPATIAL)/; $type = $type || $special || 'BTREE'; if ( $opts->{mysql_version} && $opts->{mysql_version} lt '004001000' && $engine =~ m/HEAP|MEMORY/i ) { $type = 'HASH'; # MySQL pre-4.1 supports only HASH indexes on HEAP } my ($name) = $key =~ m/(PRIMARY|`[^`]*`)/; my $unique = $key =~ m/PRIMARY|UNIQUE/ ? 1 : 0; my @cols; my @col_prefixes; foreach my $col_def ( $cols =~ m/`[^`]+`(?:\(\d+\))?/g ) { my ($name, $prefix) = $col_def =~ m/`([^`]+)`(?:\((\d+)\))?/; push @cols, $name; push @col_prefixes, $prefix; } $name =~ s/`//g; MKDEBUG && _d( $name, 'key cols:', join(', ', map { "`$_`" } @cols)); $keys->{$name} = { name => $name, type => $type, colnames => $cols, cols => \@cols, col_prefixes => \@col_prefixes, is_unique => $unique, is_nullable => scalar(grep { $is_nullable->{$_} } @cols), is_col => { map { $_ => 1 } @cols }, ddl => $key_ddl, }; if ( $engine =~ m/InnoDB/i && !$clustered_key ) { my $this_key = $keys->{$name}; if ( $this_key->{name} eq 'PRIMARY' ) { $clustered_key = 'PRIMARY'; } elsif ( $this_key->{is_unique} && !$this_key->{is_nullable} ) { $clustered_key = $this_key->{name}; } MKDEBUG && $clustered_key && _d('This key is the clustered key'); } } return $keys, $clustered_key; } sub get_fks { my ( $self, $ddl, $opts ) = @_; my $q = $self->{Quoter}; my $fks = {}; foreach my $fk ( $ddl =~ m/CONSTRAINT .* FOREIGN KEY .* REFERENCES [^\)]*\)/mg ) { my ( $name ) = $fk =~ m/CONSTRAINT `(.*?)`/; my ( $cols ) = $fk =~ m/FOREIGN KEY \(([^\)]+)\)/; my ( $parent, $parent_cols ) = $fk =~ m/REFERENCES (\S+) \(([^\)]+)\)/; my ($db, $tbl) = $q->split_unquote($parent, $opts->{database}); my %parent_tbl = (tbl => $tbl); $parent_tbl{db} = $db if $db; if ( $parent !~ m/\./ && $opts->{database} ) { $parent = $q->quote($opts->{database}) . ".$parent"; } $fks->{$name} = { name => $name, colnames => $cols, cols => [ map { s/[ `]+//g; $_; } split(',', $cols) ], parent_tbl => \%parent_tbl, parent_tblname => $parent, parent_cols => [ map { s/[ `]+//g; $_; } split(',', $parent_cols) ], parent_colnames=> $parent_cols, ddl => $fk, }; } return $fks; } sub remove_auto_increment { my ( $self, $ddl ) = @_; $ddl =~ s/(^\).*?) AUTO_INCREMENT=\d+\b/$1/m; return $ddl; } sub remove_secondary_indexes { my ( $self, $ddl ) = @_; my $sec_indexes_ddl; my $tbl_struct = $self->parse($ddl); if ( ($tbl_struct->{engine} || '') =~ m/InnoDB/i ) { my $clustered_key = $tbl_struct->{clustered_key}; $clustered_key ||= ''; my @sec_indexes = map { my $key_def = $_->{ddl}; $key_def =~ s/([\(\)])/\\$1/g; $ddl =~ s/\s+$key_def//i; my $key_ddl = "ADD $_->{ddl}"; $key_ddl .= ',' unless $key_ddl =~ m/,$/; $key_ddl; } grep { $_->{name} ne $clustered_key } values %{$tbl_struct->{keys}}; MKDEBUG && _d('Secondary indexes:', Dumper(\@sec_indexes)); if ( @sec_indexes ) { $sec_indexes_ddl = join(' ', @sec_indexes); $sec_indexes_ddl =~ s/,$//; } $ddl =~ s/,(\n\) )/$1/s; } else { MKDEBUG && _d('Not removing secondary indexes from', $tbl_struct->{engine}, 'table'); } return $ddl, $sec_indexes_ddl, $tbl_struct; } sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } 1; } # ########################################################################### # End TableParser package # ########################################################################### # ########################################################################### # MySQLDump package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/MySQLDump.pm # t/lib/MySQLDump.t # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { package MySQLDump; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; ( our $before = <<'EOF') =~ s/^ //gm; /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; /*!40101 SET NAMES utf8 */; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; EOF ( our $after = <<'EOF') =~ s/^ //gm; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; EOF sub new { my ( $class, %args ) = @_; my $self = { cache => 0, # Afaik no script uses this cache any longer because }; return bless $self, $class; } sub dump { my ( $self, $dbh, $quoter, $db, $tbl, $what ) = @_; if ( $what eq 'table' ) { my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl); return unless $ddl; if ( $ddl->[0] eq 'table' ) { return $before . 'DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . ";\n" . $ddl->[1] . ";\n"; } else { return 'DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . ";\n" . '/*!50001 DROP VIEW IF EXISTS ' . $quoter->quote($tbl) . "*/;\n/*!50001 " . $self->get_tmp_table($dbh, $quoter, $db, $tbl) . "*/;\n"; } } elsif ( $what eq 'triggers' ) { my $trgs = $self->get_triggers($dbh, $quoter, $db, $tbl); if ( $trgs && @$trgs ) { my $result = $before . "\nDELIMITER ;;\n"; foreach my $trg ( @$trgs ) { if ( $trg->{sql_mode} ) { $result .= qq{/*!50003 SET SESSION SQL_MODE='$trg->{sql_mode}' */;;\n}; } $result .= "/*!50003 CREATE */ "; if ( $trg->{definer} ) { my ( $user, $host ) = map { s/'/''/g; "'$_'"; } split('@', $trg->{definer}, 2); $result .= "/*!50017 DEFINER=$user\@$host */ "; } $result .= sprintf("/*!50003 TRIGGER %s %s %s ON %s\nFOR EACH ROW %s */;;\n\n", $quoter->quote($trg->{trigger}), @{$trg}{qw(timing event)}, $quoter->quote($trg->{table}), $trg->{statement}); } $result .= "DELIMITER ;\n\n/*!50003 SET SESSION SQL_MODE=\@OLD_SQL_MODE */;\n\n"; return $result; } else { return undef; } } elsif ( $what eq 'view' ) { my $ddl = $self->get_create_table($dbh, $quoter, $db, $tbl); return '/*!50001 DROP TABLE IF EXISTS ' . $quoter->quote($tbl) . "*/;\n" . '/*!50001 DROP VIEW IF EXISTS ' . $quoter->quote($tbl) . "*/;\n" . '/*!50001 ' . $ddl->[1] . "*/;\n"; } else { die "You didn't say what to dump."; } } sub _use_db { my ( $self, $dbh, $quoter, $new ) = @_; if ( !$new ) { MKDEBUG && _d('No new DB to use'); return; } my $sql = 'USE ' . $quoter->quote($new); MKDEBUG && _d($dbh, $sql); $dbh->do($sql); return; } sub get_create_table { my ( $self, $dbh, $quoter, $db, $tbl ) = @_; if ( !$self->{cache} || !$self->{tables}->{$db}->{$tbl} ) { my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, ' . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), } . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, ' . '@@SQL_QUOTE_SHOW_CREATE := 1 */'; MKDEBUG && _d($sql); eval { $dbh->do($sql); }; MKDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); $self->_use_db($dbh, $quoter, $db); $sql = "SHOW CREATE TABLE " . $quoter->quote($db, $tbl); MKDEBUG && _d($sql); my $href; eval { $href = $dbh->selectrow_hashref($sql); }; if ( $EVAL_ERROR ) { warn "Failed to $sql. The table may be damaged.\nError: $EVAL_ERROR"; return; } $sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, ' . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */'; MKDEBUG && _d($sql); $dbh->do($sql); my ($key) = grep { m/create table/i } keys %$href; if ( $key ) { MKDEBUG && _d('This table is a base table'); $self->{tables}->{$db}->{$tbl} = [ 'table', $href->{$key} ]; } else { MKDEBUG && _d('This table is a view'); ($key) = grep { m/create view/i } keys %$href; $self->{tables}->{$db}->{$tbl} = [ 'view', $href->{$key} ]; } } return $self->{tables}->{$db}->{$tbl}; } sub get_columns { my ( $self, $dbh, $quoter, $db, $tbl ) = @_; MKDEBUG && _d('Get columns for', $db, $tbl); if ( !$self->{cache} || !$self->{columns}->{$db}->{$tbl} ) { $self->_use_db($dbh, $quoter, $db); my $sql = "SHOW COLUMNS FROM " . $quoter->quote($db, $tbl); MKDEBUG && _d($sql); my $cols = $dbh->selectall_arrayref($sql, { Slice => {} }); $self->{columns}->{$db}->{$tbl} = [ map { my %row; @row{ map { lc $_ } keys %$_ } = values %$_; \%row; } @$cols ]; } return $self->{columns}->{$db}->{$tbl}; } sub get_tmp_table { my ( $self, $dbh, $quoter, $db, $tbl ) = @_; my $result = 'CREATE TABLE ' . $quoter->quote($tbl) . " (\n"; $result .= join(",\n", map { ' ' . $quoter->quote($_->{field}) . ' ' . $_->{type} } @{$self->get_columns($dbh, $quoter, $db, $tbl)}); $result .= "\n)"; MKDEBUG && _d($result); return $result; } sub get_triggers { my ( $self, $dbh, $quoter, $db, $tbl ) = @_; if ( !$self->{cache} || !$self->{triggers}->{$db} ) { $self->{triggers}->{$db} = {}; my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, ' . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), } . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, ' . '@@SQL_QUOTE_SHOW_CREATE := 1 */'; MKDEBUG && _d($sql); eval { $dbh->do($sql); }; MKDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); $sql = "SHOW TRIGGERS FROM " . $quoter->quote($db); MKDEBUG && _d($sql); my $sth = $dbh->prepare($sql); $sth->execute(); if ( $sth->rows ) { my $trgs = $sth->fetchall_arrayref({}); foreach my $trg (@$trgs) { my %trg; @trg{ map { lc $_ } keys %$trg } = values %$trg; push @{ $self->{triggers}->{$db}->{ $trg{table} } }, \%trg; } } $sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, ' . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */'; MKDEBUG && _d($sql); $dbh->do($sql); } if ( $tbl ) { return $self->{triggers}->{$db}->{$tbl}; } return values %{$self->{triggers}->{$db}}; } sub get_databases { my ( $self, $dbh, $quoter, $like ) = @_; if ( !$self->{cache} || !$self->{databases} || $like ) { my $sql = 'SHOW DATABASES'; my @params; if ( $like ) { $sql .= ' LIKE ?'; push @params, $like; } my $sth = $dbh->prepare($sql); MKDEBUG && _d($sql, @params); $sth->execute( @params ); my @dbs = map { $_->[0] } @{$sth->fetchall_arrayref()}; $self->{databases} = \@dbs unless $like; return @dbs; } return @{$self->{databases}}; } sub get_table_status { my ( $self, $dbh, $quoter, $db, $like ) = @_; if ( !$self->{cache} || !$self->{table_status}->{$db} || $like ) { my $sql = "SHOW TABLE STATUS FROM " . $quoter->quote($db); my @params; if ( $like ) { $sql .= ' LIKE ?'; push @params, $like; } MKDEBUG && _d($sql, @params); my $sth = $dbh->prepare($sql); $sth->execute(@params); my @tables = @{$sth->fetchall_arrayref({})}; @tables = map { my %tbl; # Make a copy with lowercased keys @tbl{ map { lc $_ } keys %$_ } = values %$_; $tbl{engine} ||= $tbl{type} || $tbl{comment}; delete $tbl{type}; \%tbl; } @tables; $self->{table_status}->{$db} = \@tables unless $like; return @tables; } return @{$self->{table_status}->{$db}}; } sub get_table_list { my ( $self, $dbh, $quoter, $db, $like ) = @_; if ( !$self->{cache} || !$self->{table_list}->{$db} || $like ) { my $sql = "SHOW /*!50002 FULL*/ TABLES FROM " . $quoter->quote($db); my @params; if ( $like ) { $sql .= ' LIKE ?'; push @params, $like; } MKDEBUG && _d($sql, @params); my $sth = $dbh->prepare($sql); $sth->execute(@params); my @tables = @{$sth->fetchall_arrayref()}; @tables = map { my %tbl = ( name => $_->[0], engine => ($_->[1] || '') eq 'VIEW' ? 'VIEW' : '', ); \%tbl; } @tables; $self->{table_list}->{$db} = \@tables unless $like; return @tables; } return @{$self->{table_list}->{$db}}; } sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } 1; } # ########################################################################### # End MySQLDump package # ########################################################################### # ########################################################################### # Daemon package # This package is a copy without comments from the original. The original # with comments and its test file can be found in the Bazaar repository at, # lib/Daemon.pm # t/lib/Daemon.t # See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { package Daemon; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; use POSIX qw(setsid); sub new { my ( $class, %args ) = @_; foreach my $arg ( qw(o) ) { die "I need a $arg argument" unless $args{$arg}; } my $o = $args{o}; my $self = { o => $o, log_file => $o->has('log') ? $o->get('log') : undef, PID_file => $o->has('pid') ? $o->get('pid') : undef, }; check_PID_file(undef, $self->{PID_file}); MKDEBUG && _d('Daemonized child will log to', $self->{log_file}); return bless $self, $class; } sub daemonize { my ( $self ) = @_; MKDEBUG && _d('About to fork and daemonize'); defined (my $pid = fork()) or die "Cannot fork: $OS_ERROR"; if ( $pid ) { MKDEBUG && _d('I am the parent and now I die'); exit; } $self->{PID_owner} = $PID; $self->{child} = 1; POSIX::setsid() or die "Cannot start a new session: $OS_ERROR"; chdir '/' or die "Cannot chdir to /: $OS_ERROR"; $self->_make_PID_file(); $OUTPUT_AUTOFLUSH = 1; if ( -t STDIN ) { close STDIN; open STDIN, '/dev/null' or die "Cannot reopen STDIN to /dev/null: $OS_ERROR"; } if ( $self->{log_file} ) { close STDOUT; open STDOUT, '>>', $self->{log_file} or die "Cannot open log file $self->{log_file}: $OS_ERROR"; close STDERR; open STDERR, ">&STDOUT" or die "Cannot dupe STDERR to STDOUT: $OS_ERROR"; } else { if ( -t STDOUT ) { close STDOUT; open STDOUT, '>', '/dev/null' or die "Cannot reopen STDOUT to /dev/null: $OS_ERROR"; } if ( -t STDERR ) { close STDERR; open STDERR, '>', '/dev/null' or die "Cannot reopen STDERR to /dev/null: $OS_ERROR"; } } MKDEBUG && _d('I am the child and now I live daemonized'); return; } sub check_PID_file { my ( $self, $file ) = @_; my $PID_file = $self ? $self->{PID_file} : $file; MKDEBUG && _d('Checking PID file', $PID_file); if ( $PID_file && -f $PID_file ) { my $pid; eval { chomp($pid = `cat $PID_file`); }; die "Cannot cat $PID_file: $OS_ERROR" if $EVAL_ERROR; MKDEBUG && _d('PID file exists; it contains PID', $pid); if ( $pid ) { my $pid_is_alive = kill 0, $pid; if ( $pid_is_alive ) { die "The PID file $PID_file already exists " . " and the PID that it contains, $pid, is running"; } else { warn "Overwriting PID file $PID_file because the PID that it " . "contains, $pid, is not running"; } } else { die "The PID file $PID_file already exists but it does not " . "contain a PID"; } } else { MKDEBUG && _d('No PID file'); } return; } sub make_PID_file { my ( $self ) = @_; if ( exists $self->{child} ) { die "Do not call Daemon::make_PID_file() for daemonized scripts"; } $self->_make_PID_file(); $self->{PID_owner} = $PID; return; } sub _make_PID_file { my ( $self ) = @_; my $PID_file = $self->{PID_file}; if ( !$PID_file ) { MKDEBUG && _d('No PID file to create'); return; } $self->check_PID_file(); open my $PID_FH, '>', $PID_file or die "Cannot open PID file $PID_file: $OS_ERROR"; print $PID_FH $PID or die "Cannot print to PID file $PID_file: $OS_ERROR"; close $PID_FH or die "Cannot close PID file $PID_file: $OS_ERROR"; MKDEBUG && _d('Created PID file:', $self->{PID_file}); return; } sub _remove_PID_file { my ( $self ) = @_; if ( $self->{PID_file} && -f $self->{PID_file} ) { unlink $self->{PID_file} or warn "Cannot remove PID file $self->{PID_file}: $OS_ERROR"; MKDEBUG && _d('Removed PID file'); } else { MKDEBUG && _d('No PID to remove'); } return; } sub DESTROY { my ( $self ) = @_; $self->_remove_PID_file() if ($self->{PID_owner} || 0) == $PID; return; } sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } 1; } # ########################################################################### # End Daemon package # ########################################################################### # ########################################################################### # This is a combination of modules and programs in one -- a runnable module. # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last # Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition. # # Check at the end of this package for the call to main() which actually runs # the program. # ########################################################################### package pt_find; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; $OUTPUT_AUTOFLUSH = 1; # ############################################################################ # Lookup tables and global variables # ############################################################################ my $o; # OptionParser obj my %fmt_for; # Interpolated strings my %time_for; # Holds time constants for mmin, mtime etc my %connections; # Holds a list of thread IDs connected my $server_id; # Holds the server's @@SERVER_ID my $dbh; # This program's $dbh my $exec_dbh; # The $dbh to use for exec and exec-plus my $tp; my $du; # Functions to call while evaluating tests. my %test_for = ( autoinc => sub { my ( $table ) = @_; return test_number($table, 'Auto_increment', $o->get('autoinc')); }, avgrowlen => sub { my ( $table ) = @_; return test_number($table, 'Avg_row_length', $o->get('avgrowlen')); }, checksum => sub { my ( $table ) = @_; return test_number($table, 'Checksum', $o->get('checksum')); }, cmin => sub { my ( $table ) = @_; return test_date($table, 'Create_time', 'cmin'); }, collation => sub { my ( $table ) = @_; return test_regex($table, 'Collation', $o->get('collation')); }, 'column-name' => sub { my ( $table ) = @_; my $struct = $table->{struct}; return unless $struct; my $test = $o->get('column-name'); if ( $o->get('case-insensitive') ) { $test = "(?i)$test"; } foreach my $col ( @{$struct->{cols}} ) { return 1 if $col =~ m/$test/; } return 0; }, 'column-type' => sub { my ( $table ) = @_; my $struct = $table->{struct}; return unless $struct; my $test = lc($o->get('column-type')); my $type_for = $struct->{type_for}; foreach my $col ( keys %$type_for ) { return 1 if $type_for->{$col} eq $test; } return 0; }, comment => sub { my ( $table ) = @_; return test_regex($table, 'Comment', $o->get('comment')); }, createopts => sub { my ( $table ) = @_; return test_regex($table, 'Create_options', $o->get('createopts')); }, ctime => sub { my ( $table ) = @_; return test_date($table, 'Create_time', 'ctime'); }, datafree => sub { my ( $table ) = @_; return test_number($table, 'Data_free', $o->get('datafree')); }, datasize => sub { my ( $table ) = @_; return test_number($table, 'Data_length', $o->get('datasize')); }, dbregex => sub { my ( $table ) = @_; return test_regex($table, 'Database', $o->get('dbregex')); }, empty => sub { my ( $table ) = @_; return test_number($table, 'Rows', '0'); }, engine => sub { my ( $table ) = @_; return test_regex($table, 'Engine', $o->get('engine')); }, function => sub { my ( $table ) = @_; return unless $table->{stored_code} && $table->{stored_code} eq 'FUNCTION'; my $def = $table->{def}; return unless $def; my $test = $o->get('function'); if ( $o->get('case-insensitive') ) { $test = "(?i)$test"; } return $def =~ m/$test/; }, indexsize => sub { my ( $table ) = @_; return test_number($table, 'Index_length', $o->get('indexsize')); }, kmin => sub { my ( $table ) = @_; return test_date($table, 'Check_time', 'kmin'); }, ktime => sub { my ( $table ) = @_; return test_date($table, 'Check_time', 'ktime'); }, mmin => sub { my ( $table ) = @_; return test_date($table, 'Update_time', 'mmin'); }, mtime => sub { my ( $table ) = @_; return test_date($table, 'Update_time', 'mtime'); }, 'connection-id' => sub { my ( $table ) = @_; my $test = $o->get('case-insensitive') ? "(?i)".$o->get('connection-id') : $o->get('connection-id'); my ( $pid ) = $table->{Name} =~ m/$test/; return $pid && !exists $connections{$pid}; }, procedure => sub { my ( $table ) = @_; return unless $table->{stored_code} && $table->{stored_code} eq 'PROCEDURE'; my $def = $table->{def}; return unless $def; my $test = $o->get('procedure'); if ( $o->get('case-insensitive') ) { $test = "(?i)$test"; } return $def =~ m/$test/; }, rows => sub { my ( $table ) = @_; return test_number($table, 'Rows', $o->get('rows')); }, rowformat => sub { my ( $table ) = @_; return test_regex($table, 'Row_format', $o->get('rowformat')); }, 'server-id' => sub { my ( $table ) = @_; my $test = $o->get('case-insensitive') ? "(?i)".$o->get('server-id') : $o->get('server-id'); my ( $sid ) = $table->{Name} =~ m/$test/; return $sid && $sid == $server_id; }, tablesize => sub { my ( $table ) = @_; return test_number($table, 'Table_length', $o->get('tablesize')); }, tblregex => sub { my ( $table ) = @_; return test_regex($table, 'Name', $o->get('tblregex')); }, tblversion => sub { my ( $table ) = @_; return test_number($table, 'Version', $o->get('tblversion')); }, trigger => sub { my ( $table ) = @_; return unless $table->{stored_code} && $table->{stored_code} eq 'TRIGGER'; my $def = $table->{def}; return unless $def; my $test = $o->get('trigger'); if ( $o->get('case-insensitive') ) { $test = "(?i)$test"; } return $def =~ m/$test/; }, 'trigger-table' => sub { my ( $table ) = @_; return unless $table->{stored_code} && $table->{stored_code} eq 'TRIGGER'; my $test = $o->get('trigger-table'); if ( $o->get('case-insensitive') ) { $test = "(?i)$test"; } return $table->{trigger_table} =~ m/$test/; }, view => sub { my ( $table ) = @_; my $view = $table->{view}; return unless $view; my $test = $o->get('view'); if ( $o->get('case-insensitive') ) { $test = "(?i)$test"; } return $view =~ m/$test/; }, ); # Functions to call when doing actions my %action_for = ( print => sub { my ( $table ) = @_; print "$table->{Database}.$table->{Name}\n"; }, exec => sub { my ( $table ) = @_; my $sql = sprintf($fmt_for{exec}->{str}, map { defined $_ ? $_ : '' } @{$table}{@{$fmt_for{exec}->{arg_names}}}); MKDEBUG && _d($sql); $exec_dbh->do($sql); }, printf => sub { my ( $table ) = @_; printf($fmt_for{printf}->{str}, map { defined $_ ? $_ : '' } @{$table}{@{$fmt_for{printf}->{arg_names}}}); }, ); my %arg_for = ( a => 'Auto_increment', A => 'Avg_row_length', c => 'Checksum', C => 'Create_time', D => 'Database', d => 'Data_length', E => 'Engine', F => 'Data_free', f => 'Innodb_free', I => 'Index_length', K => 'Check_time', L => 'Collation', M => 'Max_data_length', N => 'Name', O => 'Comment', P => 'Create_options', R => 'Row_format', S => 'Rows', T => 'Table_length', U => 'Update_time', V => 'Version', ); my @table_struct_tests = qw( column-name column-type view ); my @stored_code_tests = qw( procedure function trigger ); sub main { @ARGV = @_; # set global ARGV for this package # ######################################################################## # Get configuration information. # ######################################################################## my $q = new Quoter(); $o = new OptionParser(); $o->get_specs(); $o->get_opts(); my $dp = $o->DSNParser(); $dp->prop('set-vars', $o->get('set-vars')); # Make sure OptionParser understands that these options are used. # cmin ctime empty kmin ktime mmin mtime exec printf # Ensure there is a capture group. if ( $o->get('connection-id') && $o->get('connection-id') !~ m/\(\\d\+\)/ ) { $o->save_error("--connection-id regex doesn't capture digits with (\\d+)"); } # Ensure there is a capture group. if ( $o->get('server-id') && $o->get('server-id') !~ m/\(\\d\+\)/ ) { $o->save_error("--server-id regex doesn't capture digits with (\\d+)"); } $o->usage_or_errors(); # Interpolate strings for printf and exec. At the same time discover whether # I must use SHOW TABLE STATUS (slower than SHOW TABLES) to fetch data. my $showstat = grep { $o->get($_) } qw( autoinc avgrowlen checksum cmin collation comment createopts ctime datasize datafree empty engine indexsize kmin ktime mmin mtime rows rowformat tablesize tblversion); foreach my $thing (qw(exec printf)) { next unless $o->get($thing); my ($str, $arg_names) = interpolate($o->get($thing)); $fmt_for{$thing} = { str => $str, arg_names => $arg_names }; if ( grep { $_ !~ m/^(Database|Name)$/ } @$arg_names ) { $showstat = 1; } } # Discover if we need to parse SHOW CREATE TABLE. my $need_table_struct = grep { $o->got($_); } @table_struct_tests; MKDEBUG && _d('Need table struct:', $need_table_struct); if ( $need_table_struct ) { $du = new MySQLDump(); $tp = new TableParser(Quoter => $q); } # ######################################################################## # If --pid, check it first since we'll die if it already exits. # ######################################################################## my $daemon; if ( $o->get('pid') ) { # We're not daemoninzing, it just handles PID stuff. Keep $daemon # in the the scope of main() because when it's destroyed it automatically # removes the PID file. $daemon = new Daemon(o=>$o); $daemon->make_PID_file(); } # ######################################################################## # Get ready to do the main work. # ######################################################################## # Connect to the database. if ( $o->get('ask-pass') ) { $o->set('password', OptionParser::prompt_noecho("Enter password: ")); } my $dsn = $dp->parse_options($o); $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), { AutoCommit => 1 } ); if ( $o->get('exec-dsn') ) { my $exec_dsn = $dp->parse($o->get('exec-dsn'), $dsn); $exec_dbh = $dp->get_dbh($dp->get_cxn_params($exec_dsn), { AutoCommit => 1 }); } else { $exec_dbh = $dbh; } # If no other action was given, the default action is to print. if ( !grep { $o->get($_) } qw( exec exec-plus print printf ) ) { $o->set('print', 1); } # Figure out the time referred to by date/time options. my $basetime; foreach my $option ( grep { defined $o->get($_) } qw(cmin ctime kmin ktime mmin mtime) ) { # Initialize a consistent point in time. $basetime ||= $dbh->selectcol_arrayref( "SELECT " . ($o->get('day-start') ? 'CURRENT_DATE' : 'CURRENT_TIMESTAMP') )->[0]; my ($val) = $o->get($option) =~ m/(\d+)/; my $inter = $option =~ m/min/ ? 'MINUTE' : 'DAY'; my $query = "SELECT DATE_SUB('$basetime', INTERVAL $val $inter)"; $time_for{$option} = $dbh->selectcol_arrayref($query)->[0]; } # Fetch and save a list of processes currently running. if ( $o->get('connection-id') ) { # Ensure I have the PROCESS privilege. my $proc = grep { m/ALL PRIVILEGES.*?\*\.\*|PROCESS/ } @{$dbh->selectcol_arrayref('SHOW GRANTS')}; if ( !$proc ) { die "--connection-id requires the PROCESS privilege for safety.\n"; } } ($server_id) = $dbh->selectrow_array('SELECT @@SERVER_ID'); # Discover if we need to get stored code. Need dbh to do this. my $vp = new VersionParser(); my $need_stored_code = $vp->version_ge($dbh, '5.0.0'); $need_stored_code = grep { $o->got($_); } @stored_code_tests if $need_stored_code; MKDEBUG && _d('Need stored code:', $need_stored_code); # ######################################################################## # Go do it. # ######################################################################## my @databases = @ARGV ? @ARGV : $o->get('dblike') ? @{$dbh->selectcol_arrayref('SHOW DATABASES LIKE ?', {}, $o->get('dblike'))} : @{$dbh->selectcol_arrayref('SHOW DATABASES')}; my @exec_plus; DATABASE: foreach my $database ( @databases ) { next DATABASE if $database =~ m/^(?:information_schema|lost\+found)$/mi; my $sta = $showstat ? ' STATUS' : 'S'; my $sth = $o->get('tbllike') ? $dbh->prepare("SHOW TABLE$sta FROM `$database` LIKE ?") : $dbh->prepare("SHOW TABLE$sta FROM `$database`"); $sth->execute($o->get('tbllike') || ()); my @tables = @{$sth->fetchall_arrayref({})}; # Must re-fetch every time; there are too many ways things can go wrong # otherwise (for example, the counter wraps over the unsigned int # boundary). if ( $o->get('connection-id') ) { %connections = map { $_ => 1 } @{$dbh->selectcol_arrayref('SHOW FULL PROCESSLIST')}; } # Make results uniform across MySQL versions, and generate additional # properties. foreach my $table ( @tables ) { if ( $showstat ) { my ($ib_free) = $table->{Comment} && $table->{Comment} =~ m/InnoDB free: (\d+) kB/; $table->{Engine} ||= $table->{Type}; $table->{Table_length} = ($table->{Index_length} || 0) + ($table->{Data_length} || 0); $table->{Innodb_free} = $ib_free ? 1_024 * $ib_free : undef; delete $table->{Type}; } else { my ($name) = values %$table; $table = { Name => $name }; } $table->{Database} = $database; if ( $need_table_struct ) { MKDEBUG && _d('Getting table struct for', $database, '.', $table->{Name}); my $ddl = $du->get_create_table($dbh,$q, $database, $table->{Name}); if ( $ddl->[0] eq 'table' ) { my $table_struct; eval { $table_struct = $tp->parse($ddl) }; if ( $EVAL_ERROR ) { MKDEBUG && _d('Failed to parse table:', $EVAL_ERROR); } $table->{struct} = $table_struct; } elsif ( $ddl->[0] eq 'view' ) { $table->{view} = $ddl->[1]; } } } if ( $need_stored_code ) { foreach my $type ( qw(PROCEDURE FUNCTION) ) { my $sql = "SELECT ROUTINE_NAME AS name, " . " ROUTINE_DEFINITION AS definition " . " FROM INFORMATION_SCHEMA.ROUTINES " . " WHERE ROUTINE_SCHEMA = '$database' " . " AND ROUTINE_TYPE = '$type'"; MKDEBUG && _d($sql); my $codes = $dbh->selectall_arrayref($sql); foreach my $code ( @$codes ) { push @tables, { Database => $database, Name => "$type $code->[0]", stored_code => $type, def => $code->[1], }; } } my $sql = "SELECT TRIGGER_NAME AS name, " . " ACTION_STATEMENT AS action, " . " EVENT_OBJECT_TABLE AS `table`, " . " EVENT_MANIPULATION AS type " . " FROM INFORMATION_SCHEMA.TRIGGERS " . " WHERE EVENT_OBJECT_SCHEMA = '$database'"; MKDEBUG && _d($sql); my $trigs = $dbh->selectall_arrayref($sql); my $codes = $dbh->selectall_arrayref($sql); foreach my $trig ( @$trigs ) { push @tables, { Database => $database, Name => "$trig->[3] TRIGGER $trig->[0] on $trig->[2]", trigger_table => $trig->[2], stored_code => 'TRIGGER', def => $trig->[1], }; } } # Apply the tests to find the matching tables. @tables = grep { my $table = $_; my @tests = grep { $o->get($_) } keys %test_for; if ( @tests ) { ($o->get('or') ? any($table, @tests) : all($table, @tests)); } else { $table; # No tests == all tables (issue 549). } } @tables; # Quote database and table names if desired. if ( $o->get('quote') ) { foreach my $table ( @tables ) { $table->{Database} = $q->quote($table->{Database}); $table->{Name} = $q->quote($table->{Name}); } } foreach my $table ( @tables ) { my @actions = grep { $o->get($_) } keys %action_for; foreach my $action ( @actions ) { $action_for{$action}->($table); } } push @exec_plus, @tables; } # Handle exec-plus. if ( $o->get('exec-plus') ) { my $table_list = join(', ',map {"$_->{Database}.$_->{Name}"} @exec_plus); (my $sql = $o->get('exec-plus')) =~ s/%s/$table_list/g; $exec_dbh->do($sql); } return 0; } # ############################################################################ # Subroutines # ############################################################################ # One test is true sub any { my ( $table, @tests ) = @_; foreach my $test ( @tests ) { return 1 if $test_for{$test}->($table); } return 0; } # All tests are true sub all { my ( $table, @tests ) = @_; foreach my $test ( @tests ) { return 0 unless $test_for{$test}->($table); } return 1; } # Checks the given property of the given table to see if it passes the test sub test_number { my ( $table, $prop, $test ) = @_; # E.g. --datasize NULL. if ( $test eq 'null' ) { return !defined $table->{$prop}; } my ($num) = $test =~ m/(\d+)/; return defined $table->{$prop} && ( ( $test =~ m/-/ && $table->{$prop} < $num ) || ( $test =~ m/\+/ && $table->{$prop} > $num ) || ( $table->{$prop} == $num )); } # Checks the given property of the given table to see if it passes the test sub test_date { my ( $table, $prop, $test ) = @_; return defined $table->{$prop} && ( ( $o->get($test) =~ m/-/ && $table->{$prop} gt $time_for{$test} ) || ( $o->get($test) =~ m/\+/ && $table->{$prop} lt $time_for{$test} ) || ( $table->{$prop} eq $time_for{$test} )); } # Checks the given property of the given table to see if it passes the test sub test_regex { my ( $table, $prop, $test ) = @_; if ( $o->get('case-insensitive') ) { $test = "(?i)$test"; } return defined $table->{$prop} && $table->{$prop} =~ m/$test/; } # Does string-interpolation and stuff. Returns the string and a list of the # properties that go into the resulting placeholders. sub interpolate { my ( $str ) = @_; my @arg_names; # Replace % directives $str =~ s/%(.)/(exists $arg_for{$1} && push @arg_names, $arg_for{$1} ) ? '\%s' : "$1"/xge; # Get Perl to interpolate escape sequences $str =~ s/(? 1_024, M => 1_048_576, G => 1_073_741_824); my ($pre, $num, $factor) = $test =~ m/([+-])?(\d+)([kMG])?/; if ( $factor ) { $num *= $factor_for{$factor}; } return "$pre$num"; } sub _d { my ($package, undef, $line) = caller 0; @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } map { defined $_ ? $_ : 'undef' } @_; print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } # ############################################################################ # Run the program. # ############################################################################ if ( !caller ) { exit main(@ARGV); } 1; # Because this is a module as well as a script. # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME pt-find - Find MySQL tables and execute actions, like GNU find. =head1 SYNOPSIS Usage: pt-find [OPTION...] [DATABASE...] pt-find searches for MySQL tables and executes actions, like GNU find. The default action is to print the database and table name. Find all tables created more than a day ago, which use the MyISAM engine, and print their names: pt-find --ctime +1 --engine MyISAM Find InnoDB tables that haven't been updated in a month, and convert them to MyISAM storage engine (data warehousing, anyone?): pt-find --mtime +30 --engine InnoDB --exec "ALTER TABLE %D.%N ENGINE=MyISAM" Find tables created by a process that no longer exists, following the name_sid_pid naming convention, and remove them. pt-find --connection-id '\D_\d+_(\d+)$' --server-id '\D_(\d+)_\d+$' --exec-plus "DROP TABLE %s" Find empty tables in the test and junk databases, and delete them: pt-find --empty junk test --exec-plus "DROP TABLE %s" Find tables more than five gigabytes in total size: pt-find --tablesize +5G Find all tables and print their total data and index size, and sort largest tables first (sort is a different program, by the way). pt-find --printf "%T\t%D.%N\n" | sort -rn As above, but this time, insert the data back into the database for posterity: pt-find --noquote --exec "INSERT INTO sysdata.tblsize(db, tbl, size) VALUES('%D', '%N', %T)" =head1 RISKS The following section is included to inform users about the potential risks, whether known or unknown, of using this tool. The two main categories of risks are those created by the nature of the tool (e.g. read-only tools vs. read-write tools) and those created by bugs. pt-find only reads and prints information by default, but L<"--exec"> and L<"--exec-plus"> can execute user-defined SQL. You should be as careful with it as you are with any command-line tool that can execute queries against your database. At the time of this release, we know of no bugs that could cause serious harm to users. The authoritative source for updated information is always the online issue tracking system. Issues that affect this tool will be marked as such. You can see a list of such issues at the following URL: L. See also L<"BUGS"> for more information on filing bugs and getting help. =head1 DESCRIPTION pt-find looks for MySQL tables that pass the tests you specify, and executes the actions you specify. The default action is to print the database and table name to STDOUT. pt-find is simpler than GNU find. It doesn't allow you to specify complicated expressions on the command line. pt-find uses SHOW TABLES when possible, and SHOW TABLE STATUS when needed. =head1 OPTION TYPES There are three types of options: normal options, which determine some behavior or setting; tests, which determine whether a table should be included in the list of tables found; and actions, which do something to the tables pt-find finds. pt-find uses standard Getopt::Long option parsing, so you should use double dashes in front of long option names, unlike GNU find. =head1 OPTIONS This tool accepts additional command-line arguments. Refer to the L<"SYNOPSIS"> and usage information for details. =over =item --ask-pass Prompt for a password when connecting to MySQL. =item --case-insensitive Specifies that all regular expression searches are case-insensitive. =item --charset short form: -A; type: string Default character set. If the value is utf8, sets Perl's binmode on STDOUT to utf8, passes the mysql_enable_utf8 option to DBD::mysql, and runs SET NAMES UTF8 after connecting to MySQL. Any other value sets binmode on STDOUT without the utf8 layer, and runs SET NAMES after connecting to MySQL. =item --config type: Array Read this comma-separated list of config files; if specified, this must be the first option on the command line. =item --day-start Measure times (for L<"--mmin">, etc) from the beginning of today rather than from the current time. =item --defaults-file short form: -F; type: string Only read mysql options from the given file. You must give an absolute pathname. =item --help Show help and exit. =item --host short form: -h; type: string Connect to host. =item --or Combine tests with OR, not AND. By default, tests are evaluated as though there were an AND between them. This option switches it to OR. Option parsing is not implemented by pt-find itself, so you cannot specify complicated expressions with parentheses and mixtures of OR and AND. =item --password short form: -p; type: string Password to use when connecting. =item --pid type: string Create the given PID file. The file contains the process ID of the script. The PID file is removed when the script exits. Before starting, the script checks if the PID file already exists. If it does not, then the script creates and writes its own PID to it. If it does, then the script checks the following: if the file contains a PID and a process is running with that PID, then the script dies; or, if there is no process running with that PID, then the script overwrites the file with its own PID and starts; else, if the file contains no PID, then the script dies. =item --port short form: -P; type: int Port number to use for connection. =item --[no]quote default: yes Quotes MySQL identifier names with MySQL's standard backtick character. Quoting happens after tests are run, and before actions are run. =item --set-vars type: string; default: wait_timeout=10000 Set these MySQL variables. Immediately after connecting to MySQL, this string will be appended to SET and executed. =item --socket short form: -S; type: string Socket file to use for connection. =item --user short form: -u; type: string User for login if not current user. =item --version Show version and exit. =back =head2 TESTS Most tests check some criterion against a column of SHOW TABLE STATUS output. Numeric arguments can be specified as +n for greater than n, -n for less than n, and n for exactly n. All numeric options can take an optional suffix multiplier of k, M or G (1_024, 1_048_576, and 1_073_741_824 respectively). All patterns are Perl regular expressions (see 'man perlre') unless specified as SQL LIKE patterns. Dates and times are all measured relative to the same instant, when pt-find first asks the database server what time it is. All date and time manipulation is done in SQL, so if you say to find tables modified 5 days ago, that translates to SELECT DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 DAY). If you specify L<"--day-start">, if course it's relative to CURRENT_DATE instead. However, table sizes and other metrics are not consistent at an instant in time. It can take some time for MySQL to process all the SHOW queries, and pt-find can't do anything about that. These measurements are as of the time they're taken. If you need some test that's not in this list, file a bug report and I'll enhance pt-find for you. It's really easy. =over =item --autoinc type: string; group: Tests Table's next AUTO_INCREMENT is n. This tests the Auto_increment column. =item --avgrowlen type: size; group: Tests Table avg row len is n bytes. This tests the Avg_row_length column. The specified size can be "NULL" to test where Avg_row_length IS NULL. =item --checksum type: string; group: Tests Table checksum is n. This tests the Checksum column. =item --cmin type: size; group: Tests Table was created n minutes ago. This tests the Create_time column. =item --collation type: string; group: Tests Table collation matches pattern. This tests the Collation column. =item --column-name type: string; group: Tests A column name in the table matches pattern. =item --column-type type: string; group: Tests A column in the table matches this type (case-insensitive). Examples of types are: varchar, char, int, smallint, bigint, decimal, year, timestamp, text, enum. =item --comment type: string; group: Tests Table comment matches pattern. This tests the Comment column. =item --connection-id type: string; group: Tests Table name has nonexistent MySQL connection ID. This tests the table name for a pattern. The argument to this test must be a Perl regular expression that captures digits like this: (\d+). If the table name matches the pattern, these captured digits are taken to be the MySQL connection ID of some process. If the connection doesn't exist according to SHOW FULL PROCESSLIST, the test returns true. If the connection ID is greater than pt-find's own connection ID, the test returns false for safety. Why would you want to do this? If you use MySQL statement-based replication, you probably know the trouble temporary tables can cause. You might choose to work around this by creating real tables with unique names, instead of temporary tables. One way to do this is to append your connection ID to the end of the table, thusly: scratch_table_12345. This assures the table name is unique and lets you have a way to find which connection it was associated with. And perhaps most importantly, if the connection no longer exists, you can assume the connection died without cleaning up its tables, and this table is a candidate for removal. This is how I manage scratch tables, and that's why I included this test in pt-find. The argument I use to L<"--connection-id"> is "\D_(\d+)$". That finds tables with a series of numbers at the end, preceded by an underscore and some non-number character (the latter criterion prevents me from examining tables with a date at the end, which people tend to do: baron_scratch_2007_05_07 for example). It's better to keep the scratch tables separate of course. If you do this, make sure the user pt-find runs as has the PROCESS privilege! Otherwise it will only see connections from the same user, and might think some tables are ready to remove when they're still in use. For safety, pt-find checks this for you. See also L<"--server-id">. =item --createopts type: string; group: Tests Table create option matches pattern. This tests the Create_options column. =item --ctime type: size; group: Tests Table was created n days ago. This tests the Create_time column. =item --datafree type: size; group: Tests Table has n bytes of free space. This tests the Data_free column. The specified size can be "NULL" to test where Data_free IS NULL. =item --datasize type: size; group: Tests Table data uses n bytes of space. This tests the Data_length column. The specified size can be "NULL" to test where Data_length IS NULL. =item --dblike type: string; group: Tests Database name matches SQL LIKE pattern. =item --dbregex type: string; group: Tests Database name matches this pattern. =item --empty group: Tests Table has no rows. This tests the Rows column. =item --engine type: string; group: Tests Table storage engine matches this pattern. This tests the Engine column, or in earlier versions of MySQL, the Type column. =item --function type: string; group: Tests Function definition matches pattern. =item --indexsize type: size; group: Tests Table indexes use n bytes of space. This tests the Index_length column. The specified size can be "NULL" to test where Index_length IS NULL. =item --kmin type: size; group: Tests Table was checked n minutes ago. This tests the Check_time column. =item --ktime type: size; group: Tests Table was checked n days ago. This tests the Check_time column. =item --mmin type: size; group: Tests Table was last modified n minutes ago. This tests the Update_time column. =item --mtime type: size; group: Tests Table was last modified n days ago. This tests the Update_time column. =item --procedure type: string; group: Tests Procedure definition matches pattern. =item --rowformat type: string; group: Tests Table row format matches pattern. This tests the Row_format column. =item --rows type: size; group: Tests Table has n rows. This tests the Rows column. The specified size can be "NULL" to test where Rows IS NULL. =item --server-id type: string; group: Tests Table name contains the server ID. If you create temporary tables with the naming convention explained in L<"--connection-id">, but also add the server ID of the server on which the tables are created, then you can use this pattern match to ensure tables are dropped only on the server they're created on. This prevents a table from being accidentally dropped on a slave while it's in use (provided that your server IDs are all unique, which they should be for replication to work). For example, on the master (server ID 22) you create a table called scratch_table_22_12345. If you see this table on the slave (server ID 23), you might think it can be dropped safely if there's no such connection 12345. But if you also force the name to match the server ID with C<--server-id '\D_(\d+)_\d+$'>, the table won't be dropped on the slave. =item --tablesize type: size; group: Tests Table uses n bytes of space. This tests the sum of the Data_length and Index_length columns. =item --tbllike type: string; group: Tests Table name matches SQL LIKE pattern. =item --tblregex type: string; group: Tests Table name matches this pattern. =item --tblversion type: size; group: Tests Table version is n. This tests the Version column. =item --trigger type: string; group: Tests Trigger action statement matches pattern. =item --trigger-table type: string; group: Tests L<"--trigger"> is defined on table matching pattern. =item --view type: string; group: Tests CREATE VIEW matches this pattern. =back =head2 ACTIONS The L<"--exec-plus"> action happens after everything else, but otherwise actions happen in an indeterminate order. If you need determinism, file a bug report and I'll add this feature. =over =item --exec type: string; group: Actions Execute this SQL with each item found. The SQL can contain escapes and formatting directives (see L<"--printf">). =item --exec-dsn type: string; group: Actions Specify a DSN in key-value format to use when executing SQL with L<"--exec"> and L<"--exec-plus">. Any values not specified are inherited from command-line arguments. =item --exec-plus type: string; group: Actions Execute this SQL with all items at once. This option is unlike L<"--exec">. There are no escaping or formatting directives; there is only one special placeholder for the list of database and table names, %s. The list of tables found will be joined together with commas and substituted wherever you place %s. You might use this, for example, to drop all the tables you found: DROP TABLE %s This is sort of like GNU find's "-exec command {} +" syntax. Only it's not totally cryptic. And it doesn't require me to write a command-line parser. =item --print group: Actions Print the database and table name, followed by a newline. This is the default action if no other action is specified. =item --printf type: string; group: Actions Print format on the standard output, interpreting '\' escapes and '%' directives. Escapes are backslashed characters, like \n and \t. Perl interprets these, so you can use any escapes Perl knows about. Directives are replaced by %s, and as of this writing, you can't add any special formatting instructions, like field widths or alignment (though I'm musing over ways to do that). Here is a list of the directives. Note that most of them simply come from columns of SHOW TABLE STATUS. If the column is NULL or doesn't exist, you get an empty string in the output. A % character followed by any character not in the following list is discarded (but the other character is printed). CHAR DATA SOURCE NOTES ---- ------------------ ------------------------------------------ a Auto_increment A Avg_row_length c Checksum C Create_time D Database The database name in which the table lives d Data_length E Engine In older versions of MySQL, this is Type F Data_free f Innodb_free Parsed from the Comment field I Index_length K Check_time L Collation M Max_data_length N Name O Comment P Create_options R Row_format S Rows T Table_length Data_length+Index_length U Update_time V Version =back =head1 DSN OPTIONS These DSN options are used to create a DSN. Each option is given like C. The options are case-sensitive, so P and p are not the same option. There cannot be whitespace before or after the C<=> and if the value contains whitespace it must be quoted. DSN options are comma-separated. See the L manpage for full details. =over =item * A dsn: charset; copy: yes Default character set. =item * D dsn: database; copy: yes Default database. =item * F dsn: mysql_read_default_file; copy: yes Only read default options from the given file =item * h dsn: host; copy: yes Connect to host. =item * p dsn: password; copy: yes Password to use when connecting. =item * P dsn: port; copy: yes Port number to use for connection. =item * S dsn: mysql_socket; copy: yes Socket file to use for connection. =item * u dsn: user; copy: yes User for login if not current user. =back =head1 ENVIRONMENT The environment variable C enables verbose debugging output to STDERR. To enable debugging and capture all output to a file, run the tool like: PTDEBUG=1 pt-find ... > FILE 2>&1 Be careful: debugging output is voluminous and can generate several megabytes of output. =head1 SYSTEM REQUIREMENTS You need Perl, DBI, DBD::mysql, and some core packages that ought to be installed in any reasonably new version of Perl. =head1 BUGS For a list of known bugs, see L. Please report bugs at L. Include the following information in your bug report: =over =item * Complete command-line used to run the tool =item * Tool L<"--version"> =item * MySQL version of all servers involved =item * Output from the tool including STDERR =item * Input files (log/dump/config files, etc.) =back If possible, include debugging output by running the tool with C; see L<"ENVIRONMENT">. =head1 DOWNLOADING Visit L to download the latest release of Percona Toolkit. Or, get the latest release from the command line: wget percona.com/get/percona-toolkit.tar.gz wget percona.com/get/percona-toolkit.rpm wget percona.com/get/percona-toolkit.deb You can also get individual tools from the latest release: wget percona.com/get/TOOL Replace C with the name of any tool. =head1 AUTHORS Baron Schwartz =head1 ABOUT PERCONA TOOLKIT This tool is part of Percona Toolkit, a collection of advanced command-line tools developed by Percona for MySQL support and consulting. Percona Toolkit was forked from two projects in June, 2011: Maatkit and Aspersa. Those projects were created by Baron Schwartz and developed primarily by him and Daniel Nichter, both of whom are employed by Percona. Visit L for more software developed by Percona. =head1 COPYRIGHT, LICENSE, AND WARRANTY This program is copyright 2007-2011 Baron Schwartz, 2011 Percona Inc. Feedback and improvements are welcome. THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2; OR the Perl Artistic License. On UNIX and similar systems, you can issue `man perlgpl' or `man perlartistic' to read these licenses. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. =head1 VERSION pt-find 1.0.1 =cut