diff --git a/bin/pt-diskstats b/bin/pt-diskstats old mode 100644 new mode 100755 index 90bf89e7..9b0f614f --- a/bin/pt-diskstats +++ b/bin/pt-diskstats @@ -1,49 +1,9 @@ -#!/usr/local/bin/perl #!/usr/bin/env perl -#!/bin/sh - + # This program is part of Percona Toolkit: http://www.percona.com/software/ # See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal # notices and disclaimers. -# The following wrapper was borrowed from Tom Christiansen's Unicode::Tussle. - -################################################################ -# -# This is an sh wrapper to run the script under -# whichever perl occurs first in your path. See -# CHOICEs 1 and 2 below for alternate strategies. -# The -x will throw off your line numbers otherwise. -# -###################################################################### -# -# The next line is legal in both shell and perl, -# but perl sees the if 0 so doesn't execute it. -# - -eval 'exec perl -x $0 ${1+"$@"}' - if 0; - -### CHOICE 1: -###################################################################### -### MAKE FOLLOWING #! line THE TOP LINE, REPLACING /usr/local/bin ### -### with wherever you have a late enough version of Perl is ### -### installed. Will run under 5.8, but the newer the better. ### -###################################################################### -#!/usr/local/bin/perl -# ^^^^^^^^^^^^^^ <=== CHANGE ME ### -###################################################################### - -### CHOICE 2: -###################################################################### -### ALTERNATELY, the following #! line does the same thing as ### -### the tricksy sh eval exec line: it finds whichever Perl is ### -### first in your path. However, it works only on BSD systems ### -### (including MacOS), but breaks under Solaris and Linux. ### -###################################################################### -#!/usr/bin/env perl -###################################################################### - use strict; use warnings FATAL => 'all'; use constant MKDEBUG => $ENV{MKDEBUG} || 0; @@ -55,65 +15,15 @@ BEGIN { } qw( OptionParser ReadKeyMini Diskstats DiskstatsGroupByAll DiskstatsGroupByDisk DiskstatsGroupBySample DiskstatsMenu pt_diskstats ); } -# This program is copyright 2007-2011 Baron Schwartz, 2011 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### # OptionParser package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/OptionParser.pm +# t/lib/OptionParser.t +# See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { -# Package: OptionParser -# OptionParser parses command line options from a tool's POD. By default -# it parses a description and usage from the POD's SYNOPSIS section and -# command line options from the OPTIONS section. -# -# The SYNOPSIS section should look like, -# (start code) -# =head1 SYNOPSIS -# -# Usage: mk-archiver [OPTION...] --source DSN --where WHERE -# -# mk-archiver nibbles records from a MySQL table. The --source and --dest -# arguments use DSN syntax; if COPY is yes, --dest defaults to the key's value -# from --source. -# -# Examples: -# ... -# (end code) -# The key, required parts are the "Usage:" line and the following description -# paragraph. -# -# The OPTIONS section shoud look like, -# (start code) -# =head1 OPTIONS -# -# Optional rules, one per line. -# -# =over -# -# =item --analyze -# -# type: string -# -# Run ANALYZE TABLE afterwards on L<"--source"> and/or L<"--dest">. -# ect. -# (end code) -# The option's full name is given as the "=item". The next, optional para -# is the option's attributes. And the next, required para is the option's -# description (the first period-terminated sentence). package OptionParser; use strict; @@ -126,25 +36,6 @@ use Getopt::Long; my $POD_link_re = '[LC]<"?([^">]+)"?>'; -# Sub: new -# -# Parameters: -# %args - Arguments -# -# Optional Arguments: -# file - Filename to parse POD stuff from. Several subs take -# a $file param mostly for testing purposes. This arg -# provides a "global" default for even easier testing. -# description - Tool's description (overrides description from SYNOPSIS). -# usage - Tool's usage line (overrides Usage from SYNOPSIS). -# head1 - head1 heading under which options are listed -# skip_rules - Don't read paras before options as rules -# item - Regex pattern to match options after =item -# attributes - Hashref of allowed option attributes -# parse_attributes - Coderef for parsing option attributes -# -# Returns: -# OptionParser object sub new { my ( $class, %args ) = @_; my @required_args = qw(); @@ -156,7 +47,6 @@ sub new { $program_name ||= $PROGRAM_NAME; my $home = $ENV{HOME} || $ENV{HOMEPATH} || $ENV{USERPROFILE} || '.'; - # Default attributes. my %attributes = ( 'type' => 1, 'short form' => 1, @@ -173,10 +63,8 @@ sub new { attributes => \%attributes, # DSN OPTIONS section. Tools parse_attributes => \&_parse_attribs, # don't tinker with these args. - # override the above optional args' default %args, - # private, not configurable args strict => 1, # disabled by a special rule program_name => $program_name, opts => {}, @@ -215,23 +103,12 @@ sub new { return bless $self, $class; } -# Sub: get_specs -# Read and parse options from the OPTIONS section of the POD. This sub -# should be called first, then . <_pod_to_specs()> -# and <_parse_specs()> do most of the work. If the POD has a -# DSN OPTIONS section then a object is created which -# can be accessed with . -# -# Parameters: -# $file - File name to read, __FILE__ if none given sub get_specs { my ( $self, $file ) = @_; $file ||= $self->{file} || __FILE__; my @specs = $self->_pod_to_specs($file); $self->_parse_specs(@specs); - # Check file for DSN OPTIONS section. If present, parse - # it and create a DSNParser obj. open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR"; my $contents = do { local $/ = undef; <$fh> }; close $fh; @@ -286,44 +163,16 @@ sub get_specs { return; } -# Sub: DSNParser -# Return the object automatically created for DSN type opts. -# -# Returns: -# object sub DSNParser { my ( $self ) = @_; return $self->{DSNParser}; }; -# Sub: get_defaults_files -# Return the program's defaults files. -# -# Returns: -# Array of defaults files sub get_defaults_files { my ( $self ) = @_; return @{$self->{default_files}}; } -# Sub: _pod_to_specs() -# Parse basic specs for each option. Each opt spec is a -# hashref like: -# (start code) -# { -# spec => GetOpt::Long specification, -# desc => short description for --help -# group => option group (default: 'default') -# } -# (end code) -# This is step 1 of 2 of parsing the POD opts. The second is -# C<_parse_specs()>. -# -# Parameters: -# $file - File name to read, __FILE__ if none given -# -# Returns: -# Array of opt spec hashrefs to pass to <_parse_specs()>. sub _pod_to_specs { my ( $self, $file ) = @_; $file ||= $self->{file} || __FILE__; @@ -333,15 +182,12 @@ sub _pod_to_specs { my @rules = (); my $para; - # Read a paragraph at a time from the file. Skip everything until options - # are reached... local $INPUT_RECORD_SEPARATOR = ''; while ( $para = <$fh> ) { next unless $para =~ m/^=head1 $self->{head1}/; last; } - # ... then read any option rules... while ( $para = <$fh> ) { last if $para =~ m/^=over/; next if $self->{skip_rules}; @@ -354,7 +200,6 @@ sub _pod_to_specs { die "POD has no $self->{head1} section" unless $para; - # ... then start reading options. do { if ( my ($option) = $para =~ m/^=item $self->{item}/ ) { chomp $para; @@ -380,19 +225,15 @@ sub _pod_to_specs { MKDEBUG && _d('Option has no attributes'); } - # Remove extra spaces and POD formatting (L<"">). $para =~ s/\s+\Z//g; $para =~ s/\s+/ /g; $para =~ s/$POD_link_re/$1/go; - # Take the first period-terminated sentence as the option's short help - # description. $para =~ s/\.(?:\n.*| [A-Z].*|\Z)//s; MKDEBUG && _d('Short help:', $para); die "No description after option spec $option" if $para =~ m/^=item/; - # Change [no]foo to foo and set negatable attrib. See issue 140. if ( my ($base_option) = $option =~ m/^\[no\](.*)/ ) { $option = $base_option; $attribs{'negatable'} = 1; @@ -421,21 +262,6 @@ sub _pod_to_specs { return @specs, @rules; } -# Sub: _parse_specs -# Parse option specs and rules. The opt specs and rules are returned -# by <_pod_to_specs()>. The following attributes are added to each opt spec: -# (start code) -# short => the option's short key (-A for --charset) -# is_cumulative => true if the option is cumulative -# is_negatable => true if the option is negatable -# is_required => true if the option is required -# type => the option's type, one of $self->{types} -# got => true if the option was given explicitly on the cmd line -# value => the option's value -# (end code) -# -# Parameters: -# @specs - Opt specs and rules from <_pod_to_specs()> sub _parse_specs { my ( $self, @specs ) = @_; my %disables; # special rule that requires deferred checking @@ -447,7 +273,6 @@ sub _parse_specs { my ( $long, $short ) = $opt->{spec} =~ m/^([\w-]+)(?:\|([^!+=]*))?/; if ( !$long ) { - # This shouldn't happen. die "Cannot parse long option from spec $opt->{spec}"; } $opt->{long} = $long; @@ -484,37 +309,23 @@ sub _parse_specs { $opt->{type} = $type; MKDEBUG && _d($long, 'type:', $type); - # This check is no longer needed because we'll create a DSNParser - # object for ourself if DSN OPTIONS exists in the POD. - # if ( $type && $type eq 'd' && !$self->{dp} ) { - # die "$opt->{long} is type DSN (d) but no dp argument " - # . "was given when this OptionParser object was created"; - # } - # Option has a non-Getopt type: HhAadzm. Use Getopt type 's'. $opt->{spec} =~ s/=./=s/ if ( $type && $type =~ m/[HhAadzm]/ ); - # Option has a default value if its desc says 'default' or 'default X'. - # These defaults from the POD may be overridden by later calls - # to set_defaults(). if ( (my ($def) = $opt->{desc} =~ m/default\b(?: ([^)]+))?/) ) { $self->{defaults}->{$long} = defined $def ? $def : 1; MKDEBUG && _d($long, 'default:', $def); } - # Handle special behavior for --config. if ( $long eq 'config' ) { $self->{defaults}->{$long} = join(',', $self->get_defaults_files()); } - # Option disable another option if its desc says 'disable'. if ( (my ($dis) = $opt->{desc} =~ m/(disables .*)/) ) { - # Defer checking till later because of possible forward references. $disables{$long} = $dis; MKDEBUG && _d('Deferring check of disables rule for', $opt, $dis); } - # Save the option. $self->{opts}->{$long} = $opt; } else { # It's an option rule, not a spec. @@ -535,8 +346,6 @@ sub _parse_specs { } if ( $opt =~ m/default to/ ) { $rule_ok = 1; - # Example: "DSN values in L<"--dest"> default to values - # from L<"--source">." $self->{defaults_to}->{$participants[0]} = $participants[1]; MKDEBUG && _d($participants[0], 'defaults to', $participants[1]); } @@ -550,9 +359,6 @@ sub _parse_specs { } @groups; } if( $opt =~ m/accepts additional command-line arguments/ ) { - # The full rule text should be: "This tool accepts additional - # command-line arguments. Refer to the synopsis and usage - # information for details." $rule_ok = 1; $self->{strict} = 0; MKDEBUG && _d("Strict mode disabled by rule"); @@ -562,9 +368,7 @@ sub _parse_specs { } } - # Check forward references in 'disables' rules. foreach my $long ( keys %disables ) { - # _get_participants() will check that each opt exists. my @participants = $self->_get_participants($disables{$long}); $self->{disables}->{$long} = \@participants; MKDEBUG && _d('Option', $long, 'disables', @participants); @@ -573,17 +377,6 @@ sub _parse_specs { return; } -# Sub: _get_participants -# Extract option names from a string. This is used to -# find the "participants" of option rules (i.e. the options to -# which a rule applies). -# -# Parameters: -# $str - String containing option names like "Options L<"--[no]foo"> and -# --bar are mutually exclusive." -# -# Returns: -# Array of option names sub _get_participants { my ( $self, $str ) = @_; my @participants; @@ -596,28 +389,18 @@ sub _get_participants { return @participants; } -# Sub: opts -# -# Returns: -# A copy of the internal opts hash sub opts { my ( $self ) = @_; my %opts = %{$self->{opts}}; return %opts; } -# Sub: short_opts -# -# Returns: -# A copy of the internal short_opts hash sub short_opts { my ( $self ) = @_; my %short_opts = %{$self->{short_opts}}; return %short_opts; } -# Sub: set_defaults -# Set default values for options. sub set_defaults { my ( $self, %defaults ) = @_; $self->{defaults} = {}; @@ -640,17 +423,12 @@ sub get_groups { return $self->{groups}; } -# Sub: _set_option -# Getopt::Long calls this sub for each opt it finds on the -# cmd line. We have to do this in order to know which opts -# were "got" on the cmd line. sub _set_option { my ( $self, $opt, $val ) = @_; my $long = exists $self->{opts}->{$opt} ? $opt : exists $self->{short_opts}->{$opt} ? $self->{short_opts}->{$opt} : die "Getopt::Long gave a nonexistent option: $opt"; - # Reassign $opt. $opt = $self->{opts}->{$long}; if ( $opt->{is_cumulative} ) { $opt->{value}++; @@ -662,15 +440,9 @@ sub _set_option { MKDEBUG && _d('Got option', $long, '=', $val); } -# Sub: get_opts -# Get command line options and enforce option rules. -# Option values are saved internally in $self->{opts} and accessed -# later by , , and . Call -# before calling this sub. sub get_opts { my ( $self ) = @_; - # Reset opts. foreach my $long ( keys %{$self->{opts}} ) { $self->{opts}->{$long}->{got} = 0; $self->{opts}->{$long}->{value} @@ -680,10 +452,8 @@ sub get_opts { } $self->{got_opts} = 0; - # Reset errors. $self->{errors} = []; - # --config is special-case; parse them manually and remove them from @ARGV if ( @ARGV && $ARGV[0] eq "--config" ) { shift @ARGV; $self->_set_option('config', shift @ARGV); @@ -691,9 +461,6 @@ sub get_opts { if ( $self->has('config') ) { my @extra_args; foreach my $filename ( split(',', $self->get('config')) ) { - # Try to open the file. If it was set explicitly, it's an error if it - # can't be opened, but the built-in defaults are to be ignored if they - # can't be opened. eval { push @extra_args, $self->_read_config_file($filename); }; @@ -711,7 +478,6 @@ sub get_opts { Getopt::Long::Configure('no_ignore_case', 'bundling'); GetOptions( - # Make Getopt::Long specs for each option with custom handler subs. map { $_->{spec} => sub { $self->_set_option(@_); } } grep { $_->{long} ne 'config' } # --config is handled specially above. values %{$self->{opts}} @@ -731,7 +497,6 @@ sub get_opts { $self->save_error("Unrecognized command-line options @ARGV"); } - # Check mutex options. foreach my $mutex ( @{$self->{mutex}} ) { my @set = grep { $self->{opts}->{$_}->{got} } @$mutex; if ( @set > 1 ) { @@ -758,11 +523,6 @@ sub get_opts { return; } -# Sub: _check_opts -# Check options against rules and group restrictions. -# -# Parameters: -# @long - Array of option names sub _check_opts { my ( $self, @long ) = @_; my $long_last = scalar @long; @@ -772,7 +532,6 @@ sub _check_opts { next unless $long; my $opt = $self->{opts}->{$long}; if ( $opt->{got} ) { - # Rule: opt disables other opts. if ( exists $self->{disables}->{$long} ) { my @disable_opts = @{$self->{disables}->{$long}}; map { $self->{opts}->{$_}->{value} = undef; } @disable_opts; @@ -780,11 +539,7 @@ sub _check_opts { 'because', $long,'disables them'); } - # Group restrictions. if ( exists $self->{allowed_groups}->{$long} ) { - # This option is only allowed with other options from - # certain groups. Check that no options from restricted - # groups were gotten. my @restricted_groups = grep { !exists $self->{allowed_groups}->{$long}->{$_} @@ -841,17 +596,11 @@ sub _check_opts { return; } -# Sub: _validate_type -# Validate special option types like sizes and DSNs. -# -# Parameters: -# $opt - Long option name to validate sub _validate_type { my ( $self, $opt ) = @_; return unless $opt; if ( !$opt->{type} ) { - # Magic opts like --help and --version. $opt->{parsed} = 1; return; } @@ -861,7 +610,6 @@ sub _validate_type { if ( $val && $opt->{type} eq 'm' ) { # type time MKDEBUG && _d('Parsing option', $opt->{long}, 'as a time value'); my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/; - # The suffix defaults to 's' unless otherwise specified. if ( !$suffix ) { my ( $s ) = $opt->{desc} =~ m/\(suffix (.)\)/; $suffix = $s || 's'; @@ -882,9 +630,6 @@ sub _validate_type { } elsif ( $val && $opt->{type} eq 'd' ) { # type DSN MKDEBUG && _d('Parsing option', $opt->{long}, 'as a DSN'); - # DSN vals for this opt may come from 3 places, in order of precedence: - # the opt itself, the defaults to/copies from opt (prev), or - # --host, --port, etc. (defaults). my $prev = {}; my $from_key = $self->{defaults_to}->{ $opt->{long} }; if ( $from_key ) { @@ -920,15 +665,6 @@ sub _validate_type { return; } -# Sub: get -# Get an option's value. The option can be either a -# short or long name (e.g. -A or --charset). -# -# Parameters: -# $opt - Option name, long (--charset) or short (-A) -# -# Returns: -# The option's value sub get { my ( $self, $opt ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); @@ -937,14 +673,6 @@ sub get { return $self->{opts}->{$long}->{value}; } -# Sub: got -# Test if an option was explicitly given on the command line. -# -# Parameters: -# $opt - Option name, long (--charset) or short (-A) -# -# Returns: -# Bool sub got { my ( $self, $opt ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); @@ -953,27 +681,12 @@ sub got { return $self->{opts}->{$long}->{got}; } -# Sub: has -# Test if an option exists (i.e. is specified in the tool's POD). -# -# Parameters: -# $opt - Option name, long (--charset) or short (-A) -# -# Returns: -# Bool sub has { my ( $self, $opt ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); return defined $long ? exists $self->{opts}->{$long} : 0; } -# Sub: set -# Set an option's value. No type checking is done so be careful to -# not set, for example, an integer option with a DSN. -# -# Parameters: -# $opt - Option name, long (--charset) or short (-A) -# $val - Option's new value sub set { my ( $self, $opt, $val ) = @_; my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt); @@ -983,19 +696,12 @@ sub set { return; } -# Sub: save_error -# Save an error message to be reported later by . -# -# Parameters: -# $error - Error message sub save_error { my ( $self, $error ) = @_; push @{$self->{errors}}, $error; return; } -# Sub: errors -# Used for testing. sub errors { my ( $self ) = @_; return $self->{errors}; @@ -1014,8 +720,6 @@ sub descr { . " For more details, please use the --help option, " . "or try 'perldoc $PROGRAM_NAME' " . "for complete documentation."; - # DONT_BREAK_LINES is set in OptionParser.t so the output can - # be tested reliably. $descr = join("\n", $descr =~ m/(.{0,80})(?:\s+|$)/g) unless $ENV{DONT_BREAK_LINES}; $descr =~ s/ +$//mg; @@ -1026,8 +730,6 @@ sub usage_or_errors { my ( $self, $file, $return ) = @_; $file ||= $self->{file} || __FILE__; - # First make sure we have a description and usage, else print_usage() - # and print_errors() will die. if ( !$self->{description} || !$self->{usage} ) { MKDEBUG && _d("Getting description and usage from SYNOPSIS in", $file); my %synop = $self->_parse_synopsis($file); @@ -1049,8 +751,6 @@ sub usage_or_errors { return; } -# Explains what errors were found while processing command-line arguments and -# gives a brief overview so you can get more information. sub print_errors { my ( $self ) = @_; my $usage = $self->usage() . "\n"; @@ -1061,19 +761,11 @@ sub print_errors { return $usage . "\n" . $self->descr(); } -# Prints out command-line help. The format is like this: -# --foo=s -F Description of --foo -# --bars -B Description of --bar -# --longopt Description of --longopt -# Note that the short options are aligned along the right edge of their longest -# long option, but long options that don't have a short option are allowed to -# protrude past that. sub print_usage { my ( $self ) = @_; die "Run get_opts() before print_usage()" unless $self->{got_opts}; my @opts = values %{$self->{opts}}; - # Find how wide the widest long option is. my $maxl = max( map { length($_->{long}) # option long name @@ -1082,7 +774,6 @@ sub print_usage { } @opts); - # Find how wide the widest option with a short option is. my $maxs = max(0, map { length($_) @@ -1091,19 +782,14 @@ sub print_usage { } values %{$self->{short_opts}}); - # Find how wide the 'left column' (long + short opts) is, and therefore how - # much space to give options and how much to give descriptions. my $lcol = max($maxl, ($maxs + 3)); my $rcol = 80 - $lcol - 6; my $rpad = ' ' x ( 80 - $rcol ); - # Adjust the width of the options that have long and short both. $maxs = max($lcol - 3, $maxs); - # Format and return the options. my $usage = $self->descr() . "\n" . $self->usage(); - # Sort groups alphabetically but make 'default' first. my @groups = reverse sort grep { $_ ne 'default'; } keys %{$self->{groups}}; push @groups, 'default'; @@ -1118,11 +804,8 @@ sub print_usage { my $short = $opt->{short}; my $desc = $opt->{desc}; - # Append option type to long option name. - # http://code.google.com/p/maatkit/issues/detail?id=1177 $long .= $opt->{type} ? "=$opt->{type}" : ""; - # Expand suffix help for time options. if ( $opt->{type} && $opt->{type} eq 'm' ) { my ($s) = $desc =~ m/\(suffix (.)\)/; $s ||= 's'; @@ -1130,7 +813,6 @@ sub print_usage { $desc .= ". Optional suffix s=seconds, m=minutes, h=hours, " . "d=days; if no suffix, $s is used."; } - # Wrap long descriptions $desc = join("\n$rpad", grep { $_ } $desc =~ m/(.{0,$rcol})(?:\s+|$)/g); $desc =~ s/ +$//mg; if ( $short ) { @@ -1167,9 +849,6 @@ sub print_usage { return $usage; } -# Tries to prompt and read the answer without echoing the answer to the -# terminal. This isn't really related to this package, but it's too handy not -# to put here. OK, it's related, it gets config information from the user. sub prompt_noecho { shift @_ if ref $_[0] eq __PACKAGE__; my ( $prompt ) = @_; @@ -1191,8 +870,6 @@ sub prompt_noecho { return $response; } -# Reads a configuration file and returns it as a list. Inspired by -# Config::Tiny. sub _read_config_file { my ( $self, $filename ) = @_; open my $fh, "<", $filename or die "Cannot open $filename: $OS_ERROR\n"; @@ -1203,14 +880,9 @@ sub _read_config_file { LINE: while ( my $line = <$fh> ) { chomp $line; - # Skip comments and empty lines next LINE if $line =~ m/^\s*(?:\#|\;|$)/; - # Remove inline comments $line =~ s/\s+#.*$//g; - # Remove whitespace $line =~ s/^\s+|\s+$//g; - # Watch for the beginning of the literal values (not to be interpreted as - # options) if ( $line eq '--' ) { $prefix = ''; $parse = 0; @@ -1232,26 +904,6 @@ sub _read_config_file { return @args; } -# Sub: read_para_after -# Read the POD paragraph after a magical regex. This is used, -# for exmaple, to get default CREATE TABLE from the POD. We write something -# like: -# (start code) -# This is the default MAGIC_foo_table: -# -# CREATE TABLE `foo` (i INT) -# -# Blah blah... -# (end code) -# Then to get that CREATE TABLE, you pass "MAGIC_foo_table" as the -# magical regex. -# -# Parameters: -# $file - File to read -# $regex - Regex to find something magical before the desired POD paragraph -# -# Returns: -# POD paragraph after magical regex sub read_para_after { my ( $self, $file, $regex ) = @_; open my $fh, "<", $file or die "Can't open $file: $OS_ERROR"; @@ -1271,13 +923,9 @@ sub read_para_after { return $para; } -# Returns a lightweight clone of ourself. Currently, only the basic -# opts are copied. This is used for stuff like "final opts" in -# mk-table-checksum. sub clone { my ( $self ) = @_; - # Deep-copy contents of hashrefs; do not just copy the refs. my %clone = map { my $hashref = $self->{$_}; my $val_copy = {}; @@ -1291,7 +939,6 @@ sub clone { $_ => $val_copy; } qw(opts short_opts defaults); - # Re-assign scalar values. foreach my $scalar ( qw(got_opts) ) { $clone{$scalar} = $self->{$scalar}; } @@ -1302,7 +949,6 @@ sub clone { sub _parse_size { my ( $self, $opt, $val ) = @_; - # Special case used by mk-find to do things like --datasize null. if ( lc($val || '') eq 'null' ) { MKDEBUG && _d('NULL size for', $opt->{long}); $opt->{value} = 'null'; @@ -1325,8 +971,6 @@ sub _parse_size { return; } -# Parse the option's attributes and return a GetOpt type. -# E.g. "foo type:int" == "foo=i"; "[no]bar" == "bar!", etc. sub _parse_attribs { my ( $self, $option, $attribs ) = @_; my $types = $self->{types}; @@ -1342,7 +986,6 @@ sub _parse_synopsis { $file ||= $self->{file} || __FILE__; MKDEBUG && _d("Parsing SYNOPSIS in", $file); - # Slurp the file. local $INPUT_RECORD_SEPARATOR = ''; # read paragraphs open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR"; my $para; @@ -1359,11 +1002,9 @@ sub _parse_synopsis { die "The SYNOPSIS section in $file is not formatted properly" unless $usage && $desc; - # Strip "Usage:" from the usage string. $usage =~ s/^\s*Usage:\s+(.+)/$1/; chomp $usage; - # Make the description one long string without newlines. $desc =~ s/\n/ /g; $desc =~ s/\s{2,}/ /g; $desc =~ s/\. ([A-Z][a-z])/. $1/g; @@ -1383,9 +1024,6 @@ sub _d { print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; } -# This is debug code I want to run for all tools, and this is a module I -# certainly include in all tools, but otherwise there's no real reason to put -# it here. if ( MKDEBUG ) { print '# ', $^X, ' ', $], "\n"; if ( my $uname = `uname -a` ) { @@ -1402,22 +1040,6 @@ if ( MKDEBUG ) { # End OptionParser package # ########################################################################### -# This program is copyright 2010-2011 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### # ReadKeyMini # ########################################################################### @@ -1443,7 +1065,6 @@ use strict; use English qw(-no_match_vars); use constant MKDEBUG => $ENV{MKDEBUG} || 0; -use Carp qw( croak ); use POSIX qw( :termios_h ); use base qw( Exporter ); @@ -1456,13 +1077,15 @@ BEGIN { Term::ReadKey->import(@EXPORT_OK); } else { + # If we don't have Term::ReadKey, fake it. We clobber our own glob, + # ReadKeyMini::Function, and the Term::ReadKey glob, so callers can + # both import it if requested, or even use the fully-qualified name + # without issues. *ReadMode = *Term::ReadKey::ReadMode = \&_ReadMode; *GetTerminalSize = *Term::ReadKey::GetTerminalSize = \&_GetTerminalSize; } } -our $VERSION = '0.01'; - my %modes = ( original => 0, restore => 0, @@ -1480,21 +1103,20 @@ my %modes = ( my $fd_stdin = fileno(STDIN); my $term = POSIX::Termios->new(); $term->getattr($fd_stdin); - my $oterm = $term->getlflag(); - - my $echo = ECHO | ECHOK | ICANON; - my $noecho = $oterm & ~$echo; + my $oterm = $term->getlflag(); + my $echo = ECHO | ECHOK | ICANON; + my $noecho = $oterm & ~$echo; sub _ReadMode { my $mode = $modes{ $_[0] }; if ( $mode == $modes{normal} ) { - cooked(); + cooked(); } elsif ( $mode == $modes{cbreak} || $mode == $modes{noecho} ) { - cbreak( $mode == $modes{noecho} ? $noecho : $oterm ); + cbreak( $mode == $modes{noecho} ? $noecho : $oterm ); } else { - croak("ReadMore('$_[0]') not supported"); + die("ReadMore('$_[0]') not supported"); } } @@ -1516,29 +1138,31 @@ my %modes = ( } sub readkey { - my $key = ''; - cbreak(); - sysread(STDIN, $key, 1); - my $timeout = 0.1; - if ( $key eq "\033" ) { # Ugly and broken hack, but good enough for the two minutes it took to write. - { - my $x = ''; - STDIN->blocking(0); - sysread(STDIN, $x, 2); - STDIN->blocking(1); - $key .= $x; - redo if $key =~ /\[[0-2](?:[0-9];)?$/ - } - } - cooked(); - return $key; + my $key = ''; + cbreak(); + sysread(STDIN, $key, 1); + my $timeout = 0.1; + if ( $key eq "\033" ) { # Ugly and broken hack, but good enough for the two minutes it took to write. + # Namely, Ctrl escapes, the F keys, and other stuff you can send from the keyboard + # take more than one "character" to represent, and wrong be wrong to break into pieces. + { + my $x = ''; + STDIN->blocking(0); + sysread(STDIN, $x, 2); + STDIN->blocking(1); + $key .= $x; + redo if $key =~ /\[[0-2](?:[0-9];)?$/ + } + } + cooked(); + return $key; } # As per perlfaq8: sub _GetTerminalSize { if ( @_ ) { - croak "My::Term::ReadKey doesn't implement GetTerminalSize with arguments"; + die "My::Term::ReadKey doesn't implement GetTerminalSize with arguments"; } eval { require 'sys/ioctl.ph' }; if ( !defined &TIOCGWINSZ ) { @@ -1551,10 +1175,10 @@ sub _GetTerminalSize { : 0x40087468; }; } - open( TTY, "+<", "/dev/tty" ) or croak "No tty: $OS_ERROR"; + open( TTY, "+<", "/dev/tty" ) or die "No tty: $OS_ERROR"; my $winsize = ''; unless ( ioctl( TTY, &TIOCGWINSZ, $winsize ) ) { - croak sprintf "$0: ioctl TIOCGWINSZ (%08x: $OS_ERROR)\n", &TIOCGWINSZ; + die sprintf "$0: ioctl TIOCGWINSZ (%08x: $OS_ERROR)\n", &TIOCGWINSZ; } my ( $row, $col, $xpixel, $ypixel ) = unpack( 'S4', $winsize ); return ( $col, $row, $xpixel, $ypixel ); @@ -1567,29 +1191,15 @@ sub _GetTerminalSize { # End ReadKeyMini package # ########################################################################### -# This program is copyright 2011 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### # Diskstats package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/Diskstats.pm +# t/lib/Diskstats.t +# See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { -# Package: Diskstats -# This package implements most of the logic in the old shell pt-diskstats; -# it parses data from /proc/diskstats, calculcates deltas, and prints those. package Diskstats; @@ -1601,61 +1211,52 @@ use constant MKDEBUG => $ENV{MKDEBUG} || 0; use IO::Handle; use List::Util qw( max first ); -BEGIN { - # This BEGIN block checks if we can use Storable::dclone: If we can't, - # it clobbers this package's dclone glob (*{ __PACKAGE__ . "::dclone" }) - # with an anonymous function that provides more or less what we need. - my $have_storable = eval { require Storable }; - - if ( $have_storable ) { - Storable->import(qw(dclone)); - } - else { - require Scalar::Util; - - # An extrenely poor man's dclone. - # Nevermind the prototype. dclone has it, so it's here only it for - # the sake of completeness. - *dclone = sub ($) { - my ($ref) = @_; - my $reftype = Scalar::Util::reftype($ref) || ''; - - if ( $reftype eq ref({}) ) { - # Only one level of depth. Not worth making it any deeper/recursive, I think. - return { map { $_ => {%{$ref->{$_}}} } keys %$ref }; - } - else { - die "This basic dclone does not support [$reftype]"; - } - }; - } -} - sub new { my ( $class, %args ) = @_; + my @required_args = qw(OptionParser); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($o) = @args{@required_args}; + my $self = { - # Defaults filename => '/proc/diskstats', column_regex => qr/cnc|rt|busy|prg|time|io_s/, device_regex => qr/(?=)/, block_size => 512, out_fh => \*STDOUT, - filter_zeroed_rows => 0, - sample_time => 0, + filter_zeroed_rows => $o->get('zero-rows') ? undef : 1, + sample_time => $o->get('sample-time') || 0, interactive => 0, _stats_for => {}, - _sorted_devs => [], + _ordered_devs => [], _ts => {}, _first => 1, - # Internal for now, but might need APIfying. _save_curr_as_prev => 1, _print_header => 1, }; - # If they passed us an attribute explicitly, we use those. + if ( $o->get('memory-for-speed') ) { + eval { + require Memoize; Memoize::memoize('_parse_diskstats_line') + }; + if ($EVAL_ERROR) { + warn "Can't trade memory for speed: $EVAL_ERROR. Continuing as usual."; + } + } + + my %pod_to_attribute = ( + columns => 'column_regex', + devices => 'device_regex' + ); + for my $key ( grep { defined $o->get($_) } keys %pod_to_attribute ) { + my $re = $o->get($key) || '(?=)'; + $self->{ $pod_to_attribute{$key} } = qr/$re/i; + } + for my $attribute ( grep { !/^_/ && defined $args{$_} } keys %$self ) { $self->{$attribute} = $args{$attribute}; } @@ -1663,7 +1264,6 @@ sub new { return bless $self, $class; } -# The next lot are accessors, plus some convenience functions. sub _ts_common { my ($self, $key, $val) = @_; @@ -1673,14 +1273,14 @@ sub _ts_common { return $self->{_ts}->{$key}; } -sub current_ts { +sub curr_ts { my ($self, $val) = @_; - return $self->_ts_common("current", $val); + return $self->_ts_common("curr", $val); } -sub previous_ts { +sub prev_ts { my ($self, $val) = @_; - return $self->_ts_common("previous", $val); + return $self->_ts_common("prev", $val); } sub first_ts { @@ -1712,15 +1312,10 @@ sub interactive { return $self->{interactive}; } -# What this method does is thee-fold: -# It sets or returns the currently set filehandle, kind of like a poor man's -# select(); but also, it checks whenever said filehandle is open. If it's not, -# it defaults to STDOUT. sub out_fh { my ( $self, $new_fh ) = @_; - # ->opened comes from IO::Handle. if ( $new_fh && ref($new_fh) && $new_fh->opened ) { $self->{out_fh} = $new_fh; } @@ -1748,10 +1343,10 @@ sub device_regex { sub filename { my ( $self, $new_filename ) = @_; - if ( defined $new_filename ) { + if ( $new_filename ) { return $self->{filename} = $new_filename; } - return $self->{filename} || '/proc/diskstats'; + return $self->{filename}; } sub block_size { @@ -1759,33 +1354,33 @@ sub block_size { return $self->{block_size}; } -sub sorted_devs { - my ( $self, $new_dev ) = @_; - if ( $new_dev && ref($new_dev) eq ref( [] ) ) { - $self->{_sorted_devs} = $new_dev; + +sub ordered_devs { + my ( $self, $replacement_list ) = @_; + if ( $replacement_list ) { + $self->{_ordered_devs} = $replacement_list; } - return @{ $self->{_sorted_devs} }; + return @{ $self->{_ordered_devs} }; } -sub add_sorted_devs { +sub add_ordered_dev { my ( $self, $new_dev ) = @_; if ( !$self->{_seen_devs}->{$new_dev}++ ) { - push @{ $self->{_sorted_devs} }, $new_dev; + push @{ $self->{_ordered_devs} }, $new_dev; } + return; } -# clear_stuff methods. Like the name says, they clear state stored inside -# the object. sub clear_state { my ($self) = @_; $self->{_first} = 1; $self->{_print_header} = 1; - $self->clear_current_stats(); - $self->clear_previous_stats(); + $self->clear_curr_stats(); + $self->clear_prev_stats(); $self->clear_first_stats(); $self->clear_ts(); - $self->clear_sorted_devs(); + $self->clear_ordered_devs(); } sub clear_ts { @@ -1793,10 +1388,10 @@ sub clear_ts { $self->{_ts} = {}; } -sub clear_sorted_devs { +sub clear_ordered_devs { my $self = shift; $self->{_seen_devs} = {}; - $self->sorted_devs( [] ); + $self->ordered_devs( [] ); } sub _clear_stats_common { @@ -1811,14 +1406,14 @@ sub _clear_stats_common { } } -sub clear_current_stats { +sub clear_curr_stats { my ( $self, @args ) = @_; $self->_clear_stats_common( "_stats_for", @args ); } -sub clear_previous_stats { +sub clear_prev_stats { my ( $self, @args ) = @_; - $self->_clear_stats_common( "_previous_stats_for", @args ); + $self->_clear_stats_common( "_prev_stats_for", @args ); } sub clear_first_stats { @@ -1840,9 +1435,9 @@ sub stats_for { $self->_stats_for_common( $dev, '_stats_for' ); } -sub previous_stats_for { +sub prev_stats_for { my ( $self, $dev ) = @_; - $self->_stats_for_common( $dev, '_previous_stats_for' ); + $self->_stats_for_common( $dev, '_prev_stats_for' ); } sub first_stats_for { @@ -1852,39 +1447,45 @@ sub first_stats_for { sub has_stats { my ($self) = @_; + my $stats = $self->stats_for; - return $self->stats_for - && scalar grep 1, @{ $self->stats_for }{ $self->sorted_devs }; -} - -sub _save_current_as_previous { - my ( $self, $curr_hashref ) = @_; - - if ( $self->{_save_curr_as_prev} ) { - $self->{_previous_stats_for} = $curr_hashref; - for my $dev (keys %$curr_hashref) { - $self->{_previous_stats_for}->{$dev}->{sum_ios_in_progress} += - $curr_hashref->{$dev}->{ios_in_progress}; - } - $self->previous_ts($self->current_ts()); + for my $key ( keys %$stats ) { + return 1 if $stats->{$key} && %{ $stats->{$key} } } return; } -sub _save_current_as_first { - my ($self, $curr_hashref) = @_; +sub _save_curr_as_prev { + my ( $self, $curr ) = @_; + + if ( $self->{_save_curr_as_prev} ) { + $self->{_prev_stats_for} = $curr; + for my $dev (keys %$curr) { + $self->{_prev_stats_for}->{$dev}->{sum_ios_in_progress} += + $curr->{$dev}->{ios_in_progress}; + } + $self->prev_ts($self->curr_ts()); + } + + return; +} + +sub _save_curr_as_first { + my ($self, $curr) = @_; if ( $self->{_first} ) { - $self->{_first_stats_for} = $curr_hashref; - $self->first_ts($self->current_ts()); + $self->{_first_stats_for} = { + map { $_ => {%{$curr->{$_}}} } keys %$curr + }; + $self->first_ts($self->curr_ts()); $self->{_first} = undef; } } sub _save_stats { - my ( $self, $hashref ) = @_; - $self->{_stats_for} = $hashref; + my ( $self, $stats ) = @_; + return $self->{_stats_for} = $stats; } sub trim { @@ -1896,18 +1497,17 @@ sub trim { sub col_ok { my ( $self, $column ) = @_; - my $regex = $self->column_regex; - return $column =~ $regex || trim($column) =~ $regex; + my $regex = $self->column_regex(); + return ($column =~ $regex) || (trim($column) =~ $regex); } sub dev_ok { my ( $self, $device ) = @_; - my $regex = $self->device_regex; + my $regex = $self->device_regex(); return $device =~ $regex; } my @columns_in_order = ( - # Column # Format # Key name [ " rd_s" => "%7.1f", "reads_sec", ], [ "rd_avkb" => "%7.1f", "avg_read_sz", ], [ "rd_mb_s" => "%7.1f", "mbytes_read_sec", ], @@ -1951,27 +1551,13 @@ my @columns_in_order = ( } -# Method: design_print_formats() -# What says on the label. Returns three things: the format for the header and the -# data, and an arrayref of the columns used to make it. -# -# Parameters: -# %args - Arguments -# -# Optional Arguments: -# columns - An arrayref with column names. If absent, uses ->col_ok to -# decide which columns to use. -# max_device_length - How much space to leave for device names. Defaults at 6. -# sub design_print_formats { my ( $self, %args ) = @_; my ( $dev_length, $columns ) = @args{qw( max_device_length columns )}; - $dev_length ||= max 6, map length, $self->sorted_devs; + $dev_length ||= max 6, map length, $self->ordered_devs; my ( $header, $format ); - # For each device, print out the following: The timestamp offset and - # device name. $header = $format = qq{%5s %-${dev_length}s }; if ( !$columns ) { @@ -1987,47 +1573,40 @@ sub design_print_formats { return ( $header, $format, $columns ); } -sub parse_diskstats_line { - my ( $self, $line, $block_size ) = @_; - my @keys = qw( - reads reads_merged read_sectors ms_spent_reading - writes writes_merged written_sectors ms_spent_writing - ios_in_progress ms_spent_doing_io ms_weighted - ); - my ( $dev, %dev_stats ); +{ +my @diskstats_fields = qw( + reads reads_merged read_sectors ms_spent_reading + writes writes_merged written_sectors ms_spent_writing + ios_in_progress ms_spent_doing_io ms_weighted +); - if ( ( @dev_stats{qw( major minor )}, $dev, @dev_stats{@keys} ) = - $line =~ /^ - # Disk format - \s* (\d+) # major - \s+ (\d+) # minor - \s+ (.+?) # Device name - \s+ (\d+) # # of reads issued - \s+ (\d+) # # of reads merged - \s+ (\d+) # # of sectors read - \s+ (\d+) # # of milliseconds spent reading - \s+ (\d+) # # of writes completed - \s+ (\d+) # # of writes merged - \s+ (\d+) # # of sectors written - \s+ (\d+) # # of milliseconds spent writing - \s+ (\d+) # # of IOs currently in progress - \s+ (\d+) # # of milliseconds spent doing IOs - \s+ (\d+) # weighted # of milliseconds spent doing IOs - \s*$/x - ) +sub parse_diskstats_line { shift; goto &_parse_diskstats_line } +sub _parse_diskstats_line { + my ( $line, $block_size ) = @_; + my $dev; + keys my %dev_stats = 30; # Pre-expand the amount of buckets for this hash. + + + if ( 14 == (( @dev_stats{qw( major minor )}, $dev, @dev_stats{@diskstats_fields} ) = + split " ", $line, 14 ) ) { - $dev_stats{read_bytes} = $dev_stats{read_sectors} * $block_size; - $dev_stats{written_bytes} = - $dev_stats{written_sectors} * $block_size; - $dev_stats{read_kbs} = $dev_stats{read_bytes} / 1024; - $dev_stats{written_kbs} = $dev_stats{written_bytes} / 1024; - $dev_stats{ios_requested} += $dev_stats{reads} + $dev_stats{writes}; - $dev_stats{ios_in_bytes} += $dev_stats{read_bytes} + $dev_stats{written_bytes}; + $dev_stats{read_kbs} = + ( $dev_stats{read_bytes} = $dev_stats{read_sectors} + * $block_size ) / 1024; + $dev_stats{written_kbs} = + ( $dev_stats{written_bytes} = $dev_stats{written_sectors} + * $block_size ) / 1024; + $dev_stats{ios_requested} = $dev_stats{reads} + + $dev_stats{writes}; + + $dev_stats{ios_in_bytes} = $dev_stats{read_bytes} + + $dev_stats{written_bytes}; return ( $dev, \%dev_stats ); } - elsif ((@dev_stats{qw( major minor )}, $dev, @dev_stats{ qw( reads read_sectors writes written_sectors ) }) = $line =~ /^ - # Partition format + elsif ((@dev_stats{qw( major minor )}, $dev, + @dev_stats{ qw( reads read_sectors writes written_sectors ) }) = + $line =~ /^ \s* (\d+) # major \s+ (\d+) # minor \s+ (.+?) # Device name @@ -2035,18 +1614,19 @@ sub parse_diskstats_line { \s+ (\d+) # # of sectors read \s+ (\d+) # # of writes issued \s+ (\d+) # # of sectors written - \s*$/x) { - for my $key ( @keys ) { + \s*$/x) + { + for my $key ( @diskstats_fields ) { $dev_stats{$key} ||= 0; } - # Copypaste from above, abstract? $dev_stats{read_bytes} = $dev_stats{read_sectors} * $block_size; $dev_stats{written_bytes} = $dev_stats{written_sectors} * $block_size; $dev_stats{read_kbs} = $dev_stats{read_bytes} / 1024; $dev_stats{written_kbs} = $dev_stats{written_bytes} / 1024; - $dev_stats{ttreq} += $dev_stats{reads} + $dev_stats{writes}; - $dev_stats{ttbyt} += $dev_stats{read_bytes} + $dev_stats{written_bytes}; + $dev_stats{ios_requested} = $dev_stats{reads} + $dev_stats{writes}; + $dev_stats{ios_in_bytes} = $dev_stats{read_bytes} + + $dev_stats{written_bytes}; return ( $dev, \%dev_stats ); } @@ -2054,93 +1634,70 @@ sub parse_diskstats_line { return; } } +} -# Method: parse_from() -# Parses data from one of the sources. -# -# Parameters: -# %args - Arguments -# -# Optional Arguments: -# filehandle - Reads data from a filehandle by calling readline() on it. -# data - Reads data one line at a time. -# filename - Opens a filehandle to the file and reads it one line at a time. -# sample_callback - Called each time a sample is processed, passed the latest timestamp. -# sub parse_from { - my ( $self, %args ) = @_; + my ( $self, %args ) = @_; - my $lines_read = $args{filehandle} - ? $self->parse_from_filehandle( @args{qw( filehandle sample_callback )} ) : - $args{data} - ? $self->parse_from_data( @args{qw( data sample_callback )} ) : - $self->parse_from_filename( @args{qw( filename sample_callback )} ); - return $lines_read; + my $lines_read = $args{filehandle} + ? $self->parse_from_filehandle( @args{qw( filehandle sample_callback )} ) + : $args{data} + ? $self->parse_from_data( @args{qw( data sample_callback )} ) + : $self->parse_from_filename( @args{qw( filename sample_callback )} ); + return $lines_read; } + sub parse_from_filename { my ( $self, $filename, $sample_callback ) = @_; - $filename ||= $self->filename; + $filename ||= $self->filename(); open my $fh, "<", $filename - or die "Couldn't open ", $filename, ": $OS_ERROR"; + or die "Cannot parse $filename: $OS_ERROR"; my $lines_read = $self->parse_from_filehandle( $fh, $sample_callback ); - close($fh) or die "Couldn't close: $OS_ERROR"; + close $fh or die "Cannot close: $OS_ERROR"; return $lines_read; } -# Method: parse_from_filehandle() -# Parses data received from using readline() on the filehandle. This is -# particularly useful, as you could pass in a filehandle to a pipe, or -# a tied filehandle, or a PerlIO::Scalar handle. Or your normal -# run of the mill filehandle. -# -# Parameters: -# filehandle - -# sample_callback - Called each time a sample is processed, passed the latest timestamp. -# sub parse_from_filehandle { my ( $self, $filehandle, $sample_callback ) = @_; - return $self->_load( $filehandle, $sample_callback );; + return $self->_load( $filehandle, $sample_callback ); } sub parse_from_data { my ( $self, $data, $sample_callback ) = @_; - open my $fh, "<", \$data - or die "Couldn't open scalar as filehandle: $OS_ERROR"; + open( my $fh, "<", ref($data) ? $data : \$data ) + or die "Couldn't parse data: $OS_ERROR"; my $lines_read = $self->parse_from_filehandle( $fh, $sample_callback ); - close($fh); + close $fh or die ""; return $lines_read; } -# Method: INTERNAL: _load() -# Reads from the filehandle, either saving the data as needed if dealing -# with a diskstats-formatted line, or if it finds a TS line and has a -# callback, defering to that. sub _load { my ( $self, $fh, $sample_callback ) = @_; - my $block_size = $self->block_size; + my $block_size = $self->block_size(); my $current_ts = 0; my $new_cur = {}; while ( my $line = <$fh> ) { - if ( my ( $dev, $dev_stats ) = $self->parse_diskstats_line($line, $block_size) ) { + if ( my ( $dev, $dev_stats ) = $self->parse_diskstats_line($line, $block_size) ) + { $new_cur->{$dev} = $dev_stats; - $self->add_sorted_devs($dev); + $self->add_ordered_dev($dev); } elsif ( my ($new_ts) = $line =~ /TS\s+([0-9]+(?:\.[0-9]+)?)/ ) { - if ( $current_ts && %{$new_cur} ) { - $self->_save_current_as_previous( $self->stats_for() ); + if ( $current_ts && %$new_cur ) { + $self->_save_curr_as_prev( $self->stats_for() ); $self->_save_stats($new_cur); - $self->current_ts($current_ts); - $self->_save_current_as_first( dclone($self->stats_for) ); + $self->curr_ts($current_ts); + $self->_save_curr_as_first( $new_cur ); $new_cur = {}; } if ($sample_callback) { @@ -2150,28 +1707,33 @@ sub _load { } else { chomp($line); - die "Line [$line] isn't in the diskstats format"; + warn "Line $INPUT_LINE_NUMBER: [$line] isn't in the diskstats format"; } } - if ( eof $fh && $current_ts ) { + if ( $current_ts ) { if ( %{$new_cur} ) { - $self->_save_current_as_previous( $self->stats_for() ); + $self->_save_curr_as_prev( $self->stats_for() ); $self->_save_stats($new_cur); - $self->current_ts($current_ts); - $self->_save_current_as_first( dclone($self->stats_for) ); + $self->curr_ts($current_ts); + $self->_save_curr_as_first( $new_cur ); $new_cur = {}; } if ($sample_callback) { $self->$sample_callback($current_ts); } } - # Seems like this could be useful. return $INPUT_LINE_NUMBER; } sub _calc_read_stats { - my ( $self, $delta_for, $elapsed, $devs_in_group ) = @_; + my ( $self, %args ) = @_; + + my @required_args = qw( delta_for elapsed devs_in_group ); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($delta_for, $elapsed, $devs_in_group) = @args{ @required_args }; my %read_stats = ( reads_sec => $delta_for->{reads} / $elapsed, @@ -2202,7 +1764,13 @@ sub _calc_read_stats { } sub _calc_write_stats { - my ( $self, $delta_for, $elapsed, $devs_in_group ) = @_; + my ( $self, %args ) = @_; + + my @required_args = qw( delta_for elapsed devs_in_group ); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($delta_for, $elapsed, $devs_in_group) = @args{ @required_args }; my %write_stats = ( writes_sec => $delta_for->{writes} / $elapsed, @@ -2234,13 +1802,15 @@ sub _calc_write_stats { } -# Compute the numbers for reads and writes together, the things for -# which we do not have separate statistics. -# Busy is what iostat calls %util. This is the percent of -# wall-clock time during which the device has I/O happening. sub _calc_misc_stats { - my ( $self, $delta_for, $elapsed, $devs_in_group, $stats ) = @_; + my ( $self, %args ) = @_; + + my @required_args = qw( delta_for elapsed devs_in_group stats ); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($delta_for, $elapsed, $devs_in_group, $stats) = @args{ @required_args }; my %extra_stats; $extra_stats{busy} = @@ -2248,62 +1818,84 @@ sub _calc_misc_stats { $delta_for->{ms_spent_doing_io} / ( 1000 * $elapsed * $devs_in_group ); - my $number_of_ios = $stats->{ios_requested}; - my $total_ms_spent_on_io = $delta_for->{ms_spent_reading} + $delta_for->{ms_spent_writing}; + my $number_of_ios = $stats->{ios_requested}; + my $total_ms_spent_on_io = $delta_for->{ms_spent_reading} + + $delta_for->{ms_spent_writing}; - $extra_stats{qtime} = $number_of_ios ? $total_ms_spent_on_io / $number_of_ios : 0; - $extra_stats{stime} = $number_of_ios ? $delta_for->{ms_spent_doing_io} / $number_of_ios : 0; + if ( $number_of_ios ) { + $extra_stats{qtime} = $total_ms_spent_on_io / $number_of_ios; + $extra_stats{stime} = $delta_for->{ms_spent_doing_io} / $number_of_ios; + } + else { + $extra_stats{qtime} = 0; + $extra_stats{stime} = 0; + } $extra_stats{s_spent_doing_io} = $total_ms_spent_on_io / 1000; $extra_stats{line_ts} = $self->compute_line_ts( first_ts => $self->first_ts(), - current_ts => $self->current_ts(), + curr_ts => $self->curr_ts(), ); return %extra_stats; } sub _calc_delta_for { - my ( $self, $current, $against ) = @_; - return { - map { ( $_ => $current->{$_} - $against->{$_} ) } + my ( $self, $curr, $against ) = @_; + my %deltas = ( + map { ( $_ => ($curr->{$_} || 0) - ($against->{$_} || 0) ) } qw( reads reads_merged read_sectors ms_spent_reading writes writes_merged written_sectors ms_spent_writing read_kbs written_kbs ms_spent_doing_io ms_weighted ) - }; + ); + return \%deltas; } sub _calc_stats_for_deltas { my ( $self, $elapsed ) = @_; my @end_stats; + my @devices = $self->ordered_devs(); - for my $dev ( grep { $self->dev_ok($_) && $self->stats_for($_) } $self->sorted_devs ) { - my $curr = $self->stats_for($dev); - my $against = $self->delta_against($dev); + my $devs_in_group = $self->compute_devs_in_group(); - my $delta_for = $self->_calc_delta_for( $curr, $against ); + foreach my $dev ( + grep { $self->dev_ok($_) && $self->stats_for($_) } + @devices ) + { + my $curr = $self->stats_for($dev); + my $against = $self->delta_against($dev); - my $in_progress = $curr->{"ios_in_progress"}; + my $delta_for = $self->_calc_delta_for( $curr, $against ); + my $in_progress = $curr->{"ios_in_progress"}; my $tot_in_progress = $against->{"sum_ios_in_progress"} || 0; - my $devs_in_group = $self->compute_devs_in_group; - - # Compute the per-second stats for reads, writes, and overall. my %stats = ( - $self->_calc_read_stats( $delta_for, $elapsed, $devs_in_group ), - $self->_calc_write_stats( $delta_for, $elapsed, $devs_in_group ), + $self->_calc_read_stats( + delta_for => $delta_for, + elapsed => $elapsed, + devs_in_group => $devs_in_group, + ), + $self->_calc_write_stats( + delta_for => $delta_for, + elapsed => $elapsed, + devs_in_group => $devs_in_group, + ), in_progress => $self->compute_in_progress( $in_progress, $tot_in_progress ), ); - my %extras = $self->_calc_misc_stats( $delta_for, $elapsed, $devs_in_group, \%stats ); - while ( my ($k, $v) = each %extras ) { - $stats{$k} = $v; - } + my %extras = $self->_calc_misc_stats( + delta_for => $delta_for, + elapsed => $elapsed, + devs_in_group => $devs_in_group, + stats => \%stats, + ); + + @stats{ keys %extras } = values %extras; $stats{dev} = $dev; @@ -2315,7 +1907,7 @@ sub _calc_stats_for_deltas { sub _calc_deltas { my ( $self ) = @_; - my $elapsed = $self->current_ts() - $self->delta_against_ts(); + my $elapsed = $self->curr_ts() - $self->delta_against_ts(); die "Time elapsed is [$elapsed]" unless $elapsed; return $self->_calc_stats_for_deltas($elapsed); @@ -2324,14 +1916,16 @@ sub _calc_deltas { sub print_header { my ($self, $header, @args) = @_; if ( $self->{_print_header} ) { - printf { $self->out_fh } $header . "\n", @args; + printf { $self->out_fh() } $header . "\n", @args; } } sub print_rest { my ($self, $format, $cols, $stat) = @_; if ( $self->filter_zeroed_rows() ) { - return unless grep { sprintf("%7.1f", $_) != 0 } @{$stat}{ grep { $self->col_ok($_) } @$cols }; + return unless grep { + sprintf("%7.1f", $_) != 0 + } @{$stat}{ @$cols }; } printf { $self->out_fh() } $format . "\n", @{$stat}{ qw( line_ts dev ), @$cols }; @@ -2347,18 +1941,18 @@ sub print_deltas { return unless $self->delta_against_ts(); @$cols = map { $self->_column_to_key($_) } @$cols; - my ( $header_cb, $rest_cb ) = @args{qw( header_cb rest_cb )}; + my ( $header_callback, $rest_callback ) = @args{qw( header_callback rest_callback )}; - if ( $header_cb ) { - $self->$header_cb( $header, "#ts", "device" ); + if ( $header_callback ) { + $self->$header_callback( $header, "#ts", "device" ); } else { $self->print_header( $header, "#ts", "device" ); } for my $stat ( $self->_calc_deltas() ) { - if ($rest_cb) { - $self->$rest_cb( $format, $cols, $stat ); + if ($rest_callback) { + $self->$rest_callback( $format, $cols, $stat ); } else { $self->print_rest( $format, $cols, $stat ); @@ -2368,9 +1962,9 @@ sub print_deltas { sub compute_line_ts { my ( $self, %args ) = @_; - return $args{first_ts} > 0 - ? sprintf( "%5.1f", $args{current_ts} - $args{first_ts} ) - : sprintf( "%5.1f", 0 ); + return sprintf( "%5.1f", $args{first_ts} > 0 + ? $args{curr_ts} - $args{first_ts} + : 0 ); } sub compute_in_progress { @@ -2395,34 +1989,20 @@ sub group_by { } 1; - } # ########################################################################### # End Diskstats package # ########################################################################### -# This program is copyright 2011 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### # DiskstatsGroupByAll package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/DiskstatsGroupByAll.pm +# t/lib/DiskstatsGroupByAll.t +# See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { -# Package: DiskstatsGroupByAll -# package DiskstatsGroupByAll; @@ -2433,23 +2013,17 @@ use constant MKDEBUG => $ENV{MKDEBUG} || 0; use base qw( Diskstats ); -sub group_by { - my $self = shift; - $self->group_by_all(@_); -} - sub group_by_all { my ($self, %args) = @_; - if ( !$args{clear_state} ) { - $self->clear_state(); - } + $self->clear_state(); if (!$self->interactive) { $self->parse_from( sample_callback => sub { $self->print_deltas( - map { ( $_ => $args{$_} ) } qw( header_cb rest_cb ), + map { ( $_ => $args{$_} ) } + qw( header_callback rest_callback ), ); }, map( { ($_ => $args{$_}) } qw(filehandle filename data) ), @@ -2460,20 +2034,20 @@ sub group_by_all { $self->parse_from( sample_callback => sub { $self->print_deltas( - header_cb => sub { + header_callback => sub { my $self = shift; if ( $self->{_print_header} ) { - my $meth = $args{header_cb} || "print_header"; + my $meth = $args{header_callback} || "print_header"; $self->$meth(@_); } $self->{_print_header} = undef; }, - rest_cb => $args{rest_cb}, + rest_callback => $args{rest_callback}, ); }, map( { ($_ => $args{$_}) } qw(filehandle filename data) ), ); - if (!$self->previous_ts) { + if (!$self->prev_ts) { seek $args{filehandle}, $orig, 0; } return; @@ -2481,6 +2055,12 @@ sub group_by_all { $self->clear_state(); } + +sub group_by { + my $self = shift; + $self->group_by_all(@_); +} + sub clear_state { my $self = shift; if (!$self->interactive()) { @@ -2495,12 +2075,20 @@ sub clear_state { sub delta_against { my ($self, $dev) = @_; - return $self->previous_stats_for($dev); + return $self->prev_stats_for($dev); } sub delta_against_ts { my ($self) = @_; - return $self->previous_ts(); + return $self->prev_ts(); +} + +sub compute_line_ts { + my ($self, %args) = @_; + if ( $self->interactive() ) { + $args{first_ts} = $self->prev_ts(); + } + return $self->SUPER::compute_line_ts(%args); } 1; @@ -2509,28 +2097,15 @@ sub delta_against_ts { # End DiskstatsGroupByAll package # ########################################################################### -# This program is copyright 2011 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### # DiskstatsGroupByDisk package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/DiskstatsGroupByDisk.pm +# t/lib/DiskstatsGroupByDisk.t +# See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { -# Package: DiskstatsGroupByDisk -# package DiskstatsGroupByDisk; @@ -2544,25 +2119,21 @@ use base qw( Diskstats ); sub new { my ($class, %args) = @_; my $self = $class->SUPER::new(%args); - $self->{_iterations} = 0; + $self->{_iterations} = 0; $self->{_print_header} = 1; return $self; } sub group_by { - my $self = shift; - $self->group_by_disk(@_); + my ($self, @args) = @_; + $self->group_by_disk(@args); } -# Prints out one line for each disk, summing over the interval from first to -# last sample. sub group_by_disk { - my ($self, %args) = @_; - my ($header_cb, $rest_cb) = $args{ qw( header_cb rest_cb ) }; + my ($self, %args) = @_; + my ($header_callback, $rest_callback) = $args{ qw( header_callback rest_callback ) }; - if (!$self->interactive()) { - $self->clear_state(); - } + $self->clear_state() unless $self->interactive(); my $original_offset = $args{filehandle} ? tell($args{filehandle}) : undef; @@ -2570,37 +2141,40 @@ sub group_by_disk { sample_callback => sub { my ($self, $ts) = @_; - if ( $self->has_stats ) { + if ( $self->has_stats() ) { $self->{_iterations}++; if ($self->interactive() && $self->{_iterations} >= 2) { - my $elapsed = - ( $self->current_ts() || 0 ) - - ( $self->first_ts() || 0 ); + my $elapsed = ( $self->curr_ts() || 0 ) + - ( $self->first_ts() || 0 ); if ( $ts > 0 && $elapsed >= $self->sample_time() ) { $self->print_deltas( - header_cb => sub { + header_callback => sub { my ($self, @args) = @_; if ( $self->{_print_header} ) { - my $meth = $args{header_cb} || "print_header"; - $self->$meth(@args); + my $method = $args{header_callback} + || "print_header"; + $self->$method(@args); } $self->{_print_header} = undef; }, - rest_cb => $args{rest_cb}, + rest_callback => $args{rest_callback}, ); $self->{_iterations} = -1; - return "Stop interactive reading"; + return; } } } }, - map({ ($_ => $args{$_}) } qw(filehandle filename data)), + filehandle => $args{filehandle}, + filename => $args{filename}, + data => $args{data}, ); if ($self->interactive) { - if ($self->{_iterations} == -1 && defined($original_offset) && eof($args{filehandle})) { + if ($self->{_iterations} == -1 && defined($original_offset) + && eof($args{filehandle})) { $self->clear_state; seek $args{filehandle}, $original_offset, 0; } @@ -2611,9 +2185,12 @@ sub group_by_disk { return; } - $self->print_deltas( map( { ( $_ => $args{$_} ) } qw( header_cb rest_cb ) ) ); + $self->print_deltas( + header_callback => $args{header_callback}, + rest_callback => $args{rest_callback}, + ); - $self->clear_state; + $self->clear_state(); return $lines_read; } @@ -2652,28 +2229,15 @@ sub compute_in_progress { # End DiskstatsGroupByDisk package # ########################################################################### -# This program is copyright 2011 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### # DiskstatsGroupBySample package +# This package is a copy without comments from the original. The original +# with comments and its test file can be found in the Bazaar repository at, +# lib/DiskstatsGroupBySample.pm +# t/lib/DiskstatsGroupBySample.t +# See https://launchpad.net/percona-toolkit for more information. # ########################################################################### { -# Package: DiskstatsGroupBySample -# package DiskstatsGroupBySample; @@ -2698,54 +2262,50 @@ sub group_by { $self->group_by_sample(@_); } -# Prints out one line for each disk, summing over the interval from first to -# last sample. sub group_by_sample { my ( $self, %args ) = @_; - my ( $header_cb, $rest_cb ) = $args{qw( header_cb rest_cb )}; + my ( $header_callback, $rest_callback ) = $args{qw( header_callback rest_callback )}; - if (!$self->interactive) { - $self->clear_state; - } + $self->clear_state() unless $self->interactive(); $self->parse_from( sample_callback => $self->can("_sample_callback"), - map( { ( $_ => $args{$_} ) } qw(filehandle filename data) ), + filehandle => $args{filehandle}, + filename => $args{filename}, + data => $args{data}, ); - if (!$self->interactive) { - $self->clear_state; - } + $self->clear_state() unless $self->interactive(); + return; } sub _sample_callback { my ( $self, $ts, %args ) = @_; my $printed_a_line = 0; - if ( $self->has_stats ) { + if ( $self->has_stats() ) { $self->{_iterations}++; } - my $elapsed = - ( $self->current_ts() || 0 ) - - ( $self->previous_ts() || 0 ); + my $elapsed = ($self->curr_ts() || 0) + - ($self->prev_ts() || 0); if ( $ts > 0 && $elapsed >= $self->sample_time() ) { $self->print_deltas( - max_device_length => 6, - header_cb => sub { + max_device_length => 6, + header_callback => sub { my ( $self, $header, @args ) = @_; if ( $self->{_print_header} ) { - my $method = $args{header_cb} || "print_header"; + my $method = $args{header_callback} || "print_header"; $self->$method( $header, @args ); $self->{_print_header} = undef; } }, - rest_cb => sub { + rest_callback => sub { my ( $self, $format, $cols, $stat ) = @_; - my $method = $args{rest_cb} || "print_rest"; + my $method = $args{rest_callback} || "print_rest"; $self->$method( $format, $cols, $stat ); $printed_a_line = 1; } @@ -2753,23 +2313,24 @@ sub _sample_callback { } if ( $self->{_iterations} == 1 || $printed_a_line == 1 ) { $self->{_save_curr_as_prev} = 1; - $self->_save_current_as_previous( $self->stats_for() ); + $self->_save_curr_as_prev( $self->stats_for() ); $self->{_save_curr_as_prev} = 0; } + return; } sub delta_against { my ( $self, $dev ) = @_; - return $self->previous_stats_for($dev); + return $self->prev_stats_for($dev); } sub delta_against_ts { my ( $self ) = @_; - return $self->previous_ts(); + return $self->prev_ts(); } sub clear_state { - my ( $self, @args ) = @_; + my ( $self, @args ) = @_; $self->{_iterations} = 0; $self->{_save_curr_as_prev} = 0; $self->{_print_header} = 1; @@ -2778,26 +2339,27 @@ sub clear_state { sub compute_devs_in_group { my ($self) = @_; + my $stats = $self->stats_for(); + my $re = $self->device_regex(); return scalar grep { - # Got stats for that device, and we want to print it - $self->stats_for($_) && $self->dev_ok($_) - } $self->sorted_devs; + $stats->{$_} && $_ =~ $re + } $self->ordered_devs; } sub compute_dev { - my ( $self, $dev ) = @_; - return $self->compute_devs_in_group() > 1 - ? "{" . $self->compute_devs_in_group() . "}" - : ( $self->sorted_devs )[0]; + my ( $self, $devs ) = @_; + $devs ||= $self->compute_devs_in_group(); + return $devs > 1 + ? "{" . $devs . "}" + : ( $self->ordered_devs )[0]; } -# Terrible breach of encapsulation, but it'll have to do for the moment. sub _calc_stats_for_deltas { my ( $self, $elapsed ) = @_; my $delta_for; - for my $dev ( grep { $self->dev_ok($_) } $self->sorted_devs ) { + foreach my $dev ( grep { $self->dev_ok($_) } $self->ordered_devs ) { my $curr = $self->stats_for($dev); my $against = $self->delta_against($dev); @@ -2808,51 +2370,45 @@ sub _calc_stats_for_deltas { } } - my $in_progress = $delta_for->{ios_in_progress}; #$curr->{"ios_in_progress"}; - my $tot_in_progress = 0; #$against->{"sum_ios_in_progress"} || 0; - - my $devs_in_group = $self->compute_devs_in_group() || 1; + my $in_progress = $delta_for->{ios_in_progress}; + my $tot_in_progress = 0; + my $devs_in_group = $self->compute_devs_in_group() || 1; my %stats = ( - $self->_calc_read_stats( $delta_for, $elapsed, $devs_in_group ), - $self->_calc_write_stats( $delta_for, $elapsed, $devs_in_group ), + $self->_calc_read_stats( + delta_for => $delta_for, + elapsed => $elapsed, + devs_in_group => $devs_in_group, + ), + $self->_calc_write_stats( + delta_for => $delta_for, + elapsed => $elapsed, + devs_in_group => $devs_in_group, + ), in_progress => - $self->compute_in_progress( $in_progress, $tot_in_progress ), + $self->compute_in_progress( $in_progress, $tot_in_progress ), ); - my %extras = $self->_calc_misc_stats( $delta_for, $elapsed, $devs_in_group, \%stats ); - while ( my ($k, $v) = each %extras ) { - $stats{$k} = $v; - } + my %extras = $self->_calc_misc_stats( + delta_for => $delta_for, + elapsed => $elapsed, + devs_in_group => $devs_in_group, + stats => \%stats, + ); - $stats{dev} = $self->compute_dev( \%stats ); + @stats{ keys %extras } = values %extras; + + $stats{dev} = $self->compute_dev( $devs_in_group ); return \%stats; } 1; - } # ########################################################################### # End DiskstatsGroupBySample package # ########################################################################### -# This program is copyright 2011 Percona Inc. -# Feedback and improvements are welcome. -# -# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar -# systems, you can issue `man perlgpl' or `man perlartistic' to read these -# licenses. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. # ########################################################################### # DiskstatsMenu # ########################################################################### @@ -2876,8 +2432,6 @@ require DiskstatsGroupByAll; require DiskstatsGroupByDisk; require DiskstatsGroupBySample; -our $VERSION = '0.01'; - my %actions = ( 'A' => \&group_by, 'D' => \&group_by, @@ -2888,7 +2442,7 @@ my %actions = ( 'c' => get_new_regex_for( "column_regex", "Enter a column pattern: " ), '/' => get_new_regex_for( "device_regex", "Enter a disk/device pattern: " ), 'q' => sub { return 'last' }, - 'p' => \&pause, + 'p' => sub { print "Paused - press any key to continue\n"; pause(@_); return; }, '?' => \&help, ); @@ -2898,52 +2452,39 @@ my %option_to_object = ( S => "DiskstatsGroupBySample", ); -my %object_to_option = reverse %option_to_object; +sub new { + bless {}, shift; +} sub run_interactive { my ($self, %args) = @_; - die "I need an [o] argument" unless $args{o} && blessed($args{o}) - && ( - $args{o}->isa("OptionParser") - || $args{o}->can("get") - ); - my $o = $args{o}; + my @required_args = qw(OptionParser); + foreach my $arg ( @required_args ) { + die "I need a $arg argument" unless $args{$arg}; + } + my ($o) = @args{@required_args}; my %opts = ( - save_samples => $o->get('save-samples') || undef, - samples_to_gather => $o->get('iterations') || undef, - sampling_interval => $o->get('interval') || 1, - redisplay_interval => 1, - sample_time => $o->get('sample-time') || 1, - column_regex => $o->get('columns') || undef, - device_regex => $o->get('devices') || undef, interactive => 1, - filter_zeroed_rows => !$o->get('zero-rows'), + OptionParser => $o, ); - for my $re_key ( grep { $opts{$_} } qw( column_regex device_regex ) ) { - $opts{$re_key} = qr/$opts{$re_key}/i; - } - my ($tmp_fh, $filename, $child_pid, $child_fh); # Here's a big crux of the program. If we have a filename, we don't # need to fork and create a child, just read from it. - if ( $args{filename} ) { - $filename = $args{filename}; - open $tmp_fh, "<", $filename or die "Couldn't open [$filename]: $OS_ERROR"; + if ( $filename = $args{filename} ) { + open $tmp_fh, "<", $filename or die "Cannot open $filename: $OS_ERROR"; } else { - ($tmp_fh, $filename) = file_to_use( $opts{save_samples} ); + ($tmp_fh, $filename) = file_to_use( $o->get('save-samples') ); # fork(), but future-proofing it in case we ever need to speak to # the child $child_pid = open $child_fh, "|-"; - if (not defined $child_pid) { - die "Couldn't fork: $OS_ERROR"; - } + die "Cannot fork: $OS_ERROR" unless defined $child_pid; if ( !$child_pid ) { # Child @@ -2952,52 +2493,55 @@ sub run_interactive { # so it's easier to track in things like ps. local $PROGRAM_NAME = "$PROGRAM_NAME (data-gathering daemon)"; - close($tmp_fh); - - open my $fh, ">>", $filename or die $!; + close $tmp_fh; gather_samples( gather_while => sub { getppid() }, - samples_to_gather => $opts{samples_to_gather}, - sampling_interval => $opts{sampling_interval}, - filehandle => $fh, + samples_to_gather => $o->get('iterations'), + sampling_interval => $o->get('interval'), + filename => $filename, ); - close $fh or die $!; - unlink $filename unless $opts{save_samples}; + unlink $filename unless $o->get('save-samples'); exit(0); } } + # I don't think either of these are needed actually, since piped opens + # are supposed to deal with children on their own, but it doesn't hurt. local $SIG{CHLD} = 'IGNORE'; local $SIG{PIPE} = 'IGNORE'; STDOUT->autoflush; STDIN->blocking(0); - my $sel = IO::Select->new(\*STDIN); - my $class = $option_to_object{ substr ucfirst($o->get('group-by') || 'Disk'), 0, 1 }; - $opts{obj} = $class->new( %opts ); + my $sel = IO::Select->new(\*STDIN); + my $group_by = $o->get('group-by') || 'disk'; + my $class = $group_by =~ m/disk/i ? 'DiskstatsGroupByDisk' + : $group_by =~ m/sample/i ? 'DiskstatsGroupBySample' + : $group_by =~ m/all/i ? 'DiskstatsGroupByAll' + : die "Invalid --group-by: $group_by"; + $opts{obj} = $class->new( %opts ); if ( $args{filename} ) { group_by( - header_cb => sub { shift->print_header(@_) }, + header_callback => sub { shift->print_header(@_) }, select_obj => $sel, options => \%opts, filehandle => $tmp_fh, - got => substr(ucfirst($o->get('group-by') || 'Disk'), 0, 1), + input => substr(ucfirst($group_by), 0, 1), ); } ReadKeyMini::cbreak(); MAIN_LOOP: while (1) { - if ( my $got = read_command_timeout($sel, $opts{redisplay_interval} ) ) { - if ($actions{$got}) { - my $ret = $actions{$got}->( + if ( my $input = read_command_timeout($sel, $o->get('redisplay-interval') ) ) { + if ($actions{$input}) { + my $ret = $actions{$input}->( select_obj => $sel, options => \%opts, - got => $got, + input => $input, filehandle => $tmp_fh, ) || ''; last MAIN_LOOP if $ret eq 'last'; @@ -3012,7 +2556,7 @@ sub run_interactive { # If we are gathering samples (don't have a filename), and we have a sample # limit (set by --iterations), the child process just calls it quits once # it gathers enough samples. When that happens, we are also done. - if ( !$args{filename} && $opts{samples_to_gather} && kill 0, $child_pid ) { + if ( !$args{filename} && $o->get('iterations') && kill 0, $child_pid ) { last MAIN_LOOP; } @@ -3028,8 +2572,8 @@ sub run_interactive { waitpid $child_pid, 0; } - close($tmp_fh) or die "Couldn't close: $OS_ERROR"; - return; + close($tmp_fh) or die "Cannot close: $OS_ERROR"; + return 0; # Exit status } sub read_command_timeout { @@ -3045,23 +2589,27 @@ sub gather_samples { my $samples = 0; STDIN->blocking(0); - my $sel = IO::Select->new(\*STDIN); + my $sel = IO::Select->new(\*STDIN); + my $filename = $opts{filename}; GATHER_DATA: while ( $opts{gather_while}->() ) { if ( read_command_timeout( $sel, $opts{sampling_interval} ) ) { last GATHER_DATA; } + + open my $fh, ">>", $filename or die $OS_ERROR; open my $diskstats_fh, "<", "/proc/diskstats" - or die $!; + or die $OS_ERROR; my @to_print = `date +'TS %s.%N %F %T'`; push @to_print, <$diskstats_fh>; # Lovely little method from IO::Handle: turns on autoflush, # prints, and then restores the original autoflush state. - $opts{filehandle}->printflush(@to_print); - close $diskstats_fh or die $!; + $fh->printflush(@to_print); + close $diskstats_fh or die $OS_ERROR; + close $fh or die $OS_ERROR; $samples++; if ( defined($opts{samples_to_gather}) && $samples >= $opts{samples_to_gather} ) { @@ -3074,16 +2622,16 @@ sub gather_samples { sub group_by { my (%args) = @_; - my $got = $args{got}; + my $input = $args{input}; - if ( ref( $args{options}->{obj} ) ne $option_to_object{$got} ) { + if ( ref( $args{options}->{obj} ) ne $option_to_object{$input} ) { # Particularly important! Otherwise we would depend on the # object's ->new being smart about discarding unrecognized # values. delete $args{options}->{obj}; # This would fail on a stricter constructor, so it probably # needs fixing. - $args{options}->{obj} = $option_to_object{$got}->new( %{$args{options}}); + $args{options}->{obj} = $option_to_object{$input}->new( %{$args{options}}); } seek $args{filehandle}, 0, 0; @@ -3098,7 +2646,7 @@ sub group_by { $obj->group_by( filehandle => $args{filehandle}, # Only print the header once, as if in interactive. - header_cb => $args{header_cb} || sub { + header_callback => $args{header_callback} || sub { my $print_header; return sub { unless ($print_header++) { @@ -3112,38 +2660,18 @@ sub group_by { } } -# regexp_pattern is used for pretty-printing regexen, since they can stringify to -# different things depending on the version of Perl. Unfortunately, 5.8 -# lacks this, so in that version, we put in a facsimile. -BEGIN { - local $EVAL_ERROR; - - eval { require re; re::regexp_pattern(qr//) }; - if ( $EVAL_ERROR ) { - *regexp_pattern = sub { - my ($re) = @_; - (my $string_re = $re) =~ s/\A\(\?[^:]*?:(.*)\)\z/$1/sm; - return $string_re; - }; - } - else { - re->import("regexp_pattern"); - } -} - sub help { my (%args) = @_; my $obj = $args{options}->{obj}; - my $mode = $object_to_option{ref($obj)}; - my ($column_re) = regexp_pattern( $obj->column_regex() ); - my ($device_re) = regexp_pattern( $obj->device_regex() ); + my $mode = substr ref($obj), 16, 1; + my $column_re = $args{options}->{OptionParser}->get('columns'); + my $device_re = $args{options}->{OptionParser}->get('devices'); my $interval = $obj->sample_time() || '(none)'; - my $disp_int = $args{options}->{redisplay_interval} || '(none)'; + my $disp_int = $args{options}->{OptionParser}->get('redisplay-interval'); my $inact_disk = $obj->filter_zeroed_rows() ? 'yes' : 'no'; for my $re ( $column_re, $device_re ) { $re ||= '(none)'; - $re =~ s/^\Q(?=)\E$/(none)/; } print <<"HELP"; @@ -3160,6 +2688,7 @@ sub help { ------------------- Press any key to continue ----------------------- HELP pause(@_); + return; } sub file_to_use { @@ -3171,7 +2700,7 @@ sub file_to_use { if ( $filename ) { open my $fh, "<", $filename - or die "Couldn't open $filename: $OS_ERROR"; + or die "Cannot open $filename: $OS_ERROR"; return $fh, $filename; } else { @@ -3204,55 +2733,58 @@ sub get_input { } sub hide_inactive_disks { - my (%args) = @_; - my $new_val = !!get_input("Filter inactive rows? (Leave blank for 'No') "); + my (%args) = @_; + my $new_val = get_input("Filter inactive rows? (Leave blank for 'No') "); - $args{options}->{filter_zeroed_rows} = $new_val; + # Eeep. In OptionParser, "true" means show; in Diskstats, "true" means hide. + # Thus !$new_val for OptionParser + $args{options}->{OptionParser}->set('zero-rows', !$new_val); $args{options}->{obj}->filter_zeroed_rows($new_val); + return; } sub get_new_value_for { my ($looking_for, $message) = @_; + (my $looking_for_o = $looking_for) =~ tr/_/-/; return sub { my (%args) = @_; - my $new_interval = get_input($message); + my $new_interval = get_input($message) || 0; - $new_interval ||= 0; - - if ( looks_like_number($new_interval) ) { - if ( $args{options}->{obj}->can($looking_for) ) { - $args{options}->{obj}->$looking_for($new_interval); - } - return $args{options}->{$looking_for} = $new_interval; - } - else { - die("invalid timeout specification"); + die "invalid timeout specification" unless looks_like_number($new_interval); + + if ( $args{options}->{obj}->can($looking_for) ) { + $args{options}->{obj}->$looking_for($new_interval); } + $args{options}->{OptionParser}->set($looking_for_o, $new_interval); + return $new_interval; }; } sub get_new_regex_for { my ($looking_for, $message) = @_; + (my $looking_for_o = $looking_for) =~ s/_.*$/s/; return sub { - my (%args) = @_; + my (%args) = @_; my $new_regex = get_input($message); local $EVAL_ERROR; if ( $new_regex && (my $re = eval { qr/$new_regex/i }) ) { - $args{options}->{$looking_for} = $re; + $args{options}->{obj}->$looking_for( $re ); + $args{options}->{OptionParser}->set($looking_for_o, $new_regex); } elsif ( !$EVAL_ERROR && !$new_regex ) { # This might seem weird, but an empty pattern is # somewhat magical, and basically just asking for trouble. # Instead we give them what awk would, a pattern that always # matches. - $args{options}->{$looking_for} = qr/(?=)/; + $args{options}->{obj}->$looking_for( qr/(?=)/ ); + $args{options}->{OptionParser}->set($looking_for_o, ''); } else { - die("invalid regex specification: $EVAL_ERROR"); + die "invalid regex specification: $EVAL_ERROR"; } - $args{options}->{obj}->$looking_for( $args{options}->{$looking_for} ); + return; }; } @@ -3271,6 +2803,14 @@ sub pause { # End DiskstatsMenu package # ########################################################################### { +# ########################################################################### +# This is a combination of modules and programs in one -- a runnable module. +# http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last +# Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition. +# +# Check at the end of this package for the call to main() which actually runs +# the program. +# ########################################################################### package pt_diskstats; use strict; @@ -3289,7 +2829,6 @@ local $SIG{__DIE__} = sub { } if MKDEBUG; sub main { - shift; local @ARGV = @_; # set global ARGV for this package # ######################################################################## @@ -3299,17 +2838,25 @@ sub main { $o->get_specs(); $o->get_opts(); - # Interactive mode. Delegate to Diskstats::Menu - return DiskstatsMenu->run_interactive( o => $o, filename => $ARGV[0] ); + # Interactive mode. Delegate to DiskstatsMenu::run_interactive + return DiskstatsMenu->run_interactive( OptionParser => $o, filename => $ARGV[0] ); } # Somewhat important if STDOUT is tied to a terminal. END { close STDOUT or die "Couldn't close stdout: $OS_ERROR" } -__PACKAGE__->main(@ARGV) unless caller; +# ############################################################################ +# Run the program. +# ############################################################################ +if ( !caller ) { exit main(@ARGV); } 1; } + +# ############################################################################# +# Documentation. +# ############################################################################# + =pod =head1 NAME @@ -3519,6 +3066,12 @@ type: int When in interactive mode, stop after N samples. +=item --redisplay-interval + +type: int; default: 1 + +When in interactive mode, wait N seconds before printing to the screen. + =item --interval type: int; default: 1 @@ -3529,6 +3082,10 @@ Sample /proc/diskstats every N seconds. Show rows with all zero values. +=item --memory-for-speed + +XXX TODO INTERNAL yadda + =item --help Show help and exit.