diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 86c6d710..f7524ce8 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -342,7 +342,7 @@ sub _split_url { or die(qq/SSL certificate not valid for $host\n/); } } - + $self->{host} = $host; $self->{port} = $port; @@ -469,11 +469,11 @@ sub _split_url { sub write_header_lines { (@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/); - my($self, $headers) = @_; + my($self, $headers) = @_; my $buf = ''; while (my ($k, $v) = each %$headers) { - my $field_name = lc $k; + my $field_name = lc $k; $field_name =~ /\A [\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5A\x5E-\x7A\x7C\x7E]+ \z/x or croak(q/Invalid HTTP header field name: / . $Printable->($field_name)); $field_name =~ s/\b(\w)/\u$1/g; @@ -817,7 +817,7 @@ my @vc_dirs = ( } PTDEBUG && _d('Version check file', $file, 'in', $ENV{PWD}); return $file; # in the CWD - } + } } sub version_check_time_limit { @@ -834,11 +834,11 @@ sub version_check { PTDEBUG && _d('FindBin::Bin:', $FindBin::Bin); if ( !$args{force} ) { if ( $FindBin::Bin - && (-d "$FindBin::Bin/../.bzr" || + && (-d "$FindBin::Bin/../.bzr" || -d "$FindBin::Bin/../../.bzr" || - -d "$FindBin::Bin/../.git" || - -d "$FindBin::Bin/../../.git" - ) + -d "$FindBin::Bin/../.git" || + -d "$FindBin::Bin/../../.git" + ) ) { PTDEBUG && _d("$FindBin::Bin/../.bzr disables --version-check"); return; @@ -862,7 +862,7 @@ sub version_check { PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); return unless @$instances_to_check; - my $protocol = 'https'; + my $protocol = 'https'; eval { require IO::Socket::SSL; }; if ( $EVAL_ERROR ) { PTDEBUG && _d($EVAL_ERROR); @@ -1039,7 +1039,7 @@ sub get_uuid { close $fh; return $uuid; -} +} sub _generate_uuid { return sprintf+($}="%04x")."$}-$}-$}-$}-".$}x3,map rand 65537,0..7; @@ -1088,7 +1088,7 @@ sub pingback { ); die "Failed to parse server requested programs: $response->{content}" if !scalar keys %$items; - + my $versions = get_versions( items => $items, instances => $instances, @@ -1347,7 +1347,7 @@ sub get_from_mysql { if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { @{$item->{vars}} = grep { $_ eq 'version' || $_ eq 'version_comment' } @{$item->{vars}}; } - + my @versions; my %version_for; @@ -1477,7 +1477,7 @@ sub parse { foreach my $key ( keys %$opts ) { PTDEBUG && _d('Finding value for', $key); $final_props{$key} = $given_props{$key}; - if ( !defined $final_props{$key} + if ( !defined $final_props{$key} && defined $prev->{$key} && $opts->{$key}->{copy} ) { $final_props{$key} = $prev->{$key}; @@ -1621,7 +1621,7 @@ sub get_dbh { my $dbh; my $tries = 2; while ( !$dbh && $tries-- ) { - PTDEBUG && _d($cxn_string, ' ', $user, ' ', $pass, + PTDEBUG && _d($cxn_string, ' ', $user, ' ', $pass, join(', ', map { "$_=>$defaults->{$_}" } keys %$defaults )); $dbh = eval { DBI->connect($cxn_string, $user, $pass, $defaults) }; @@ -1819,7 +1819,7 @@ sub set_vars { } } - return; + return; } sub _d { @@ -1900,7 +1900,7 @@ sub new { rules => [], # desc of rules for --help mutex => [], # rule: opts are mutually exclusive atleast1 => [], # rule: at least one opt is required - disables => {}, # rule: opt disables other opts + disables => {}, # rule: opt disables other opts defaults_to => {}, # rule: opt defaults to value of other opt DSNParser => undef, default_files => [ @@ -2063,7 +2063,7 @@ sub _pod_to_specs { } push @specs, { - spec => $self->{parse_attributes}->($self, $option, \%attribs), + spec => $self->{parse_attributes}->($self, $option, \%attribs), desc => $para . (defined $attribs{default} ? " (default $attribs{default})" : ''), group => ($attribs{'group'} ? $attribs{'group'} : 'default'), @@ -2154,7 +2154,7 @@ sub _parse_specs { $self->{opts}->{$long} = $opt; } else { # It's an option rule, not a spec. - PTDEBUG && _d('Parsing rule:', $opt); + PTDEBUG && _d('Parsing rule:', $opt); push @{$self->{rules}}, $opt; my @participants = $self->_get_participants($opt); my $rule_ok = 0; @@ -2199,7 +2199,7 @@ sub _parse_specs { PTDEBUG && _d('Option', $long, 'disables', @participants); } - return; + return; } sub _get_participants { @@ -2286,7 +2286,7 @@ sub _set_option { } sub get_opts { - my ( $self ) = @_; + my ( $self ) = @_; foreach my $long ( keys %{$self->{opts}} ) { $self->{opts}->{$long}->{got} = 0; @@ -2417,7 +2417,7 @@ sub _check_opts { else { $err = join(', ', map { "--$self->{opts}->{$_}->{long}" } - grep { $_ } + grep { $_ } @restricted_opts[0..scalar(@restricted_opts) - 2] ) . ' or --'.$self->{opts}->{$restricted_opts[-1]}->{long}; @@ -2427,7 +2427,7 @@ sub _check_opts { } } - elsif ( $opt->{is_required} ) { + elsif ( $opt->{is_required} ) { $self->save_error("Required option --$long must be specified"); } @@ -2811,7 +2811,7 @@ sub clone { $clone{$scalar} = $self->{$scalar}; } - return bless \%clone; + return bless \%clone; } sub _parse_size { @@ -3325,7 +3325,7 @@ sub extends { sub _load_module { my ($class) = @_; - + (my $file = $class) =~ s{::|'}{/}g; $file .= '.pm'; { local $@; eval { require "$file" } } # or warn $@; @@ -3356,7 +3356,7 @@ sub has { my $caller = scalar caller(); my $class_metadata = Lmo::Meta->metadata_for($caller); - + for my $attribute ( ref $names ? @$names : $names ) { my %args = @_; my $method = ($args{is} || '') eq 'ro' @@ -3375,16 +3375,16 @@ sub has { if ( my $type_check = $args{isa} ) { my $check_name = $type_check; - + if ( my ($aggregate_type, $inner_type) = $type_check =~ /\A(ArrayRef|Maybe)\[(.*)\]\z/ ) { $type_check = Lmo::Types::_nested_constraints($attribute, $aggregate_type, $inner_type); } - + my $check_sub = sub { my ($new_val) = @_; Lmo::Types::check_type_constaints($attribute, $type_check, $check_name, $new_val); }; - + $class_metadata->{$attribute}{isa} = [$check_name, $check_sub]; my $orig_method = $method; $method = sub { @@ -3763,7 +3763,7 @@ sub get_id { my $sql = q{SHOW STATUS LIKE 'wsrep\_local\_index'}; my (undef, $wsrep_local_index) = $cxn->dbh->selectrow_array($sql); PTDEBUG && _d("Got cluster wsrep_local_index: ",$wsrep_local_index); - $unique_id = $wsrep_local_index."|"; + $unique_id = $wsrep_local_index."|"; foreach my $val ('server\_id', 'wsrep\_sst\_receive\_address', 'wsrep\_node\_name', 'wsrep\_node\_address') { my $sql = "SHOW VARIABLES LIKE '$val'"; PTDEBUG && _d($cxn->name, $sql); @@ -3793,7 +3793,7 @@ sub is_cluster_node { PTDEBUG && _d($sql); #don't invoke name() if it's not a Cxn! } else { - $dbh = $cxn->dbh(); + $dbh = $cxn->dbh(); PTDEBUG && _d($cxn->name, $sql); } @@ -3925,7 +3925,7 @@ sub find_cluster_nodes { my $dp = $args{DSNParser}; my $make_cxn = $args{make_cxn}; - + my $sql = q{SHOW STATUS LIKE 'wsrep\_incoming\_addresses'}; PTDEBUG && _d($sql); my (undef, $addresses) = $dbh->selectrow_array($sql); @@ -4005,7 +4005,7 @@ sub autodetect_nodes { my $new_nodes = []; return $new_nodes unless @$nodes; - + for my $node ( @$nodes ) { my $nodes_found = $self->find_cluster_nodes( dbh => $node->dbh(), @@ -4037,12 +4037,12 @@ sub autodetect_nodes { ); my @new_slave_nodes = grep { $self->is_cluster_node($_) } @$new_slaves; - + my $slaves_of_slaves = $self->autodetect_nodes( %args, nodes => \@new_slave_nodes, ); - + my @autodetected_nodes = ( @$new_nodes, @$new_slaves, @$slaves_of_slaves ); return \@autodetected_nodes; } @@ -4120,7 +4120,7 @@ sub split_unquote { s/`\z//; s/``/`/g; } - + return ($db, $tbl); } @@ -4567,9 +4567,9 @@ sub parse { sub remove_quoted_text { my ($string) = @_; - $string =~ s/[^\\]`[^`]*[^\\]`//g; - $string =~ s/[^\\]"[^"]*[^\\]"//g; - $string =~ s/[^\\]'[^']*[^\\]'//g; + $string =~ s/[^\\]`[^`]*[^\\]`//g; + $string =~ s/[^\\]"[^"]*[^\\]"//g; + $string =~ s/[^\\]'[^']*[^\\]'//g; return $string; } @@ -4876,7 +4876,7 @@ sub generate_asc_stmt { die "Index '$index' does not exist in table" unless exists $tbl_struct->{keys}->{$index}; - PTDEBUG && _d('Will ascend index', $index); + PTDEBUG && _d('Will ascend index', $index); my @asc_cols = @{$tbl_struct->{keys}->{$index}->{cols}}; if ( $args{asc_first} ) { @@ -5120,22 +5120,22 @@ use warnings FATAL => 'all'; use English qw(-no_match_vars); use constant PTDEBUG => $ENV{PTDEBUG} || 0; -sub check_recursion_method { +sub check_recursion_method { my ($methods) = @_; - if ( @$methods != 1 ) { - if ( grep({ !m/processlist|hosts/i } @$methods) - && $methods->[0] !~ /^dsn=/i ) - { - die "Invalid combination of recursion methods: " - . join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". " - . "Only hosts and processlist may be combined.\n" - } - } - else { + if ( @$methods != 1 ) { + if ( grep({ !m/processlist|hosts/i } @$methods) + && $methods->[0] !~ /^dsn=/i ) + { + die "Invalid combination of recursion methods: " + . join(", ", map { defined($_) ? $_ : 'undef' } @$methods) . ". " + . "Only hosts and processlist may be combined.\n" + } + } + else { my ($method) = @$methods; - die "Invalid recursion method: " . ( $method || 'undef' ) - unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i; - } + die "Invalid recursion method: " . ( $method || 'undef' ) + unless $method && $method =~ m/^(?:processlist$|hosts$|none$|cluster$|dsn=)/i; + } } sub new { @@ -5164,7 +5164,7 @@ sub get_slaves { my $methods = $self->_resolve_recursion_methods($args{dsn}); return $slaves unless @$methods; - + if ( grep { m/processlist|hosts/i } @$methods ) { my @required_args = qw(dbh dsn); foreach my $arg ( @required_args ) { @@ -5177,7 +5177,7 @@ sub get_slaves { { dbh => $dbh, dsn => $dsn, slave_user => $o->got('slave-user') ? $o->get('slave-user') : '', - slave_password => $o->got('slave-password') ? $o->get('slave-password') : '', + slave_password => $o->got('slave-password') ? $o->get('slave-password') : '', callback => sub { my ( $dsn, $dbh, $level, $parent ) = @_; return unless $level; @@ -5209,7 +5209,7 @@ sub get_slaves { else { die "Unexpected recursion methods: @$methods"; } - + return $slaves; } @@ -5746,7 +5746,7 @@ sub short_host { } sub is_replication_thread { - my ( $self, $query, %args ) = @_; + my ( $self, $query, %args ) = @_; return unless $query; my $type = lc($args{type} || 'all'); @@ -5761,7 +5761,7 @@ sub is_replication_thread { if ( !$match ) { if ( ($query->{User} || $query->{user} || '') eq "system user" ) { PTDEBUG && _d("Slave replication thread"); - if ( $type ne 'all' ) { + if ( $type ne 'all' ) { my $state = $query->{State} || $query->{state} || ''; if ( $state =~ m/^init|end$/ ) { @@ -5774,7 +5774,7 @@ sub is_replication_thread { |Reading\sevent\sfrom\sthe\srelay\slog |Has\sread\sall\srelay\slog;\swaiting |Making\stemp\sfile - |Waiting\sfor\sslave\smutex\son\sexit)/xi; + |Waiting\sfor\sslave\smutex\son\sexit)/xi; $match = $type eq 'slave_sql' && $slave_sql ? 1 : $type eq 'slave_io' && !$slave_sql ? 1 @@ -5838,7 +5838,7 @@ sub get_replication_filters { replicate_do_db replicate_ignore_db replicate_do_table - replicate_ignore_table + replicate_ignore_table replicate_wild_do_table replicate_wild_ignore_table ); @@ -5849,7 +5849,7 @@ sub get_replication_filters { $filters{slave_skip_errors} = $row->[1] if $row->[1] && $row->[1] ne 'OFF'; } - return \%filters; + return \%filters; } @@ -5966,11 +5966,11 @@ sub make_row_checksum { die "all columns are excluded by --columns or --ignore-columns" unless @{$cols->{select}}; - + my $query; if ( !$args{no_cols} ) { $query = join(', ', - map { + map { my $col = $_; if ( $col =~ m/UNIX_TIMESTAMP/ ) { my ($real_col) = /^UNIX_TIMESTAMP\((.+?)\)/; @@ -6114,7 +6114,7 @@ sub get_crc_args { my $func = $args{func} || $self->_get_hash_func(%args); my $crc_width = $args{crc_width}|| $self->_get_crc_width(%args, func=>$func); my $crc_type = $args{crc_type} || $self->_get_crc_type(%args, func=>$func); - my $opt_slice; + my $opt_slice; if ( $args{dbh} && $crc_type !~ m/int$/ ) { $opt_slice = $self->_optimize_xor(%args, func=>$func); } @@ -6287,7 +6287,7 @@ sub find_replication_differences { } my ($dbh, $repl_table) = @args{@required_args}; - my $tries = $self->{'OptionParser'}->get('replicate-check-retries') || 1; + my $tries = $self->{'OptionParser'}->get('replicate-check-retries') || 1; my $diffs; while ($tries--) { @@ -6425,7 +6425,7 @@ sub new { sub switch_to_nibble { my $self = shift; - my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols}, + my $params = _nibble_params($self->{nibble_params}, $self->{tbl}, $self->{args}, $self->{cols}, $self->{chunk_size}, $self->{where}, $self->{comments}, $self->{Quoter}); $self->{one_nibble} = 0; @@ -6462,7 +6462,7 @@ sub _one_nibble { my $explain_nibble_sql = "EXPLAIN SELECT " . ($args->{select} ? $args->{select} - : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + : join(', ', map{ $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' ? "CAST(".$q->quote($_)." AS UNSIGNED)" : $q->quote($_) } @$cols)) . " FROM $tbl->{name}" . ($where ? " WHERE $where" : '') @@ -6493,9 +6493,6 @@ sub _nibble_params { ); PTDEBUG && _d('Ascend params:', Dumper($asc)); - my $force_concat_enums; - - my $from = "$tbl->{name} FORCE INDEX(`$index`)"; my $order_by = join(', ', map {$q->quote($_)} @{$index_cols}); my $order_by_dec = join(' DESC,', map {$q->quote($_)} @{$index_cols}); @@ -6559,7 +6556,7 @@ sub _nibble_params { . " /*$comments->{nibble}*/"; PTDEBUG && _d('Nibble statement:', $nibble_sql); - my $explain_nibble_sql + my $explain_nibble_sql = "EXPLAIN SELECT " . ($args->{select} ? $args->{select} : join(', ', map { $q->quote($_) } @{$asc->{cols}})) @@ -6648,7 +6645,7 @@ sub next { sleep($self->{sleep}); } } - + if ( !$self->{have_rows} ) { $self->{nibbleno}++; PTDEBUG && _d('Nibble:', $self->{nibble_sth}->{Statement}, 'params:', @@ -6678,7 +6675,7 @@ sub next { } $self->{rowno} = 0; $self->{have_rows} = 0; - + } PTDEBUG && _d('Done nibbling'); @@ -6815,7 +6812,7 @@ sub can_nibble { } my $pause_file = ($o->has('pause-file') && $o->get('pause-file')) || undef; - + return { row_est => $row_est, # nibble about this many rows index => $index, # using this index @@ -6860,7 +6857,7 @@ sub _find_best_index { push @possible_indexes, $want_index; } } - + if (!$best_index) { PTDEBUG && _d('Auto-selecting best index'); foreach my $index ( $tp->sort_indexes($tbl_struct) ) { @@ -6958,7 +6955,7 @@ sub _prepare_sths { return; } -sub _get_bounds { +sub _get_bounds { my ($self) = @_; if ( $self->{one_nibble} ) { @@ -6971,7 +6968,7 @@ sub _get_bounds { my $dbh = $self->{Cxn}->dbh(); $self->{first_lower} = $dbh->selectrow_arrayref($self->{first_lb_sql}); - PTDEBUG && _d('First lower boundary:', Dumper($self->{first_lower})); + PTDEBUG && _d('First lower boundary:', Dumper($self->{first_lower})); if ( my $nibble = $self->{resume} ) { if ( defined $nibble->{lower_boundary} @@ -6985,9 +6982,9 @@ sub _get_bounds { } } else { - $self->{next_lower} = $self->{first_lower}; + $self->{next_lower} = $self->{first_lower}; } - PTDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower})); + PTDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower})); if ( !$self->{next_lower} ) { PTDEBUG && _d('At end of table, or no more boundaries to resume'); @@ -7073,7 +7070,7 @@ sub _next_boundaries { $self->{upper} = $dbh->selectrow_arrayref($self->{last_ub_sql}); PTDEBUG && _d('Last upper boundary:', Dumper($self->{upper})); $self->{no_more_boundaries} = 1; # for next call - + $self->{last_upper} = $self->{upper}; } $self->{ub_sth}->finish(); @@ -7191,7 +7188,7 @@ sub new { . $tail_sql . " /*past upper chunk*/"; PTDEBUG && _d('Past upper statement:', $past_upper_sql); - + my $explain_past_upper_sql = "EXPLAIN SELECT " . ($args{past_select} @@ -7376,7 +7373,7 @@ sub run { $parent_exit->($child_pid) if $parent_exit; exit 0; } - + POSIX::setsid() or die "Cannot start a new session: $OS_ERROR"; chdir '/' or die "Cannot chdir to /: $OS_ERROR"; @@ -7402,7 +7399,7 @@ sub run { close STDERR; open STDERR, ">&STDOUT" - or die "Cannot dupe STDERR to STDOUT: $OS_ERROR"; + or die "Cannot dupe STDERR to STDOUT: $OS_ERROR"; } else { if ( -t STDOUT ) { @@ -7440,7 +7437,7 @@ sub _make_pid_file { eval { sysopen(PID_FH, $pid_file, O_RDWR|O_CREAT|O_EXCL) or die $OS_ERROR; print PID_FH $PID, "\n"; - close PID_FH; + close PID_FH; }; if ( my $e = $EVAL_ERROR ) { if ( $e =~ m/file exists/i ) { @@ -7925,10 +7922,10 @@ sub table_is_allowed { |slave_master_info |slave_relay_log_info |slave_worker_info - |slow_log + |slow_log )$/x; - if ( $filter->{'ignore-tables'}->{'*'}->{$tbl} + if ( $filter->{'ignore-tables'}->{'*'}->{$tbl} || $filter->{'ignore-tables'}->{$db}->{$tbl}) { PTDEBUG && _d('Table', $tbl, 'is in --ignore-tables list'); return 0; @@ -7941,7 +7938,7 @@ sub table_is_allowed { } if ( $filter->{'tables'} - && (!$filter->{'tables'}->{'*'}->{$tbl} && !$filter->{'tables'}->{$db}->{$tbl}) ) { + && (!$filter->{'tables'}->{'*'}->{$tbl} && !$filter->{'tables'}->{$db}->{$tbl}) ) { PTDEBUG && _d('Table', $tbl, 'is not in --tables list, ignoring'); return 0; } @@ -8425,7 +8422,7 @@ sub value_to_json { my $b_obj = B::svref_2object(\$value); # for round trip problem my $flags = $b_obj->FLAGS; - return $value # as is + return $value # as is if $flags & ( B::SVp_IOK | B::SVp_NOK ) and !( $flags & B::SVp_POK ); # SvTYPE is IV or NV? my $type = ref($value); @@ -8704,7 +8701,7 @@ sub wait { }; } - my @lagged_slaves = map { {cxn=>$_, lag=>undef} } @$slaves; + my @lagged_slaves = map { {cxn=>$_, lag=>undef} } @$slaves; while ( $oktorun->() && @lagged_slaves ) { PTDEBUG && _d('Checking slave lag'); for my $i ( 0..$#lagged_slaves ) { @@ -8884,13 +8881,13 @@ sub _parse_config { } handle_special_vars(\%config_data); - + return %config_data; } sub handle_special_vars { my ($config_data) = @_; - + if ( $config_data->{vars}->{wsrep_provider_options} ) { my $vars = $config_data->{vars}; my $dupes = $config_data->{duplicate_vars}; @@ -8953,7 +8950,7 @@ sub _parse_config_output { vars => $vars, ); } - + return ( format => $format, vars => $vars, @@ -9027,7 +9024,7 @@ sub parse_mysqld { # First look for the list of option files like # Default options are read from the following files in the given order: - # /etc/my.cnf /usr/local/mysql/etc/my.cnf ~/.my.cnf + # /etc/my.cnf /usr/local/mysql/etc/my.cnf ~/.my.cnf my @opt_files; if ( $output =~ m/^Default options are read.+\n/mg ) { my ($opt_files) = $output =~ m/\G^(.+)\n/m; @@ -9050,7 +9047,7 @@ sub parse_mysqld { # It also ends with something like # # wait_timeout 28800 - # + # # To see what values a running MySQL server is using, type # 'mysqladmin variables' instead of 'mysqld --verbose --help'. # @@ -9136,7 +9133,7 @@ sub _preprocess_varvals { } my ($var, $val) = ($1, $2); - + # Variable names are usually specified like "log-bin" # but in SHOW VARIABLES they're all like "log_bin". $var =~ tr/-/_/; @@ -9147,7 +9144,7 @@ sub _preprocess_varvals { if ( !defined $val ) { $val = ''; } - + # Strip leading and trailing whitespace. for my $item ($var, $val) { $item =~ s/^\s+//; @@ -9169,7 +9166,7 @@ sub _parse_varvals { # Config built from parsing the given varvals. my %config; - # Discover duplicate vars. + # Discover duplicate vars. my %duplicates; while ( my ($var, $vals) = each %$vars ) { @@ -9249,7 +9246,7 @@ sub _mimic_show_variables { die "I need a $arg arugment" unless $args{$arg}; } my ($vars, $format) = @args{@required_args}; - + foreach my $var ( keys %$vars ) { if ( $vars->{$var} eq '' ) { if ( $format eq 'mysqld' ) { @@ -9460,7 +9457,7 @@ sub _parse_spec { } } - return \%max_val_for; + return \%max_val_for; } sub max_values { @@ -9999,7 +9996,7 @@ our %PTC_EXIT_STATUS = ( # Completely ignore these error codes. my %ignore_code = ( # Error: 1592 SQLSTATE: HY000 (ER_BINLOG_UNSAFE_STATEMENT) - # Message: Statement may not be safe to log in statement format. + # Message: Statement may not be safe to log in statement format. # Ignore this warning because we have purposely set statement-based # replication. 1592 => 1, @@ -10024,7 +10021,7 @@ my %warn_code = ( sub main { # Reset global vars else tests will fail in strange ways. local @ARGV = @_; - $oktorun = 1; + $oktorun = 1; $print_header = 1; $exit_status = 0; @@ -10175,7 +10172,7 @@ sub main { die "Error setting SQL_MODE" . ": $EVAL_ERROR"; } - + # https://bugs.launchpad.net/percona-toolkit/+bug/919352 # The tool shouldn't blindly attempt to change binlog_format; @@ -10275,7 +10272,7 @@ sub main { } my @ignored_engines = keys %{$o->get('ignore-engines')}; - my @rocksdb_ignored = grep(/^ROCKSDB$/i, @ignored_engines); + my @rocksdb_ignored = grep(/^ROCKSDB$/i, @ignored_engines); if (!@rocksdb_ignored) { print STDOUT "Checking if all tables can be checksummed ...\n"; my $mysql_config = MySQLConfig->new(dbh => $master_dbh); @@ -10373,7 +10370,7 @@ sub main { # ######################################################################## # Create --plugin. - # ######################################################################## + # ######################################################################## my $plugin; if ( my $file = $o->get('plugin') ) { die "--plugin file $file does not exist\n" unless -f $file; @@ -10513,7 +10510,7 @@ sub main { die $master_cxn->name() . " is a cluster node but no other nodes " . "or regular replicas were found. Use --recursion-method=dsn " . "to specify the other nodes in the cluster.\n"; - } + } } # Make sure the master and all node are in the same cluster. @@ -10675,7 +10672,7 @@ sub main { # ##################################################################### if ( $o->get('replicate-check') && $o->get('replicate-check-only') ) { PTDEBUG && _d('Will --replicate-check and exit'); - + # --plugin hook if ( $plugin && $plugin->can('before_replicate_check') ) { $plugin->before_replicate_check(); @@ -10691,7 +10688,7 @@ sub main { $diffs = filter_tables_replicate_check_only($diffs, $o); if ( @$diffs ) { $exit_status |= $PTC_EXIT_STATUS{TABLE_DIFF}; - if ( $o->get('quiet') < 2 ) { + if ( $o->get('quiet') < 2 ) { print_checksum_diffs( cxn => $slave, diffs => $diffs, @@ -10785,8 +10782,8 @@ sub main { }; my $get_lag; - # The plugin is able to override the slavelag check so tools like - # pt-heartbeat or other replicators (Tungsten...) can be used to + # The plugin is able to override the slavelag check so tools like + # pt-heartbeat or other replicators (Tungsten...) can be used to # measure replication lag if ( $plugin && $plugin->can('get_slave_lag') ) { $get_lag = $plugin->get_slave_lag(oktorun => \$oktorun); @@ -10816,7 +10813,7 @@ sub main { return $slave_lag; }; } - + $replica_lag = new ReplicaLagWaiter( slaves => $slave_lag_cxns, max_lag => $o->get('max-lag'), @@ -10856,7 +10853,7 @@ sub main { . "SHOW GLOBAL STATUS. Current value for this option is:\n" . " --max-load " . (join(',', @{$o->get('max-load')})) . "\n"; } - + if ( $o->get('progress') ) { $replica_lag_pr = new Progress( jobsize => scalar @$slaves, @@ -10944,7 +10941,7 @@ sub main { my $total_rate = 0; my $tn = new TableNibbler(TableParser => $tp, Quoter => $q); my $retry = new Retry(); - + # --chunk-size-limit has two purposes. The 1st, as documented, is # to prevent oversized chunks when the chunk index is not unique. # The 2nd is to determine if the table can be processed in one chunk @@ -10988,7 +10985,7 @@ sub main { print "Resuming from $tbl->{db}.$tbl->{tbl} chunk " . "$last_chunk->{chunk}, timestamp $last_chunk->{ts}\n"; } - } + } else { # Problem resuming or no next lower boundary. PTDEBUG && _d('No more chunks; resuming from next table'); @@ -11134,7 +11131,7 @@ sub main { elsif ( !$key_len ) { die "The key_len of the $key index is " . (defined $key_len ? "zero" : "NULL") - . ", but this should not be possible. " + . ", but this should not be possible. " . "See --[no]check-plan in the documentation for more " . "information."; } @@ -11275,7 +11272,7 @@ sub main { my (%args) = @_; my $tbl = $args{tbl}; my $nibble_iter = $args{NibbleIterator}; - + # Don't need to do anything here if we're just --explain'ing. return if $o->get('explain'); @@ -11401,8 +11398,8 @@ sub main { # Wait for the last checksum of this table to replicate # to each slave. - # MySQL 8+ replication is slower than 5.7 and the old wait_for_last_checksum alone - # was failing. The new wait_for_slaves checks that Read_Master_Log_Pos on slaves is + # MySQL 8+ replication is slower than 5.7 and the old wait_for_last_checksum alone + # was failing. The new wait_for_slaves checks that Read_Master_Log_Pos on slaves is # greather or equal Position in the master if (!$args{Cxn}->is_cluster_node()) { wait_for_slaves(master_dbh => $args{Cxn}->dbh(), master_slave => $ms, slaves => $slaves); @@ -11437,10 +11434,11 @@ sub main { map { $diff_chunks{ $_->{chunk} }++ } @$diffs; $exit_status |= $PTC_EXIT_STATUS{TABLE_DIFF}; } + my $max_cnt_diff=0; for my $diff (@$diffs) { - if ( $diff->{cnt_diff} > $max_cnt_diff ) { - $tbl->{checksum_results}->{max_rows_cnt_diff} = $diff->{cnt_diff}; + if (abs($diff->{cnt_diff}) > $max_cnt_diff) { + $tbl->{checksum_results}->{max_rows_cnt_diff} = abs($diff->{cnt_diff}); } } }; @@ -11472,7 +11470,7 @@ sub main { # ######################################################################## # Init the --plugin. - # ######################################################################## + # ######################################################################## # --plugin hook if ( $plugin && $plugin->can('init') ) { @@ -11601,7 +11599,7 @@ sub main { # from the done callback, uses this start time. $tbl->{checksum_results}->{start_time} = time; 1 while $nibble_iter->next(); - + # --plugin hook if ( $plugin && $plugin->can('after_checksum_table') ) { $plugin->after_checksum_table(); @@ -11690,7 +11688,7 @@ sub nibble_is_safe { # Ensure that MySQL is using the chunk index if the table is being chunked. if ( !$nibble_iter->one_nibble() - && lc($expl->{key} || '') ne lc($nibble_iter->nibble_index() || '') ) { + && lc($expl->{key} || '') ne lc($nibble_iter->nibble_index() || '') ) { if ( !$tbl->{warned}->{not_using_chunk_index}++ && $o->get('quiet') < 2 ) { warn ts("Skipping chunk " . $nibble_iter->nibble_number() @@ -11705,7 +11703,7 @@ sub nibble_is_safe { # Ensure that the chunk isn't too large if there's a --chunk-size-limit. # If single-chunking the table, this has already been checked, so it # shouldn't have changed. If chunking the table with a non-unique key, - # oversize chunks are possible. + # oversize chunks are possible. if ( my $limit = $o->get('chunk-size-limit') ) { my $oversize_chunk = ($expl->{rows} || 0) >= $tbl->{chunk_size} * $limit; if ( $oversize_chunk @@ -11773,7 +11771,7 @@ sub exec_nibble { # ################################################################### # Start timing the checksum query. # ################################################################### - my $t_start = time; + my $t_start = time; # Execute the REPLACE...SELECT checksum query. # XXX This call and others like it are relying on a Perl oddity. @@ -11923,7 +11921,7 @@ sub print_checksum_results { ts(), $res->{errors} || 0, $res->{diffs} || 0, - $res->{n_rows} || 0, + $res->{n_rows} || 0, $tbl->{checksum_results}->{max_rows_cnt_diff} || 0, $res->{n_chunks} || 0, $res->{skipped} || 0, @@ -12056,7 +12054,7 @@ sub check_repl_table { } } - + # USE the correct db (probably the repl db, but maybe --replicate-database). use_repl_db(%args); @@ -12170,11 +12168,11 @@ sub check_repl_table { if ( $o->get('binary-index') ) { PTDEBUG && _d('--binary-index : checking if replicate table has binary type columns'); my $create_table = $tp->get_create_table( $dbh, $db, $tbl ); - if ( $create_table !~ /lower_boundary`?\s+BLOB/si - || $create_table !~ /upper_boundary`?\s+BLOB/si ) + if ( $create_table !~ /lower_boundary`?\s+BLOB/si + || $create_table !~ /upper_boundary`?\s+BLOB/si ) { - die "--binary-index was specified but the current checksum table ($db.$tbl) uses" - ." TEXT columns. To use BLOB columns, drop the current checksum table, then recreate" + die "--binary-index was specified but the current checksum table ($db.$tbl) uses" + ." TEXT columns. To use BLOB columns, drop the current checksum table, then recreate" ." it by specifying --create-replicate-table --binary-index."; } } @@ -12362,7 +12360,7 @@ sub create_repl_table { die "I need a $arg argument" unless $args{$arg}; } my ($dbh, $repl_table, $o) = @args{@required_args}; - PTDEBUG && _d('Creating --replicate table', $repl_table); + PTDEBUG && _d('Creating --replicate table', $repl_table); my $sql = $o->read_para_after(__FILE__, qr/MAGIC_create_replicate/); $sql =~ s/CREATE TABLE checksums/CREATE TABLE IF NOT EXISTS $repl_table/; $sql =~ s/;$//; @@ -12464,7 +12462,7 @@ sub have_more_chunks { if (lc($last_chunk->{chunk_index} || '') ne $chunk_index) { warn ts("Cannot resume from table $tbl->{db}.$tbl->{tbl} chunk " . "$last_chunk->{chunk} because the chunk indexes are different: " - . ($last_chunk->{chunk_index} ? $last_chunk->{chunk_index} + . ($last_chunk->{chunk_index} ? $last_chunk->{chunk_index} : "no index") . " was used originally but " . ($chunk_index ? $chunk_index : "no index") @@ -12815,11 +12813,11 @@ can try something like the following: ' AND ', upper_boundary), '1=1'); Take into consideration that by default, pt-table-checksum use C checksums. -C is not a cryptographic algorithm and for that reason it is prone to have +C is not a cryptographic algorithm and for that reason it is prone to have collisions. On the other hand, C algorithm is faster and less CPU-intensive than C and C. -Related reading material: +Related reading material: Percona Toolkit UDFs: L How to avoid hash collisions when using MySQL’s CRC32 function: L @@ -12932,11 +12930,11 @@ More Information: (L. @@ -12968,6 +12966,11 @@ are printed. The number of rows selected and checksummed from the table. It might be different from the number of rows in the table if you use the --where option. +=item DIFF_ROWS + +The maximum number of differences per chunk. If a chunk has 2 different rows and +another chunk has 3 different rows, this value will be 3. + =item CHUNKS The number of chunks into which the table was divided. @@ -13114,9 +13117,9 @@ See "Replicas using row-based replication" under L<"LIMITATIONS">. =item --binary-index This option modifies the behavior of L<"--create-replicate-table"> such that the -replicate table's upper and lower boundary columns are created with the BLOB +replicate table's upper and lower boundary columns are created with the BLOB data type. -This is useful in cases where you have trouble checksuming tables with keys that +This is useful in cases where you have trouble checksuming tables with keys that include a binary data type or that have non-standard character sets. See L<"--replicate">. @@ -13405,30 +13408,6 @@ the values are converted to strings by the CONCAT() function, and MySQL chooses the string representation. If you specify a value of 2, for example, then the values 1.008 and 1.009 will be rounded to 1.01, and will checksum as equal. -=item --force-concat-enums - -The NibbleIterator in Percona Toolkit can detect indexes having ENUM fields and -if the items it has are sorted or not. According to MySQL documentation at -L: - -ENUM values are sorted based on their index numbers, which depend on the order in -which the enumeration members were listed in the column specification. -For example, 'b' sorts before 'a' for ENUM('b', 'a'). -The empty string sorts before nonempty strings, and NULL values sort before all other -enumeration values. - -To prevent unexpected results when using the ORDER BY clause on an ENUM column, -use one of these techniques: -- Specify the ENUM list in alphabetic order. -- Make sure that the column is sorted lexically rather than by index number by coding -ORDER BY CAST(col AS CHAR) or ORDER BY CONCAT(col). - -The NibbleIterator in Percona Toolkit uses CONCAT(col) but, doing that, adds overhead -since MySQL cannot use the column directly and has to calculate the result of CONCAT -for every row. -To make this scenario vissible to the user, if there are indexes having ENUM fields -with usorted items, it is necessary to specify the C<--force-concat-enums> parameter. - =item --function type: string @@ -13558,7 +13537,7 @@ If password contains commas they must be escaped with a backslash: "exam\,ple" =item --pause-file -type: string +type: string Execution will be paused while the file specified by this param exists. @@ -13630,7 +13609,7 @@ performs several L<"REPLICA CHECKS"> before and while running. Although replicas are not required to run pt-table-checksum, the tool cannot detect diffs on slaves that it cannot discover. Therefore, -a warning is printed and the L<"EXIT STATUS"> is non-zero if no replicas +a warning is printed and the L<"EXIT STATUS"> is non-zero if no replicas are found and the method is not C. If this happens, try a different recursion method, or use the C method to specify the replicas to check. @@ -13678,12 +13657,12 @@ DSNs are ordered by C, but C and C are otherwise ignored. The C column contains a replica DSN like it would be given on the command line, for example: C<"h=replica_host,u=repl_user,p=repl_pass">. -The C method makes the tool ignore all slaves and cluster nodes. This -method is not recommended because it effectively disables the -L<"REPLICA CHECKS"> and no differences can be found. It is useful, however, if -you only need to write checksums on the master or a single cluster node. The -safer alternative is C<--no-replicate-check>: the tool finds replicas and -cluster nodes, performs the L<"REPLICA CHECKS">, but does not check for +The C method makes the tool ignore all slaves and cluster nodes. This +method is not recommended because it effectively disables the +L<"REPLICA CHECKS"> and no differences can be found. It is useful, however, if +you only need to write checksums on the master or a single cluster node. The +safer alternative is C<--no-replicate-check>: the tool finds replicas and +cluster nodes, performs the L<"REPLICA CHECKS">, but does not check for differences. See L<"--[no]replicate-check">. =item --replicate @@ -13752,7 +13731,7 @@ type: int; default: 1 Retry checksum comparison this many times when a difference is encountered. Only when a difference persists after this number of checks is it considered valid. -Using this option with a value of 2 or more alleviates spurious differences that +Using this option with a value of 2 or more alleviates spurious differences that arise when using the --resume option. =item --replicate-database @@ -13849,10 +13828,10 @@ Socket file to use for connection. =item --slave-skip-tolerance -type: float; default: 1.0 +type: float; default: 1.0 When a master table is marked to be checksumed in only one chunk but a slave -table exceeds the maximum accepted size for this, the table is skipped. +table exceeds the maximum accepted size for this, the table is skipped. Since number of rows are often rough estimates, many times tables are skipped needlessly for very small differences. This option provides a max row excess tolerance to prevent this. @@ -13875,16 +13854,16 @@ Checksum only tables whose names match this Perl regex. Add TRIM() to VARCHAR columns (helps when comparing 4.1 to >= 5.0). This is useful when you don't care about the trailing space differences between -MySQL versions that vary in their handling of trailing spaces. MySQL 5.0 and -later all retain trailing spaces in VARCHAR, while previous versions would +MySQL versions that vary in their handling of trailing spaces. MySQL 5.0 and +later all retain trailing spaces in VARCHAR, while previous versions would remove them. These differences will cause false checksum differences. =item --truncate-replicate-table Truncate the replicate table before starting the checksum. This parameter differs from L<--empty-replicate-table> which only deletes the rows -for the table being checksumed when starting the checksum for that table, while -L<--truncate-replicate-table> will truncate the replicate table at the beginning of the +for the table being checksumed when starting the checksum for that table, while +L<--truncate-replicate-table> will truncate the replicate table at the beginning of the process and thus, all previous checksum information will be losti, even if the process stops due to an error. @@ -13922,7 +13901,7 @@ done for the first time. Any updates or known problems are printed to STDOUT before the tool's normal output. This feature should never interfere with the normal operation of the -tool. +tool. For more information, visit L. diff --git a/t/pt-table-checksum/pt-1766.t b/t/pt-table-checksum/pt-1766.t new file mode 100644 index 00000000..8d77b6ad --- /dev/null +++ b/t/pt-table-checksum/pt-1766.t @@ -0,0 +1,69 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +use strict; +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More; + +use PerconaTest; +use Sandbox; +use SqlModes; +use threads; +use Time::HiRes qw( usleep ); +use constant PTDEBUG => $ENV{PTDEBUG} || 0; + +require "$trunk/bin/pt-table-checksum"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $dbh = $sb->get_dbh_for('master'); +my $slave1_dbh = $sb->get_dbh_for('slave1'); + +if ( !$dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} +else { + plan tests => 2; +} + +my $num_rows = 1000; +my $table = 't1'; + +$dbh->do("DROP DATABASE IF EXISTS test"); +$dbh->do("CREATE DATABASE IF NOT EXISTS test"); +$dbh->do("CREATE TABLE `test`.`$table` (id INT PRIMARY KEY AUTO_INCREMENT, name VARCHAR(5)) Engine=InnoDB"); + +diag(`util/mysql_random_data_load --host=127.0.0.1 --port=12345 --user=msandbox --password=msandbox test $table $num_rows`); +$slave1_dbh->do("DELETE FROM `test`.`$table` WHERE 1=1"); + +diag("Starting checksum"); +# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic +# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die. +# And --max-load "" prevents waiting for status variables. +my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox'; +my @args = ($master_dsn, qw(--no-check-binlog-format --chunk-size 10)); +my $output; + +$output = output( + sub { pt_table_checksum::main(@args) }, + stderr => 1, +); + +like( + $output, + qr/100\s+1000\s+10\s+102\s+0\s+\d+\.\d+\s+test.t1/, + "Truncating tables while checksum is running" +); + +# ############################################################################# +# Done. +# ############################################################################# +$sb->wipe_clean($dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +exit;