percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-09 17:21:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

First working enhanced --set-vars in pt-osc: no "default: ...", instead default values are under MAGIC_set_vars, picked up by OptionParser and passed to DSNParser. Also increaes --retries from 3 to 10. TODO: more testing.

Browse Source
This commit is contained in:
Daniel Nichter
2013-02-27 18:47:00 -07:00
parent 934709abad
commit c6958b497d
13 changed files with 217 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
bin/pt-online-schema-change
View File

@@ -80,6 +80,7 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use List::Util qw(max);
use Getopt::Long;
use Data::Dumper;
my $POD_link_re = '[LC]<"?([^">]+)"?>';
@@ -1063,6 +1064,45 @@ sub _parse_synopsis {
);
};
sub set_vars {
my ($self, $file) = @_;
$file ||= $self->{file} || __FILE__;
my %user_vars;
my $user_vars = $self->has('set-vars') ? $self->get('set-vars') : undef;
if ( $user_vars ) {
foreach my $var_val ( @$user_vars ) {
my ($var, $val) = $var_val =~ m/([^\s=]+)=(\S+)/;
die "Invalid --set-vars value: $var_val\n" unless $var && $val;
$user_vars{$var} = {
val => $val,
default => 0,
};
}
}
my %default_vars;
my $default_vars = $self->read_para_after($file, qr/MAGIC_set_vars/);
if ( $default_vars ) {
%default_vars = map {
my $var_val = $_;
my ($var, $val) = $var_val =~ m/([^\s=]+)=(\S+)/;
die "Invalid --set-vars value: $var_val\n" unless $var && $val;
$var => {
val => $val,
default => 1,
};
} split("\n", $default_vars);
}
my %vars = (
%default_vars, # first the tool's defaults
%user_vars, # then the user's which overwrite the defaults
);
PTDEBUG && _d('--set-vars:', Dumper(\%vars));
return \%vars;
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
@@ -2193,7 +2233,7 @@ sub get_dbh {
if ( my ($charset) = $cxn_string =~ m/charset=([\w]+)/ ) {
$sql = qq{/*!40101 SET NAMES "$charset"*/};
PTDEBUG && _d($dbh, ':', $sql);
PTDEBUG && _d($dbh, $sql);
eval { $dbh->do($sql) };
if ( $EVAL_ERROR ) {
die "Error setting NAMES to $charset: $EVAL_ERROR";
@@ -2208,13 +2248,8 @@ sub get_dbh {
}
}
if ( my $var = $self->prop('set-vars') ) {
$sql = "SET $var";
PTDEBUG && _d($dbh, ':', $sql);
eval { $dbh->do($sql) };
if ( $EVAL_ERROR ) {
die "Error setting $var: $EVAL_ERROR";
}
if ( my $vars = $self->prop('set-vars') ) {
$self->set_vars($dbh, $vars);
}
$sql = 'SET @@SQL_QUOTE_SHOW_CREATE = 1'
@@ -2289,6 +2324,55 @@ sub copy {
return \%new_dsn;
}
sub set_vars {
my ($self, $dbh, $vars) = @_;
foreach my $var ( sort keys %$vars ) {
my $val = $vars->{$var}->{val};
(my $quoted_var = $var) =~ s/_/\\_/;
my ($var_exists, $current_val);
eval {
($var_exists, $current_val) = $dbh->selectrow_array(
"SHOW VARIABLES LIKE '$quoted_var'");
};
my $e = $EVAL_ERROR;
if ( $e ) {
PTDEBUG && _d($e);
}
if ( $vars->{$var}->{default} && !$var_exists ) {
PTDEBUG && _d('Not setting default var', $var,
'because it does not exist');
next;
}
if ( $current_val && $current_val eq $val ) {
PTDEBUG && _d('Not setting var', $var, 'because its value',
'is already', $val);
next;
}
my $sql = "SET SESSION $var=$val";
PTDEBUG && _d($dbh, $sql);
eval { $dbh->do($sql) };
if ( my $set_error = $EVAL_ERROR ) {
chomp($set_error);
$set_error =~ s/ at \S+ line \d+//;
my $msg = "Error setting $var: $set_error";
if ( $current_val ) {
$msg .= " The current value for $var is $current_val. "
. "If the variable is read only (not dynamic), specify "
. "--set-vars $var=$current_val to avoid this warning, "
. "else manually set the variable and restart MySQL.";
}
warn $msg . "\n\n";
}
}
return;
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
@@ -7876,7 +7960,7 @@ sub main {
$o->get_opts();
my $dp = $o->DSNParser();
$dp->prop('set-vars', $o->get('set-vars'));
$dp->prop('set-vars', $o->set_vars());
# The original table, i.e. the one being altered, must be specified
# on the command line via the DSN.
@@ -7981,37 +8065,7 @@ sub main {
# ########################################################################
my $set_on_connect = sub {
my ($dbh) = @_;
# See the same code in pt-table-checksum.
my $lock_wait_timeout = $o->get('lock-wait-timeout');
my $set_lwt = "SET SESSION innodb_lock_wait_timeout=$lock_wait_timeout";
PTDEBUG && _d($set_lwt);
eval {
$dbh->do($set_lwt);
};
if ( $EVAL_ERROR ) {
PTDEBUG && _d($EVAL_ERROR);
# Get the server's current value.
my $sql = "SHOW SESSION VARIABLES LIKE 'innodb_lock_wait_timeout'";
PTDEBUG && _d($dbh, $sql);
my (undef, $curr_lwt) = $dbh->selectrow_array($sql);
PTDEBUG && _d('innodb_lock_wait_timeout on server:', $curr_lwt);
if ( !defined $curr_lwt ) {
PTDEBUG && _d('innodb_lock_wait_timeout does not exist;',
'InnoDB is probably disabled');
}
elsif ( $curr_lwt > $lock_wait_timeout ) {
warn "Failed to $set_lwt: $EVAL_ERROR\n"
. "The current innodb_lock_wait_timeout value "
. "$curr_lwt is greater than the --lock-wait-timeout "
. "value $lock_wait_timeout and the variable cannot be "
. "changed. innodb_lock_wait_timeout is only dynamic when "
. "using the InnoDB plugin. To prevent this warning, either "
. "specify --lock-wait-time=$curr_lwt, or manually set "
. "innodb_lock_wait_timeout to a value less than or equal "
. "to $lock_wait_timeout and restart MySQL.\n";
}
}
return;
};
# Do not call "new Cxn(" directly; use this sub so that set_on_connect
@@ -10769,18 +10823,6 @@ short form: -h; type: string
Connect to host.
=item --lock-wait-timeout
type: int; default: 1
Set the session value of C<innodb_lock_wait_timeout>. This option helps guard
against long lock waits if the data-copy queries become slow for some reason.
Setting this option dynamically requires the InnoDB plugin, so this works only
on newer InnoDB and MySQL versions. If the setting's current value is greater
than the specified value, and the tool cannot set the value as desired, then it
prints a warning. If the tool cannot set the value but the current value is less
than or equal to the desired value, there is no error.
=item --max-lag
type: time; default: 1s
@@ -10919,7 +10961,7 @@ ignored.
=item --retries
type: int; default: 3
type: int; default: 10
Retry a chunk this many times when there is a nonfatal error. Nonfatal errors
are problems such as a lock wait timeout or the query being killed. This option
@@ -10927,10 +10969,24 @@ applies to the data copy operation.
=item --set-vars
type: string; default: wait_timeout=10000
type: Array
Set these MySQL variables. Immediately after connecting to MySQL, this string
will be appended to SET and executed.
Set the MySQL variables in this comma-separated list of C<variable=value> pairs.
By default, the tool sets:
=for comment ignore-pt-internal-value
MAGIC_set_vars
wait_timeout=10000
innodb_lock_wait_timeout=1
lock_wait_timeout=60
Variables specified on the command line override these defaults. For
example, specifying C<--set-vars wait_timeout=500> overrides the default
value of C<10000>.
The tool prints a warning and continues if a variable cannot be set.
=item --socket

60
lib/DSNParser.pm
View File

@@ -346,7 +346,7 @@ sub get_dbh {
# Set character set and binmode on STDOUT.
if ( my ($charset) = $cxn_string =~ m/charset=([\w]+)/ ) {
$sql = qq{/*!40101 SET NAMES "$charset"*/};
PTDEBUG && _d($dbh, ':', $sql);
PTDEBUG && _d($dbh, $sql);
eval { $dbh->do($sql) };
if ( $EVAL_ERROR ) {
die "Error setting NAMES to $charset: $EVAL_ERROR";
@@ -361,13 +361,8 @@ sub get_dbh {
}
}
if ( my $var = $self->prop('set-vars') ) {
$sql = "SET $var";
PTDEBUG && _d($dbh, ':', $sql);
eval { $dbh->do($sql) };
if ( $EVAL_ERROR ) {
die "Error setting $var: $EVAL_ERROR";
}
if ( my $vars = $self->prop('set-vars') ) {
$self->set_vars($dbh, $vars);
}
# Do this after set-vars so a user-set sql_mode doesn't clobber it; See
@@ -450,6 +445,55 @@ sub copy {
return \%new_dsn;
}
sub set_vars {
my ($self, $dbh, $vars) = @_;
foreach my $var ( sort keys %$vars ) {
my $val = $vars->{$var}->{val};
(my $quoted_var = $var) =~ s/_/\\_/;
my ($var_exists, $current_val);
eval {
($var_exists, $current_val) = $dbh->selectrow_array(
"SHOW VARIABLES LIKE '$quoted_var'");
};
my $e = $EVAL_ERROR;
if ( $e ) {
PTDEBUG && _d($e);
}
if ( $vars->{$var}->{default} && !$var_exists ) {
PTDEBUG && _d('Not setting default var', $var,
'because it does not exist');
next;
}
if ( $current_val && $current_val eq $val ) {
PTDEBUG && _d('Not setting var', $var, 'because its value',
'is already', $val);
next;
}
my $sql = "SET SESSION $var=$val";
PTDEBUG && _d($dbh, $sql);
eval { $dbh->do($sql) };
if ( my $set_error = $EVAL_ERROR ) {
chomp($set_error);
$set_error =~ s/ at \S+ line \d+//;
my $msg = "Error setting $var: $set_error";
if ( $current_val ) {
$msg .= " The current value for $var is $current_val. "
. "If the variable is read only (not dynamic), specify "
. "--set-vars $var=$current_val to avoid this warning, "
. "else manually set the variable and restart MySQL.";
}
warn $msg . "\n\n";
}
}
return;
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }

79
lib/OptionParser.pm
View File

@@ -18,45 +18,6 @@
# OptionParser package
# ###########################################################################
{
# Package: OptionParser
# OptionParser parses command line options from a tool's POD. By default
# it parses a description and usage from the POD's SYNOPSIS section and
# command line options from the OPTIONS section.
#
# The SYNOPSIS section should look like,
# (start code)
# =head1 SYNOPSIS
#
# Usage: mk-archiver [OPTION...] --source DSN --where WHERE
#
# mk-archiver nibbles records from a MySQL table. The --source and --dest
# arguments use DSN syntax; if COPY is yes, --dest defaults to the key's value
# from --source.
#
# Examples:
# ...
# (end code)
# The key, required parts are the "Usage:" line and the following description
# paragraph.
#
# The OPTIONS section shoud look like,
# (start code)
# =head1 OPTIONS
#
# Optional rules, one per line.
#
# =over
#
# =item --analyze
#
# type: string
#
# Run ANALYZE TABLE afterwards on L<"--source"> and/or L<"--dest">.
# ect.
# (end code)
# The option's full name is given as the "=item". The next, optional para
# is the option's attributes. And the next, required para is the option's
# description (the first period-terminated sentence).
package OptionParser;
use strict;
@@ -66,6 +27,7 @@ use constant PTDEBUG => $ENV{PTDEBUG} || 0;
use List::Util qw(max);
use Getopt::Long;
use Data::Dumper;
my $POD_link_re = '[LC]<"?([^">]+)"?>';
@@ -1318,6 +1280,45 @@ sub _parse_synopsis {
);
};
sub set_vars {
my ($self, $file) = @_;
$file ||= $self->{file} || __FILE__;
my %user_vars;
my $user_vars = $self->has('set-vars') ? $self->get('set-vars') : undef;
if ( $user_vars ) {
foreach my $var_val ( @$user_vars ) {
my ($var, $val) = $var_val =~ m/([^\s=]+)=(\S+)/;
die "Invalid --set-vars value: $var_val\n" unless $var && $val;
$user_vars{$var} = {
val => $val,
default => 0,
};
}
}
my %default_vars;
my $default_vars = $self->read_para_after($file, qr/MAGIC_set_vars/);
if ( $default_vars ) {
%default_vars = map {
my $var_val = $_;
my ($var, $val) = $var_val =~ m/([^\s=]+)=(\S+)/;
die "Invalid --set-vars value: $var_val\n" unless $var && $val;
$var => {
val => $val,
default => 1,
};
} split("\n", $default_vars);
}
my %vars = (
%default_vars, # first the tool's defaults
%user_vars, # then the user's which overwrite the defaults
);
PTDEBUG && _d('--set-vars:', Dumper(\%vars));
return \%vars;
}
sub _d {
my ($package, undef, $line) = caller 0;
@_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }

4
t/pt-online-schema-change/alter_active_table.t
View File

@@ -176,7 +176,7 @@ start_query_table(qw(pt_osc t id));
($output, $exit) = full_output(
sub { pt_online_schema_change::main(
"$master_dsn,D=pt_osc,t=t",
qw(--lock-wait-timeout 5),
qw(--set-vars innodb_lock_wait_timeout=5),
qw(--print --execute --chunk-size 100 --alter ENGINE=InnoDB)) },
stderr => 1,
);
@@ -226,7 +226,7 @@ start_query_table(qw(pt_osc t id));
($output, $exit) = full_output(
sub { pt_online_schema_change::main(
"$master_dsn,D=pt_osc,t=t",
qw(--lock-wait-timeout 5),
qw(--set-vars innodb_lock_wait_timeout=5),
qw(--print --execute --chunk-size 100 --no-check-alter),
'--alter', 'CHANGE COLUMN d q date',
) },

2
t/pt-online-schema-change/ansi_quotes.t
View File

@@ -31,7 +31,7 @@ if ( !$dbh ) {
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --lock-wait-timeout=3 else the tool will die.
my $master_dsn = 'h=127.1,P=12348,u=msandbox,p=msandbox';
my @args = (qw(--lock-wait-timeout 3));
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $sample = "t/pt-online-schema-change/samples/";

2
t/pt-online-schema-change/basics.t
View File

@@ -36,7 +36,7 @@ elsif ( !$slave_dbh ) {
my $q = new Quoter();
my $tp = new TableParser(Quoter => $q);
my @args = qw(--lock-wait-timeout 3);
my @args = qw(--set-vars innodb_lock_wait_timeout=3);
my $output = "";
my $dsn = "h=127.1,P=12345,u=msandbox,p=msandbox";
my $exit = 0;

2
t/pt-online-schema-change/bugs.t
View File

@@ -32,7 +32,7 @@ elsif ( !$slave_dbh ) {
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --lock-wait-timeout=3 else the tool will die.
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = (qw(--lock-wait-timeout 3));
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $sample = "t/pt-online-schema-change/samples/";

2
t/pt-online-schema-change/check_alter.t
View File

@@ -32,7 +32,7 @@ elsif ( !$slave_dbh ) {
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --lock-wait-timeout=3 else the tool will die.
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = (qw(--lock-wait-timeout 3));
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $sample = "t/pt-online-schema-change/samples/";

2
t/pt-online-schema-change/privs.t
View File

@@ -42,7 +42,7 @@ elsif ( !@{$master_dbh->selectall_arrayref("show databases like 'sakila'")} ) {
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --lock-wait-timeout=3 else the tool will die.
my $master_dsn = 'h=127.1,P=12345,p=msandbox';
my @args = (qw(--lock-wait-timeout 3));
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $row;
my $output;
my $exit_status;

6
t/pt-online-schema-change/pxc.t
View File

@@ -62,7 +62,7 @@ $sb->load_file('node1', "$sample/basic_no_fks.sql");
($output, $exit) = full_output(
sub { pt_online_schema_change::main(
"$node1_dsn,D=pt_osc,t=t",
qw(--lock-wait-timeout 5),
qw(--set-vars innodb_lock_wait_timeout=5),
qw(--print --execute --alter ENGINE=InnoDB)) },
stderr => 1,
);
@@ -87,7 +87,7 @@ $sb->load_file('node1', "$sample/basic_no_fks_innodb.sql");
($output, $exit) = full_output(
sub { pt_online_schema_change::main(
"$node1_dsn,D=pt_osc,t=t",
qw(--lock-wait-timeout 5),
qw(--set-vars innodb_lock_wait_timeout=5),
qw(--print --execute --alter ENGINE=MyISAM)) },
stderr => 1,
);
@@ -112,7 +112,7 @@ $node1->do("SET GLOBAL wsrep_OSU_method='RSU'");
($output, $exit) = full_output(
sub { pt_online_schema_change::main(
"$node1_dsn,D=pt_osc,t=t",
qw(--lock-wait-timeout 5),
qw(--set-vars innodb_lock_wait_timeout=5),
qw(--print --execute --alter ENGINE=MyISAM)) },
stderr => 1,
);

2
t/pt-online-schema-change/rename_columns.t
View File

@@ -32,7 +32,7 @@ elsif ( !$slave_dbh ) {
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --lock-wait-timeout=3 else the tool will die.
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = (qw(--lock-wait-timeout 3));
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $sample = "t/pt-online-schema-change/samples/";

2
t/pt-online-schema-change/sanity_checks.t
View File

@@ -31,7 +31,7 @@ if ( !$master_dbh ) {
my $q = new Quoter();
my $tp = new TableParser(Quoter => $q);
my @args = qw(--lock-wait-timeout 3);
my @args = qw(--set-vars innodb_lock_wait_timeout=3);
my $output = "";
my $dsn = "h=127.1,P=12345,u=msandbox,p=msandbox";
my $exit = 0;

2
t/pt-online-schema-change/skip_innodb.t
View File

@@ -31,7 +31,7 @@ if ( !$master_dbh ) {
}
my $master_dsn = 'h=127.1,P=12348,u=msandbox,p=msandbox';
my @args = (qw(--lock-wait-timeout 3), '--max-load', '');
my @args = (qw(--set-vars innodb_lock_wait_timeout=3), '--max-load', '');
my ($output, $retval) = full_output(
sub { pt_online_schema_change::main(@args,