percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-07 21:09:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '3.0' into PMM-1.3

Browse Source
This commit is contained in:
Kamil Dziedzic
2017-10-10 12:06:09 +02:00
parent 59add04961 b44138fd7c
commit 392fa04abd
27 changed files with 1056 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
bin/pt-archiver
View File

@@ -1965,8 +1965,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -1983,6 +1983,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -1991,24 +1996,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

49
bin/pt-duplicate-key-checker
View File

@@ -352,8 +352,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -370,6 +370,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -378,24 +383,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

49
bin/pt-find
View File

@@ -1880,8 +1880,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -1898,6 +1898,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -1906,24 +1911,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

49
bin/pt-heartbeat
View File

@@ -3499,8 +3499,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -3517,6 +3517,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -3525,24 +3530,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

49
bin/pt-index-usage
View File

@@ -3119,8 +3119,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -3137,6 +3137,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -3145,24 +3150,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

81
bin/pt-kill
View File

@@ -47,7 +47,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.0.4';
our $VERSION = '3.0.5';
use strict;
use warnings FATAL => 'all';
@@ -2945,8 +2945,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -2963,6 +2963,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -2971,24 +2976,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;
@@ -3297,6 +3312,17 @@ sub new {
foreach my $arg ( qw(MasterSlave) ) {
die "I need a $arg argument" unless $args{$arg};
}
my $kill_busy_commands = {};
if ($args{kill_busy_commands}) {
for my $command (split /,/,$args{kill_busy_commands}) {
$command =~ s/^\s+|\s+$//g;
$kill_busy_commands->{$command} = 1;
}
} else {
$kill_busy_commands->{Query} = 1;
}
$args{kill_busy_commands} = $kill_busy_commands;
my $self = {
%args,
polls => 0,
@@ -3523,7 +3549,7 @@ sub find {
next QUERY;
}
if ( $find_spec{busy_time} && ($query->{Command} || '') eq 'Query' ) {
if ( $find_spec{busy_time} && exists($self->{kill_busy_commands}->{$query->{Command} || ''}) ) {
next QUERY unless defined($query->{Time});
if ( $query->{Time} < $find_spec{busy_time} ) {
PTDEBUG && _d("Query isn't running long enough");
@@ -6788,7 +6814,7 @@ sub main {
DSNParser => $dp,
Quoter => "Quoter",
);
my $pl = new Processlist(MasterSlave => $ms);
my $pl = new Processlist(MasterSlave => $ms, kill_busy_commands => $o->get('kill-busy-commands'));
my $qr = new QueryRewriter();
my $cxn;
@@ -8198,6 +8224,21 @@ that pt-kill matched and killed a query.
See also L<"--wait-before-kill"> and L<"--wait-after-kill">.
=item --kill-busy-commands
default: Query
group: Actions
Comma sepatated list of commands that will be watched/killed if they ran for
more than L<"--busy-time"> seconds. Default: C<Query>
By default, L<"--busy-time"> kills only C<Query> commands but in some cases, it
is needed to make L<"--busy-time"> to watch and kill other commands. For example,
a prepared statement execution command is C<Execute> instead of C<Query>. In this
case, specifying C<--kill-busy-commands=Query,Execute> will also kill the prepared
stamente execution.
=item --kill-query
group: Actions

58
bin/pt-online-schema-change
View File

@@ -56,7 +56,7 @@ BEGIN {
{
package Percona::Toolkit;
our $VERSION = '3.0.4';
our $VERSION = '3.0.5';
use strict;
use warnings FATAL => 'all';
@@ -3309,8 +3309,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -3327,6 +3327,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -3335,24 +3340,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;
@@ -10111,7 +10126,7 @@ sub get_unique_index_fields {
$clean .= $suffix;
my $fields = [];
my $fields_re = qr/(?:PRIMARY|UNIQUE)\s*(?:INDEX|KEY|)\s*(?:.*?)\s*\((.*?)\)/i;
my $fields_re = qr/\s(?:PRIMARY|UNIQUE)\s+(?:INDEX|KEY|)\s*(?:.*?)\s*\((.*?)\)/i;
while($clean =~ /$fields_re/g) {
push @$fields, [ split /\s*,\s*/, $1 ];
@@ -11582,6 +11597,11 @@ The tool exits with an error if the host is a cluster node and the table
is MyISAM or is being converted to MyISAM (C<ENGINE=MyISAM>), or if
C<wsrep_OSU_method> is not C<TOI>. There is no way to disable these checks.
=head1 MySQL 5.7+ Generated columns
The tools ignores MySQL 5.7+ C<GENERATED> columns since the value for those columns
is generated according to the expresion used to compute column values.
=head1 OUTPUT
The tool prints information about its activities to STDOUT so that you can see

63
bin/pt-query-digest
View File

@@ -3251,6 +3251,17 @@ sub new {
foreach my $arg ( qw(MasterSlave) ) {
die "I need a $arg argument" unless $args{$arg};
}
my $kill_busy_commands = {};
if ($args{kill_busy_commands}) {
for my $command (split /,/,$args{kill_busy_commands}) {
$command =~ s/^\s+|\s+$//g;
$kill_busy_commands->{$command} = 1;
}
} else {
$kill_busy_commands->{Query} = 1;
}
$args{kill_busy_commands} = $kill_busy_commands;
my $self = {
%args,
polls => 0,
@@ -3465,6 +3476,7 @@ sub find {
my $ms = $self->{MasterSlave};
my @matches;
$self->{_reasons_for_matching} = undef;
QUERY:
foreach my $query ( @$proclist ) {
PTDEBUG && _d('Checking query', Dumper($query));
@@ -3476,7 +3488,7 @@ sub find {
next QUERY;
}
if ( $find_spec{busy_time} && ($query->{Command} || '') eq 'Query' ) {
if ( $find_spec{busy_time} && exists($self->{kill_busy_commands}->{$query->{Command} || ''}) ) {
next QUERY unless defined($query->{Time});
if ( $query->{Time} < $find_spec{busy_time} ) {
PTDEBUG && _d("Query isn't running long enough");
@@ -8845,8 +8857,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -8863,6 +8875,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -8871,24 +8888,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

115
bin/pt-table-checksum
View File

@@ -4446,8 +4446,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -4464,6 +4464,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -4472,24 +4477,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;
@@ -9331,36 +9346,35 @@ sub main {
die "Error setting SQL_MODE"
. ": $EVAL_ERROR";
}
if ( $o->get('check-binlog-format') ) {
# https://bugs.launchpad.net/percona-toolkit/+bug/919352
# The tool shouldn't blindly attempt to change binlog_format;
# instead, it should check if it's already set to STATEMENT.
# This is becase starting with MySQL 5.1.29, changing the format
# requires a SUPER user.
if ( VersionParser->new($dbh) >= '5.1.5' ) {
$sql = 'SELECT @@binlog_format';
PTDEBUG && _d($dbh, $sql);
my ($original_binlog_format) = $dbh->selectrow_array($sql);
PTDEBUG && _d('Original binlog_format:', $original_binlog_format);
if ( $original_binlog_format !~ /STATEMENT/i ) {
$sql = q{/*!50108 SET @@binlog_format := 'STATEMENT'*/};
eval {
PTDEBUG && _d($dbh, $sql);
$dbh->do($sql);
};
if ( $EVAL_ERROR ) {
die "Failed to $sql: $EVAL_ERROR\n"
. "This tool requires binlog_format=STATEMENT, "
. "but the current binlog_format is set to "
."$original_binlog_format and an error occurred while "
. "attempting to change it. If running MySQL 5.1.29 or newer, "
. "setting binlog_format requires the SUPER privilege. "
. "You will need to manually set binlog_format to 'STATEMENT' "
. "before running this tool.\n";
}
}
}
# https://bugs.launchpad.net/percona-toolkit/+bug/919352
# The tool shouldn't blindly attempt to change binlog_format;
# instead, it should check if it's already set to STATEMENT.
# This is becase starting with MySQL 5.1.29, changing the format
# requires a SUPER user.
if ( VersionParser->new($dbh) >= '5.1.5' ) {
$sql = 'SELECT @@binlog_format';
PTDEBUG && _d($dbh, $sql);
my ($original_binlog_format) = $dbh->selectrow_array($sql);
PTDEBUG && _d('Original binlog_format:', $original_binlog_format);
if ( $original_binlog_format !~ /STATEMENT/i ) {
$sql = q{/*!50108 SET @@binlog_format := 'STATEMENT'*/};
eval {
PTDEBUG && _d($dbh, $sql);
$dbh->do($sql);
};
if ( $EVAL_ERROR ) {
die "Failed to $sql: $EVAL_ERROR\n"
. "This tool requires binlog_format=STATEMENT, "
. "but the current binlog_format is set to "
."$original_binlog_format and an error occurred while "
. "attempting to change it. If running MySQL 5.1.29 or newer, "
. "setting binlog_format requires the SUPER privilege. "
. "You will need to manually set binlog_format to 'STATEMENT' "
. "before running this tool.\n";
}
}
}
# Set transaction isolation level. We set binlog_format to STATEMENT,
@@ -11845,6 +11859,15 @@ can try something like the following:
SET boundaries = COALESCE(CONCAT('id BETWEEN ', lower_boundary,
' AND ', upper_boundary), '1=1');
Take into consideration that by default, pt-table-checksum use C<CRC32> checksums.
C<CRC32> is not a cryptographic algorithm and for that reason it is prone to have
collisions. On the other hand, C<CRC32> algorithm is faster and less CPU-intensive
than C<MD5> and C<SHA1>.
Related reading material:
Percona Toolkit UDFs: L<https://www.percona.com/doc/percona-server/LATEST/management/udf_percona_toolkit.html>
How to avoid hash collisions when using MySQL’s CRC32 function: L<https://www.percona.com/blog/2014/10/13/how-to-avoid-hash-collisions-when-using-mysqls-crc32-function/>
=head1 LIMITATIONS
=over

49
bin/pt-table-sync
View File

@@ -2864,8 +2864,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -2882,6 +2882,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -2890,24 +2895,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

49
bin/pt-table-usage
View File

@@ -6795,8 +6795,8 @@ sub parse {
my %def_for;
@def_for{@cols} = @defs;
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -6813,6 +6813,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -6821,24 +6826,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
sub sort_indexes {
my ( $self, $tbl ) = @_;

17
lib/Processlist.pm
View File

@@ -69,6 +69,19 @@ sub new {
foreach my $arg ( qw(MasterSlave) ) {
die "I need a $arg argument" unless $args{$arg};
}
# Convert the list of kill commands (Query, Execute, etc) to a hashref for
# faster check later
my $kill_busy_commands = {};
if ($args{kill_busy_commands}) {
for my $command (split /,/,$args{kill_busy_commands}) {
$command =~ s/^\s+|\s+$//g;
$kill_busy_commands->{$command} = 1;
}
} else {
$kill_busy_commands->{Query} = 1;
}
$args{kill_busy_commands} = $kill_busy_commands;
my $self = {
%args,
polls => 0,
@@ -471,6 +484,7 @@ sub find {
my $ms = $self->{MasterSlave};
my @matches;
$self->{_reasons_for_matching} = undef;
QUERY:
foreach my $query ( @$proclist ) {
PTDEBUG && _d('Checking query', Dumper($query));
@@ -484,7 +498,8 @@ sub find {
}
# Match special busy_time.
if ( $find_spec{busy_time} && ($query->{Command} || '') eq 'Query' ) {
#if ( $find_spec{busy_time} && ($query->{Command} || '') eq 'Query' ) {
if ( $find_spec{busy_time} && exists($self->{kill_busy_commands}->{$query->{Command} || ''}) ) {
next QUERY unless defined($query->{Time});
if ( $query->{Time} < $find_spec{busy_time} ) {
PTDEBUG && _d("Query isn't running long enough");

49
lib/TableParser.pm
View File

@@ -159,8 +159,8 @@ sub parse {
# Find column types, whether numeric, whether nullable, whether
# auto-increment.
my (@nums, @null);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
my (@nums, @null, @non_generated);
my (%type_for, %is_nullable, %is_numeric, %is_autoinc, %is_generated);
foreach my $col ( @cols ) {
my $def = $def_for{$col};
@@ -180,6 +180,11 @@ sub parse {
push @null, $col;
$is_nullable{$col} = 1;
}
if ( remove_quoted_text($def) =~ m/\WGENERATED\W/i ) {
$is_generated{$col} = 1;
} else {
push @non_generated, $col;
}
$is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
}
@@ -191,24 +196,34 @@ sub parse {
my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
return {
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @cols },
null_cols => \@null,
is_nullable => \%is_nullable,
is_autoinc => \%is_autoinc,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
name => $name,
cols => \@cols,
col_posn => { map { $cols[$_] => $_ } 0..$#cols },
is_col => { map { $_ => 1 } @non_generated },
null_cols => \@null,
is_nullable => \%is_nullable,
non_generated_cols => \@non_generated,
is_autoinc => \%is_autoinc,
is_generated => \%is_generated,
clustered_key => $clustered_key,
keys => $keys,
defs => \%def_for,
numeric_cols => \@nums,
is_numeric => \%is_numeric,
engine => $engine,
type_for => \%type_for,
charset => $charset,
};
}
sub remove_quoted_text {
my ($string) = @_;
$string =~ s/[^\\]`[^`]*[^\\]`//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
$string =~ s/[^\\]"[^"]*[^\\]"//g;
return $string;
}
# Sorts indexes in this order: PRIMARY, unique, non-nullable, any (shortest
# first, alphabetical). Only BTREE indexes are considered.
# TODO: consider length as # of bytes instead of # of columns.

6
src/go/mongolib/profiler/profiler_test.go
View File

@@ -86,6 +86,7 @@ func TestRegularIterator(t *testing.T) {
},
}
prof.Start()
defer prof.Stop()
select {
case queries := <-prof.QueriesChan():
if !reflect.DeepEqual(queries, want) {
@@ -144,6 +145,7 @@ func TestIteratorTimeout(t *testing.T) {
}
prof.Start()
defer prof.Stop()
gotTimeout := false
// Get a timeout
@@ -168,8 +170,6 @@ func TestIteratorTimeout(t *testing.T) {
case <-time.After(2 * time.Second):
t.Error("Didn't get any query after 2 seconds")
}
prof.Stop()
}
func TestTailIterator(t *testing.T) {
@@ -239,6 +239,7 @@ func TestTailIterator(t *testing.T) {
},
}
prof.Start()
defer prof.Stop()
index := 0
// Since the mocked iterator has a Sleep(1500 ms) between Next methods calls,
// we are going to have two ticker ticks and on every tick it will return one document.
@@ -339,6 +340,7 @@ func TestCalcTotalStats(t *testing.T) {
prof := NewProfiler(iter, filters, nil, s)
prof.Start()
defer prof.Stop()
select {
case queries := <-prof.QueriesChan():
s := queries.CalcTotalQueriesStats(1)

2
src/go/mongolib/proto/system.profile.go
View File

@@ -7,11 +7,13 @@ import (
"gopkg.in/mgo.v2/bson"
)
// docsExamined is renamed from nscannedObjects in 3.2.0
type SystemProfile struct {
AllUsers []interface{} `bson:"allUsers"`
Client string `bson:"client"`
CursorExhausted bool `bson:"cursorExhausted"`
DocsExamined int `bson:"docsExamined"`
NscannedObjects int `bson:"nscannedObjects"`
ExecStats struct {
Advanced int `bson:"advanced"`
ExecutionTimeMillisEstimate int `bson:"executionTimeMillisEstimate"`

10
src/go/mongolib/stats/stats.go
View File

@@ -90,7 +90,12 @@ func (s *Stats) Add(doc proto.SystemProfile) error {
s.setQueryInfoAndCounters(key, qiac)
}
qiac.Count++
qiac.NScanned = append(qiac.NScanned, float64(doc.DocsExamined))
// docsExamined is renamed from nscannedObjects in 3.2.0.
if doc.NscannedObjects > 0 {
qiac.NScanned = append(qiac.NScanned, float64(doc.NscannedObjects))
} else {
qiac.NScanned = append(qiac.NScanned, float64(doc.DocsExamined))
}
qiac.NReturned = append(qiac.NReturned, float64(doc.Nreturned))
qiac.QueryTime = append(qiac.QueryTime, float64(doc.Millis))
qiac.ResponseLength = append(qiac.ResponseLength, float64(doc.ResponseLength))
@@ -269,7 +274,7 @@ func countersToStats(query QueryInfoAndCounters, uptime int64, tc totalCounters)
queryStats.QueryTime.Pct = queryStats.QueryTime.Total * 100 / tc.QueryTime
}
if tc.Bytes > 0 {
queryStats.ResponseLength.Pct = queryStats.ResponseLength.Total / tc.Bytes
queryStats.ResponseLength.Pct = queryStats.ResponseLength.Total * 100 / tc.Bytes
}
if queryStats.Returned.Total > 0 {
queryStats.Ratio = queryStats.Scanned.Total / queryStats.Returned.Total
@@ -281,6 +286,7 @@ func countersToStats(query QueryInfoAndCounters, uptime int64, tc totalCounters)
func aggregateCounters(queries []QueryInfoAndCounters) QueryInfoAndCounters {
qt := QueryInfoAndCounters{}
for _, query := range queries {
qt.Count += query.Count
qt.NScanned = append(qt.NScanned, query.NScanned...)
qt.NReturned = append(qt.NReturned, query.NReturned...)
qt.QueryTime = append(qt.QueryTime, query.QueryTime...)

11
src/go/pt-mongodb-query-digest/main.go
View File

@@ -165,7 +165,8 @@ func main() {
if opts.Limit > 0 && len(sortedQueryStats) > opts.Limit {
sortedQueryStats = sortedQueryStats[:opts.Limit]
}
for _, qs := range sortedQueryStats {
for i, qs := range sortedQueryStats {
qs.Rank = i + 1
t.Execute(os.Stdout, qs)
}
@@ -426,15 +427,11 @@ func sortQueries(queries []stats.QueryStats, orderby []string) []stats.QueryStat
case "ratio":
f = func(c1, c2 *stats.QueryStats) bool {
ratio1 := c1.Scanned.Max / c1.Returned.Max
ratio2 := c2.Scanned.Max / c2.Returned.Max
return ratio1 < ratio2
return c1.Ratio < c2.Ratio
}
case "-ratio":
f = func(c1, c2 *stats.QueryStats) bool {
ratio1 := c1.Scanned.Max / c1.Returned.Max
ratio2 := c2.Scanned.Max / c2.Returned.Max
return ratio1 > ratio2
return c1.Ratio > c2.Ratio
}
//

6
src/go/tests/profiler_docs_stats.want.json
View File

@@ -22,7 +22,7 @@
"Median": 0
},
"ResponseLength": {
"Pct": 0.9995949739087844,
"Pct": 99.95949739087844,
"Total": 1061230,
"Min": 1061230,
"Max": 1061230,
@@ -75,7 +75,7 @@
"Median": 7
},
"ResponseLength": {
"Pct": 0.00020251304560782172,
"Pct": 0.020251304560782172,
"Total": 215,
"Min": 215,
"Max": 215,
@@ -128,7 +128,7 @@
"Median": 0
},
"ResponseLength": {
"Pct": 0.00020251304560782172,
"Pct": 0.020251304560782172,
"Total": 215,
"Min": 215,
"Max": 215,

6
src/go/tests/profiler_docs_total_stats.want.json
View File

@@ -6,8 +6,8 @@
"Fingerprint": "",
"FirstSeen": "0001-01-01T00:00:00Z",
"LastSeen": "0001-01-01T00:00:00Z",
"Count": 0,
"QPS": 0,
"Count": 3,
"QPS": 3,
"Rank": 0,
"Ratio": 134.33333333333334,
"QueryTime": {
@@ -21,7 +21,7 @@
"Median": 0
},
"ResponseLength": {
"Pct": 1,
"Pct": 100,
"Total": 1061660,
"Min": 215,
"Max": 1061230,

398
t/lib/TableParser.t
View File

@@ -11,6 +11,7 @@ use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use Text::Diff;
use TableParser;
use Quoter;
use DSNParser;
@@ -71,21 +72,23 @@ like($EVAL_ERROR, qr/quoting/, 'No quoting');
$tbl = $tp->parse( load_file('t/lib/samples/t1.sql') );
is_deeply(
$tbl,
{ cols => [qw(a)],
col_posn => { a => 0 },
is_col => { a => 1 },
is_autoinc => { a => 0 },
null_cols => [qw(a)],
is_nullable => { a => 1 },
clustered_key => undef,
keys => {},
defs => { a => ' `a` int(11) default NULL' },
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
engine => 'MyISAM',
type_for => { a => 'int' },
name => 't1',
charset => 'latin1',
{ cols => [qw(a)],
col_posn => { a => 0 },
is_col => { a => 1 },
is_autoinc => { a => 0 },
null_cols => [qw(a)],
is_generated => {},
non_generated_cols => [ 'a' ],
is_nullable => { a => 1 },
clustered_key => undef,
keys => {},
defs => { a => ' `a` int(11) default NULL' },
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
engine => 'MyISAM',
type_for => { a => 'int' },
name => 't1',
charset => 'latin1',
},
'Basic table is OK',
);
@@ -94,11 +97,13 @@ $tbl = $tp->parse( load_file('t/lib/samples/TableParser-prefix_idx.sql') );
is_deeply(
$tbl,
{
name => 't1',
cols => [ 'a', 'b' ],
col_posn => { a => 0, b => 1 },
is_col => { a => 1, b => 1 },
is_autoinc => { 'a' => 0, 'b' => 0 },
name => 't1',
cols => [ 'a', 'b' ],
non_generated_cols => [ 'a', 'b' ],
col_posn => { a => 0, b => 1 },
is_col => { a => 1, b => 1 },
is_autoinc => { 'a' => 0, 'b' => 0 },
is_generated => {},
null_cols => [ 'a', 'b' ],
is_nullable => { 'a' => 1, 'b' => 1 },
clustered_key => undef,
@@ -169,6 +174,12 @@ is_deeply(
length replacement_cost rating special_features
last_update)
],
non_generated_cols => [
qw(film_id title description release_year language_id
original_language_id rental_duration rental_rate
length replacement_cost rating special_features
last_update)
],
col_posn => {
film_id => 0,
title => 1,
@@ -214,6 +225,7 @@ is_deeply(
special_features => 1,
last_update => 1,
},
is_generated => {},
null_cols => [qw(description release_year original_language_id length rating special_features )],
is_nullable => {
description => 1,
@@ -342,25 +354,109 @@ throws_ok (
$tbl = $tp->parse( load_file('t/lib/samples/temporary_table.sql') );
is_deeply(
$tbl,
{ cols => [qw(a)],
col_posn => { a => 0 },
is_col => { a => 1 },
is_autoinc => { a => 0 },
null_cols => [qw(a)],
is_nullable => { a => 1 },
clustered_key => undef,
keys => {},
defs => { a => ' `a` int(11) default NULL' },
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
engine => 'MyISAM',
type_for => { a => 'int' },
name => 't',
charset => 'latin1',
{ cols => [qw(a)],
non_generated_cols => [qw(a)],
col_posn => { a => 0 },
is_col => { a => 1 },
is_autoinc => { a => 0 },
is_generated => {},
null_cols => [qw(a)],
is_nullable => { a => 1 },
clustered_key => undef,
keys => {},
defs => { a => ' `a` int(11) default NULL' },
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
engine => 'MyISAM',
type_for => { a => 'int' },
name => 't',
charset => 'latin1',
},
'Temporary table',
);
my $want = { 'is_autoinc' => {
'sort_order' => 0,
'pfk-source_instrument_id' => 0,
'pfk-related_instrument_id' => 0
},
'null_cols' => [],
'numeric_cols' => [
'pfk-source_instrument_id', 'pfk-related_instrument_id',
'sort_order'
],
'cols' => [
'pfk-source_instrument_id', 'pfk-related_instrument_id',
'sort_order'
],
'non_generated_cols' => [
'pfk-source_instrument_id', 'pfk-related_instrument_id',
'sort_order'
],
is_autogenerated => {},
'col_posn' => {
'sort_order' => 2,
'pfk-source_instrument_id' => 0,
'pfk-related_instrument_id' => 1
},
clustered_key => 'PRIMARY',
'keys' => {
'sort_order' => {
'is_unique' => 0,
'is_col' => { 'sort_order' => 1 },
'name' => 'sort_order',
'type' => 'BTREE',
'col_prefixes' => [ undef ],
'is_nullable' => 0,
'colnames' => '`sort_order`',
'cols' => [ 'sort_order' ],
ddl => 'KEY `sort_order` (`sort_order`)',
},
'PRIMARY' => {
'is_unique' => 1,
'is_col' => {
'pfk-source_instrument_id' => 1,
'pfk-related_instrument_id' => 1
},
'name' => 'PRIMARY',
'type' => 'BTREE',
'col_prefixes' => [ undef, undef ],
'is_nullable' => 0,
'colnames' =>
'`pfk-source_instrument_id`,`pfk-related_instrument_id`',
'cols' =>
[ 'pfk-source_instrument_id', 'pfk-related_instrument_id' ],
ddl => 'PRIMARY KEY (`pfk-source_instrument_id`,`pfk-related_instrument_id`),',
}
},
'defs' => {
'sort_order' => ' `sort_order` int(11) NOT NULL',
'pfk-source_instrument_id' =>
' `pfk-source_instrument_id` int(10) unsigned NOT NULL',
'pfk-related_instrument_id' =>
' `pfk-related_instrument_id` int(10) unsigned NOT NULL'
},
'engine' => 'InnoDB',
'is_col' => {
'sort_order' => 1,
'pfk-source_instrument_id' => 1,
'pfk-related_instrument_id' => 1
},
'is_numeric' => {
'sort_order' => 1,
'pfk-source_instrument_id' => 1,
'pfk-related_instrument_id' => 1
},
'type_for' => {
'sort_order' => 'int',
'pfk-source_instrument_id' => 'int',
'pfk-related_instrument_id' => 'int'
},
'is_nullable' => {},
name => 'instrument_relation',
charset => 'latin1',
};
$tbl = $tp->parse( load_file('t/lib/samples/hyphentest.sql') );
is_deeply(
$tbl,
@@ -378,6 +474,11 @@ is_deeply(
'pfk-source_instrument_id', 'pfk-related_instrument_id',
'sort_order'
],
'non_generated_cols' => [
'pfk-source_instrument_id', 'pfk-related_instrument_id',
'sort_order'
],
is_generated => {},
'col_posn' => {
'sort_order' => 2,
'pfk-source_instrument_id' => 0,
@@ -446,13 +547,14 @@ is_deeply(
$tbl = $tp->parse( load_file('t/lib/samples/ndb_table.sql') );
is_deeply(
$tbl,
{ cols => [qw(id)],
col_posn => { id => 0 },
is_col => { id => 1 },
is_autoinc => { id => 1 },
null_cols => [],
is_nullable => {},
clustered_key => undef,
{ cols => [qw(id)],
non_generated_cols => [qw(id)],
col_posn => { id => 0 },
is_col => { id => 1 },
is_autoinc => { id => 1 },
null_cols => [],
is_nullable => {},
clustered_key => undef,
keys => {
PRIMARY => {
cols => [qw(id)],
@@ -473,6 +575,7 @@ is_deeply(
type_for => { id => 'bigint' },
name => 'pipo',
charset => 'latin1',
is_generated => {},
},
'NDB table',
);
@@ -481,9 +584,11 @@ $tbl = $tp->parse( load_file('t/lib/samples/mixed-case.sql') );
is_deeply(
$tbl,
{ cols => [qw(a b mixedcol)],
non_generated_cols => [qw(a b mixedcol)],
col_posn => { a => 0, b => 1, mixedcol => 2 },
is_col => { a => 1, b => 1, mixedcol => 1 },
is_autoinc => { a => 0, b => 0, mixedcol => 0 },
is_generated => {},
null_cols => [qw(a b mixedcol)],
is_nullable => { a => 1, b => 1, mixedcol => 1 },
clustered_key => undef,
@@ -518,14 +623,15 @@ is_deeply(
$tbl = $tp->parse( load_file('t/lib/samples/one_key.sql') );
is_deeply(
$tbl,
{ cols => [qw(a b)],
col_posn => { a => 0, b => 1 },
is_col => { a => 1, b => 1 },
is_autoinc => { a => 0, b => 0 },
null_cols => [qw(b)],
is_nullable => { b => 1 },
clustered_key => undef,
keys => {
{ cols => [qw(a b)],
non_generated_cols => [qw(a b)],
col_posn => { a => 0, b => 1 },
is_col => { a => 1, b => 1 },
is_autoinc => { a => 0, b => 0 },
null_cols => [qw(b)],
is_nullable => { b => 1 },
clustered_key => undef,
keys => {
PRIMARY => {
colnames => '`a`',
cols => [qw(a)],
@@ -538,16 +644,17 @@ is_deeply(
ddl => 'PRIMARY KEY (`a`)',
},
},
defs => {
defs => {
a => ' `a` int(11) NOT NULL',
b => ' `b` char(50) default NULL',
},
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
engine => 'MyISAM',
type_for => { a => 'int', b => 'char' },
name => 't2',
charset => 'latin1',
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
is_generated => {},
engine => 'MyISAM',
type_for => { a => 'int', b => 'char' },
name => 't2',
charset => 'latin1',
},
'No clustered key on MyISAM table'
);
@@ -751,21 +858,23 @@ cmp_ddls('v5.0 vs. v5.1', 't/lib/samples/issue_109-01-v50.sql', 't/lib/samples/i
$tbl = $tp->parse( load_file('t/lib/samples/issue_132.sql') );
is_deeply(
$tbl,
{ cols => [qw(country)],
col_posn => { country => 0 },
is_col => { country => 1 },
is_autoinc => { country => 0 },
null_cols => [qw(country)],
is_nullable => { country => 1 },
clustered_key => undef,
keys => {},
defs => { country => " `country` enum('','Cote D`ivoire') default NULL"},
numeric_cols => [],
is_numeric => {},
engine => 'MyISAM',
type_for => { country => 'enum' },
name => 'issue_132',
charset => 'latin1',
{ cols => [qw(country)],
non_generated_cols => [qw(country)],
col_posn => { country => 0 },
is_col => { country => 1 },
is_autoinc => { country => 0 },
is_generated => {},
null_cols => [qw(country)],
is_nullable => { country => 1 },
clustered_key => undef,
keys => {},
defs => { country => " `country` enum('','Cote D`ivoire') default NULL"},
numeric_cols => [],
is_numeric => {},
engine => 'MyISAM',
type_for => { country => 'enum' },
name => 'issue_132',
charset => 'latin1',
},
'ENUM col with backtick in value (issue 132)'
);
@@ -787,21 +896,23 @@ is(
$tbl = $tp->parse( load_file('t/lib/samples/issue_330_backtick_pair_in_col_comments.sql') );
is_deeply(
$tbl,
{ cols => [qw(a)],
col_posn => { a => 0 },
is_col => { a => 1 },
is_autoinc => { a => 0 },
null_cols => [qw(a)],
is_nullable => { a => 1 },
clustered_key => undef,
keys => {},
defs => { a => " `a` int(11) DEFAULT NULL COMMENT 'issue_330 `alex`'" },
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
engine => 'MyISAM',
type_for => { a => 'int' },
name => 'issue_330',
charset => 'latin1',
{ cols => [qw(a)],
non_generated_cols => [qw(a)],
col_posn => { a => 0 },
is_col => { a => 1 },
is_generated => {},
is_autoinc => { a => 0 },
null_cols => [qw(a)],
is_nullable => { a => 1 },
clustered_key => undef,
keys => {},
defs => { a => " `a` int(11) DEFAULT NULL COMMENT 'issue_330 `alex`'" },
numeric_cols => [qw(a)],
is_numeric => { a => 1 },
engine => 'MyISAM',
type_for => { a => 'int' },
name => 'issue_330',
charset => 'latin1',
},
'issue with pairing backticks in column comments (issue 330)'
);
@@ -810,24 +921,26 @@ is_deeply(
$tbl = $tp->parse( load_file('t/lib/samples/issue_pt-193_backtick_in_col_comments.sql') );
is_deeply(
$tbl,
{ cols => [qw(id f22abcde f23abc)],
col_posn => { id => 0, f22abcde => 1, f23abc => 2 },
is_col => { id => 1, f22abcde => 1, f23abc => 1 },
is_autoinc => { id => 1, f22abcde => 0, f23abc => 0 },
null_cols => [qw(f22abcde)],
is_nullable => { f22abcde => 1},
clustered_key => undef,
keys => {},
defs => { id => " `id` int(11) NOT NULL AUTO_INCREMENT",
"f22abcde" => " `f22abcde` int(10) unsigned DEFAULT NULL COMMENT 'xxx`XXx'",
"f23abc" => " `f23abc` int(10) unsigned NOT NULL DEFAULT '255' COMMENT \"`yyy\""
},
numeric_cols => [qw(id f22abcde f23abc)],
is_numeric => { id => 1, f22abcde => 1, f23abc => 1 },
engine => 'InnoDB',
type_for => { id => 'int', f22abcde => 'int', f23abc => 'int' },
name => 't3',
charset => 'latin1',
{ cols => [qw(id f22abcde f23abc)],
non_generated_cols => [qw(id f22abcde f23abc)],
col_posn => { id => 0, f22abcde => 1, f23abc => 2 },
is_col => { id => 1, f22abcde => 1, f23abc => 1 },
is_autoinc => { id => 1, f22abcde => 0, f23abc => 0 },
is_generated => {},
null_cols => [qw(f22abcde)],
is_nullable => { f22abcde => 1},
clustered_key => undef,
keys => {},
defs => { id => " `id` int(11) NOT NULL AUTO_INCREMENT",
"f22abcde" => " `f22abcde` int(10) unsigned DEFAULT NULL COMMENT 'xxx`XXx'",
"f23abc" => " `f23abc` int(10) unsigned NOT NULL DEFAULT '255' COMMENT \"`yyy\""
},
numeric_cols => [qw(id f22abcde f23abc)],
is_numeric => { id => 1, f22abcde => 1, f23abc => 1 },
engine => 'InnoDB',
type_for => { id => 'int', f22abcde => 'int', f23abc => 'int' },
name => 't3',
charset => 'latin1',
},
'issue with pairing backticks in column comments (issue 330)'
);
@@ -874,12 +987,14 @@ is_deeply(
clustered_key => undef,
col_posn => { 'first, last' => 1, id => 0 },
cols => [ 'id', 'first, last' ],
non_generated_cols => [ 'id', 'first, last' ],
defs => {
'first, last' => ' `first, last` varchar(32) default NULL',
id => ' `id` int(11) NOT NULL auto_increment',
},
engine => 'MyISAM',
is_autoinc => { 'first, last' => 0, id => 1 },
is_generated => {},
is_col => { 'first, last' => 1, id => 1 },
is_nullable => { 'first, last' => 1 },
is_numeric => { id => 1 },
@@ -1000,22 +1115,24 @@ $tbl = $tp->parse(load_file('t/lib/samples/triple-quoted-col.sql'));
is_deeply(
$tbl,
{
clustered_key => undef,
col_posn => { 'foo' => 0, bar => 1 },
cols => [ 'foo', 'bar' ],
defs => {
clustered_key => undef,
col_posn => { 'foo' => 0, bar => 1 },
is_generated => {},
cols => [ 'foo', 'bar' ],
non_generated_cols => [ 'foo', 'bar' ],
defs => {
'foo' => ' `foo` int(11) DEFAULT NULL',
'bar' => ' ```bar``` int(11) DEFAULT NULL',
},
engine => 'InnoDB',
is_autoinc => { foo => 0, bar => 0 },
is_col => { foo => 1, bar => 1 },
is_nullable => { foo => 1, bar => 1 },
is_numeric => { foo => 1, bar => 1 },
name => 't',
null_cols => [ 'foo', 'bar' ],
numeric_cols => [ 'foo', 'bar' ],
type_for => {
engine => 'InnoDB',
is_autoinc => { foo => 0, bar => 0 },
is_col => { foo => 1, bar => 1 },
is_nullable => { foo => 1, bar => 1 },
is_numeric => { foo => 1, bar => 1 },
name => 't',
null_cols => [ 'foo', 'bar' ],
numeric_cols => [ 'foo', 'bar' ],
type_for => {
foo => 'int',
bar => 'int',
},
@@ -1025,6 +1142,51 @@ is_deeply(
'Literal backticks (bug 1462904)'
);
SKIP: {
skip "generated column tests require MySQL 5.7+", 1
if $sandbox_version lt '5.7';
$tbl = $tp->parse( load_file('t/lib/samples/generated_cols.sql') );
is_deeply(
$tbl,
{
charset => 'utf8mb4',
clustered_key => 'PRIMARY',
col_posn => { column2 => 1, column3 => 2, id => 0 },
cols => [ 'id', 'column2', 'column3' ],
non_generated_cols => [ 'id', 'column2' ],
defs => {
column2 => ' `column2` int(11) DEFAULT NULL',
column3 => ' `column3` int(11) GENERATED ALWAYS AS ((`column2` + 1)) STORED',
id => ' `id` int(11) NOT NULL'
},
engine => 'InnoDB',
is_autoinc => { column2 => 0, column3 => 0, id => 0 },
is_col => { column2 => 1, column3 => 1, id => 1 },
is_generated => { column3 => 1 },
is_nullable => { column2 => 1, column3 => 1 },
is_numeric => { column2 => 1, column3 => 1, id => 1 },
keys => {
PRIMARY => {
col_prefixes => [ undef ],
colnames => '`id`',
cols => [ 'id' ],
ddl => 'PRIMARY KEY (`id`)',
is_col => { id => 1 },
is_nullable => 0,
is_unique => 1,
name => 'PRIMARY',
type => 'BTREE'
}
},
name => 't1',
null_cols => [ 'column2', 'column3' ],
numeric_cols => [ 'id', 'column2', 'column3' ],
type_for => { column2 => 'int', column3 => 'int', id => 'int' }
},
'Generated columns is OK',
) or die Data::Dumper::Dumper($tbl);
}
# #############################################################################
# Done.
# #############################################################################

6
t/lib/samples/generated_cols.sql Normal file
View File

@@ -0,0 +1,6 @@
CREATE TABLE `t1` (
`ID` int(11) NOT NULL,
`Column2` int(11) DEFAULT NULL,
`Column3` int(11) GENERATED ALWAYS AS ((`Column2` + 1)) STORED,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci

98
t/pt-kill/pt_167.t Normal file
View File

@@ -0,0 +1,98 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Time::HiRes qw(sleep);
use Test::More;
use threads;
use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-kill";
use Data::Dumper;
$Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Quotekeys = 0;
my $o = new OptionParser(description => 'Diskstats',
file => "$trunk/bin/pt-kill",
);
$o->get_specs("$trunk/bin/pt-table-checksum");
$o->get_opts();
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
$sb->load_file('master', "t/pt-kill/samples/pt_167.sql");
my $ps = $dbh->prepare('INSERT INTO test.foo VALUES (?)');
for (my $i=0; $i < 20000; $i++) {
$ps->execute($i);
}
sub start_thread {
my ($dp, $o, $dsn, $sleep_time) = @_;
diag("Thread started");
my $cxn = new Cxn(DSNParser => $dp, OptionParser => $o, dsn => $dsn);
$cxn->connect();
my $dbh = $cxn->dbh();
my $sth = $dbh->prepare('SELECT COUNT(*) FROM (SELECT a.id AS a_id, b.id AS b_id FROM foo AS a, foo AS b) AS c');
# Since this query is going to be killed, wrap the execution in an eval to prevent
# displaying the error message.
eval {
$sth->execute();
};
}
my $dsn = "h=127.1,P=12345,u=msandbox,p=msandbox,D=test;mysql_server_prepare=1";
my $thr = threads->create('start_thread', $dp, $o, $dp->parse($dsn), 1);
$thr->detach();
threads->yield();
sleep(1);
my $rows = $dbh->selectall_hashref('show processlist', 'id');
my $pid = 0; # reuse, reset
map { $pid = $_->{id} }
grep { $_->{info} && $_->{info} =~ m/SELECT COUNT\(\*\) FROM \(SELECT a.id/ }
values %$rows;
ok(
$pid,
"Got proc id of sleeping query: $pid"
);
$dsn = $sb->dsn_for('master');
my $output = output(
sub { pt_kill::main($dsn, "--kill-busy-commands","Query,Execute", qw(--run-time 3s --kill --busy-time 2s --print --match-info), "^(select|SELECT)"), },
stderr => 1,
);
like(
$output,
qr/KILL $pid \(Execute/,
'--kill-query'
) or diag($output);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing;

4
t/pt-kill/samples/pt_167.sql Normal file
View File

@@ -0,0 +1,4 @@
DROP DATABASE IF EXISTS test;
CREATE DATABASE test;
create table test.foo (id integer);

4
t/pt-online-schema-change/pt-196.t
View File

@@ -45,9 +45,7 @@ $sb->load_file('master', "$sample/pt-196.sql");
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args, "$master_dsn,D=test,t=test",
'--execute', '--alter', 'DROP COLUMN test', '--max-load',
'THREADPOOL_IDLE_THREADS=1'
),
'--execute', '--alter', 'DROP COLUMN test', ),
},
);

72
t/pt-online-schema-change/pt-200.t Normal file
View File

@@ -0,0 +1,72 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use threads;
use English qw(-no_match_vars);
use Test::More;
use Data::Dumper;
use PerconaTest;
use Sandbox;
use SqlModes;
use File::Temp qw/ tempdir /;
plan tests => 3;
require "$trunk/bin/pt-online-schema-change";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the
# tool will die.
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $sample = "t/pt-online-schema-change/samples/";
$sb->load_file('master', "$sample/pt-153.sql");
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args, "$master_dsn,D=test,t=t1",
'--execute',
'--alter', "ADD COLUMN unique_id BIGINT UNSIGNED NOT NULL DEFAULT 0, ADD INDEX(unique_id)",
),
},
);
is(
$exit_status,
0,
"PT-200 Adding a column named unique_xxx is not detected as an unique index",
);
like(
$output,
qr/Successfully altered `test`.`t1`/s,
"PT-200 Adding field having 'unique' in the name",
);
$master_dbh->do("DROP DATABASE IF EXISTS test");
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing;

76
t/pt-online-schema-change/pt-202.t Normal file
View File

@@ -0,0 +1,76 @@
#!/usr/bin/env perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use threads;
use English qw(-no_match_vars);
use Test::More;
use Data::Dumper;
use PerconaTest;
use Sandbox;
use SqlModes;
use File::Temp qw/ tempdir /;
require "$trunk/bin/pt-online-schema-change";
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $master_dbh = $sb->get_dbh_for('master');
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
if ($sandbox_version lt '5.7') {
plan skip_all => "generated column tests require MySQL 5.7+";
}
plan tests => 3;
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --set-vars innodb_lock_wait_timeout=3 else the
# tool will die.
my @args = (qw(--set-vars innodb_lock_wait_timeout=3));
my $output;
my $exit_status;
my $sample = "t/pt-online-schema-change/samples/";
$sb->load_file('master', "$sample/pt-202.sql");
($output, $exit_status) = full_output(
sub { pt_online_schema_change::main(@args, "$master_dsn,D=test,t=t1",
'--execute',
'--alter', "ADD COLUMN `Column4` VARCHAR(45) NULL AFTER `Column3`",
),
},
);
is(
$exit_status,
0,
"PT-202 Altering table having generated columns exit status 0",
);
like(
$output,
qr/Successfully altered `test`.`t1`/s,
"PT-202 Altering table having generated columns success",
);
$master_dbh->do("DROP DATABASE IF EXISTS test");
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($master_dbh);
ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
done_testing;

9
t/pt-online-schema-change/samples/pt-202.sql Normal file
View File

@@ -0,0 +1,9 @@
CREATE SCHEMA IF NOT EXISTS test;
USE test;
DROP TABLE IF EXISTS t1;
CREATE TABLE `test`.`t1` (
`ID` int(11) NOT NULL,
`Column2` int(11) DEFAULT NULL,
`Column3` int(11) GENERATED ALWAYS AS ((`Column2` + 1)) STORED,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;