diff --git a/bin/pt-online-schema-change b/bin/pt-online-schema-change index 1db26704..6750b61f 100755 --- a/bin/pt-online-schema-change +++ b/bin/pt-online-schema-change @@ -5520,11 +5520,32 @@ sub new { ); PTDEBUG && _d('Ascend params:', Dumper($asc)); + my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); + for my $index (@{$index_cols}) { + if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') { + if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) { + my @items = split(/,\s*/, $1); + my $sorted = 1; # Asume the items list is sorted to later check if this is true + for (my $i=1; $i < scalar(@items); $i++) { + if ($items[$i-1] gt $items[$i]) { + $sorted = 0; + last; + } + } + if (!$force_concat_enums && !$sorted) { + die "The index " . $index . " in table " . $tbl->{name} . + " has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n"; + } + } + } + } + + my $from = "$tbl->{name} FORCE INDEX(`$index`)"; - my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); my $first_lb_sql @@ -12249,6 +12270,30 @@ duplicate rows and this data will be lost. This options bypasses confirmation in case of using alter-foreign-keys-method = none , which might break foreign key constraints. +=item --force-concat-enums + +The NibbleIterator in Percona Toolkit can detect indexes having ENUM fields and +if the items it has are sorted or not. According to MySQL documentation at +L: + +ENUM values are sorted based on their index numbers, which depend on the order in +which the enumeration members were listed in the column specification. +For example, 'b' sorts before 'a' for ENUM('b', 'a'). +The empty string sorts before nonempty strings, and NULL values sort before all other +enumeration values. + +To prevent unexpected results when using the ORDER BY clause on an ENUM column, +use one of these techniques: +- Specify the ENUM list in alphabetic order. +- Make sure that the column is sorted lexically rather than by index number by coding +ORDER BY CAST(col AS CHAR) or ORDER BY CONCAT(col). + +The NibbleIterator in Percona Toolkit uses CONCAT(col) but, doing that, adds overhead +since MySQL cannot use the column directly and has to calculate the result of CONCAT +for every row. +To make this scenario vissible to the user, if there are indexes having ENUM fields +with usorted items, it is necessary to specify the C<--force-concat-enums> parameter. + =item --help Show help and exit. diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 427de6e0..7d88b492 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -6371,11 +6371,32 @@ sub new { ); PTDEBUG && _d('Ascend params:', Dumper($asc)); + my $force_concat_enums = $o->has('force-concat-enums') && $o->get('force-concat-enums'); + for my $index (@{$index_cols}) { + if ($tbl->{tbl_struct}->{type_for}->{$index} eq 'enum') { + if ($tbl->{tbl_struct}->{defs}->{$index} =~ m/enum\s*\((.*?)\)/) { + my @items = split(/,\s*/, $1); + my $sorted = 1; # Asume the items list is sorted to later check if this is true + for (my $i=1; $i < scalar(@items); $i++) { + if ($items[$i-1] gt $items[$i]) { + $sorted = 0; + last; + } + } + if (!$force_concat_enums && !$sorted) { + die "The index " . $index . " in table " . $tbl->{name} . + " has unsorted enum items.\nPlease read the documentation for the --force-concat-enums parameter\n"; + } + } + } + } + + my $from = "$tbl->{name} FORCE INDEX(`$index`)"; - my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + my $order_by = join(', ', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); - my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' + my $order_by_dec = join(' DESC,', map { $tbl->{tbl_struct}->{type_for}->{$_} eq 'enum' && $force_concat_enums ? "CONCAT(".$q->quote($_).")" : $q->quote($_)} @{$index_cols}); my $first_lb_sql @@ -13264,7 +13285,27 @@ values 1.008 and 1.009 will be rounded to 1.01, and will checksum as equal. =item --force-concat-enums -bla bla bla bla bla +The NibbleIterator in Percona Toolkit can detect indexes having ENUM fields and +if the items it has are sorted or not. According to MySQL documentation at +L: + +ENUM values are sorted based on their index numbers, which depend on the order in +which the enumeration members were listed in the column specification. +For example, 'b' sorts before 'a' for ENUM('b', 'a'). +The empty string sorts before nonempty strings, and NULL values sort before all other +enumeration values. + +To prevent unexpected results when using the ORDER BY clause on an ENUM column, +use one of these techniques: +- Specify the ENUM list in alphabetic order. +- Make sure that the column is sorted lexically rather than by index number by coding +ORDER BY CAST(col AS CHAR) or ORDER BY CONCAT(col). + +The NibbleIterator in Percona Toolkit uses CONCAT(col) but, doing that, adds overhead +since MySQL cannot use the column directly and has to calculate the result of CONCAT +for every row. +To make this scenario vissible to the user, if there are indexes having ENUM fields +with usorted items, it is necessary to specify the C<--force-concat-enums> parameter. =item --function diff --git a/bin/pt-table-sync b/bin/pt-table-sync index 15f7e27e..ffb74bbd 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -12408,6 +12408,30 @@ the values are converted to strings by the CONCAT() function, and MySQL chooses the string representation. If you specify a value of 2, for example, then the values 1.008 and 1.009 will be rounded to 1.01, and will checksum as equal. +=item --force-concat-enums + +The NibbleIterator in Percona Toolkit can detect indexes having ENUM fields and +if the items it has are sorted or not. According to MySQL documentation at +L: + +ENUM values are sorted based on their index numbers, which depend on the order in +which the enumeration members were listed in the column specification. +For example, 'b' sorts before 'a' for ENUM('b', 'a'). +The empty string sorts before nonempty strings, and NULL values sort before all other +enumeration values. + +To prevent unexpected results when using the ORDER BY clause on an ENUM column, +use one of these techniques: +- Specify the ENUM list in alphabetic order. +- Make sure that the column is sorted lexically rather than by index number by coding +ORDER BY CAST(col AS CHAR) or ORDER BY CONCAT(col). + +The NibbleIterator in Percona Toolkit uses CONCAT(col) but, doing that, adds overhead +since MySQL cannot use the column directly and has to calculate the result of CONCAT +for every row. +To make this scenario vissible to the user, if there are indexes having ENUM fields +with usorted items, it is necessary to specify the C<--force-concat-enums> parameter. + =item --[no]foreign-key-checks default: yes diff --git a/t/lib/NibbleIterator.t b/t/lib/NibbleIterator.t index 53379828..8574b9fc 100644 --- a/t/lib/NibbleIterator.t +++ b/t/lib/NibbleIterator.t @@ -958,6 +958,8 @@ like( '_d() works' ); +$dbh->do("DROP DATABASE test"); + $sb->wipe_clean($dbh); ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); done_testing; diff --git a/t/pt-table-checksum/basics.t b/t/pt-table-checksum/basics.t index 69ed7ae7..c912d7b1 100644 --- a/t/pt-table-checksum/basics.t +++ b/t/pt-table-checksum/basics.t @@ -45,7 +45,7 @@ elsif ( !@{$master_dbh->selectall_arrayref("show databases like 'sakila'")} ) { # so we need to specify --set-vars innodb_lock_wait_timeout=3 else the tool will die. my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox'; my $slave2_dsn = 'h=127.1,P=12347,u=msandbox,p=msandbox'; -my @args = ($master_dsn, qw(--set-vars innodb_lock_wait_timeout=3)); +my @args = ($master_dsn, qw(--set-vars innodb_lock_wait_timeout=3 --ignore-tables load_data)); my $row; my $output; my $exit_status; diff --git a/t/pt-table-checksum/samples/default-results-5.6.txt b/t/pt-table-checksum/samples/default-results-5.6.txt index ba8c6bc0..bad5c2bf 100644 --- a/t/pt-table-checksum/samples/default-results-5.6.txt +++ b/t/pt-table-checksum/samples/default-results-5.6.txt @@ -23,7 +23,6 @@ ERRORS DIFFS ROWS SKIPPED TABLE 0 0 0 0 mysql.time_zone_transition_type 0 0 2 0 mysql.user 0 0 22 0 percona_test.checksums -0 0 1 0 percona_test.load_data 0 0 1 0 percona_test.sentinel 0 0 200 0 sakila.actor 0 0 603 0 sakila.address diff --git a/t/pt-table-checksum/samples/default-results-5.7.txt b/t/pt-table-checksum/samples/default-results-5.7.txt index 3bc92c4c..4b79b4de 100644 --- a/t/pt-table-checksum/samples/default-results-5.7.txt +++ b/t/pt-table-checksum/samples/default-results-5.7.txt @@ -25,7 +25,6 @@ ERRORS DIFFS ROWS SKIPPED TABLE 0 0 0 0 mysql.time_zone_transition_type 0 0 2 0 mysql.user 0 0 23 0 percona_test.checksums -0 0 1 0 percona_test.load_data 0 0 1 0 percona_test.sentinel 0 0 200 0 sakila.actor 0 0 603 0 sakila.address diff --git a/t/pt-table-checksum/samples/default-results-8.0.txt b/t/pt-table-checksum/samples/default-results-8.0.txt index a01c9d01..6bf1e50f 100644 --- a/t/pt-table-checksum/samples/default-results-8.0.txt +++ b/t/pt-table-checksum/samples/default-results-8.0.txt @@ -27,7 +27,6 @@ ERRORS DIFFS ROWS SKIPPED TABLE 0 0 0 0 mysql.time_zone_transition_type 0 0 5 0 mysql.user 0 0 27 0 percona_test.checksums -0 0 1 0 percona_test.load_data 0 0 1 0 percona_test.sentinel 0 0 200 0 sakila.actor 0 0 603 0 sakila.address diff --git a/t/pt-table-checksum/samples/static-chunk-size-results-5.6.txt b/t/pt-table-checksum/samples/static-chunk-size-results-5.6.txt index 8a528ae0..575e2613 100644 --- a/t/pt-table-checksum/samples/static-chunk-size-results-5.6.txt +++ b/t/pt-table-checksum/samples/static-chunk-size-results-5.6.txt @@ -2,7 +2,6 @@ if all tables be checksummed checksum ... ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE 0 0 22 1 0 percona_test.checksums -0 0 1 1 0 percona_test.load_data 0 0 1 1 0 percona_test.sentinel 0 0 200 1 0 sakila.actor 0 0 603 1 0 sakila.address diff --git a/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt b/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt index 78f296d0..45e13a07 100644 --- a/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt +++ b/t/pt-table-checksum/samples/static-chunk-size-results-5.7.txt @@ -2,7 +2,6 @@ if all tables be checksummed checksum ... ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE 0 0 23 1 0 percona_test.checksums -0 0 1 1 0 percona_test.load_data 0 0 1 1 0 percona_test.sentinel 0 0 200 1 0 sakila.actor 0 0 603 1 0 sakila.address diff --git a/t/pt-table-checksum/samples/static-chunk-size-results-8.0.txt b/t/pt-table-checksum/samples/static-chunk-size-results-8.0.txt index f96d4fcf..a24afda6 100644 --- a/t/pt-table-checksum/samples/static-chunk-size-results-8.0.txt +++ b/t/pt-table-checksum/samples/static-chunk-size-results-8.0.txt @@ -2,7 +2,6 @@ if all tables be checksummed checksum ... ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE 0 0 27 1 0 percona_test.checksums -0 0 1 1 0 percona_test.load_data 0 0 1 1 0 percona_test.sentinel 0 0 200 1 0 sakila.actor 0 0 603 1 0 sakila.address