diff --git a/bin/pt-table-sync b/bin/pt-table-sync index f50ae600..c9ee839f 100755 --- a/bin/pt-table-sync +++ b/bin/pt-table-sync @@ -3456,7 +3456,7 @@ sub set_src { } elsif ( lc $src eq 'right' ) { $self->{src_db_tbl} = $self->{right_db_tbl}; - $self->{dst_db_tbl} = $self->{left_db_tbl}; + $self->{dst_db_tbl} = $self->{left_db_tbl}; } else { die "src argument must be either 'left' or 'right'" @@ -3592,13 +3592,14 @@ sub make_UPDATE { my $types = $self->{tbl_struct}->{type_for}; return "UPDATE $self->{dst_db_tbl} SET " . join(', ', map { + my $is_hex = ($types->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i; my $is_char = ($types->{$_} || '') =~ m/char|text|enum/i; my $is_float = ($types->{$_} || '') =~ m/float|double/i; $self->{Quoter}->quote($_) . '=' . $self->{Quoter}->quote_val( $row->{$_}, - is_char => $is_char, + is_char => $is_char && !$is_hex, is_float => $is_float, ); } grep { !$in_where{$_} } @cols) @@ -3622,7 +3623,7 @@ sub make_REPLACE { sub make_row { my ( $self, $verb, $row, $cols ) = @_; - my @cols; + my @cols; if ( my $dbh = $self->{fetch_back} ) { my $where = $self->make_where_clause($row, $cols); my $sql = $self->make_fetch_back_query($where); @@ -3636,37 +3637,44 @@ sub make_row { } my $q = $self->{Quoter}; my $type_for = $self->{tbl_struct}->{type_for}; + return "$verb INTO $self->{dst_db_tbl}(" . join(', ', map { $q->quote($_) } @cols) . ') VALUES (' . join(', ', map { + my $is_hex = ($type_for->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i; my $is_char = ($type_for->{$_} || '') =~ m/char|text|enum/i; my $is_float = ($type_for->{$_} || '') =~ m/float|double/i; $q->quote_val( $row->{$_}, - is_char => $is_char, + is_char => $is_char && !$is_hex, is_float => $is_float, ) } @cols) . ')'; + } sub make_where_clause { my ( $self, $row, $cols ) = @_; my @clauses = map { + my $col = $_; + $col = $self->{Quoter}->quote($col); + my $val = $row->{$_}; my $sep = defined $val ? '=' : ' IS '; my $is_char = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/char|text|enum/i; my $is_float = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/float|double/i; - $self->{Quoter}->quote($_) . $sep . $self->{Quoter}->quote_val($val, + my $is_crc32 = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/binary|text|blob/i; + $col = "CRC32($col)" if ($is_crc32); + $col . $sep . $self->{Quoter}->quote_val($val, is_char => $is_char, is_float => $is_float); } @$cols; return join(' AND ', @clauses); } - sub get_changes { my ( $self ) = @_; return %{$self->{changes}}; @@ -3676,7 +3684,7 @@ sub get_changes { sub sort_cols { my ( $self, $row ) = @_; my @cols; - if ( $self->{tbl_struct} ) { + if ( $self->{tbl_struct} ) { my $pos = $self->{tbl_struct}->{col_posn}; my @not_in_tbl; @cols = sort { @@ -3691,7 +3699,7 @@ sub sort_cols { 1; } } - keys %$row; + sort keys %$row; push @cols, @not_in_tbl if @not_in_tbl; } else { @@ -4933,7 +4941,7 @@ sub make_row_checksum { my $query; if ( !$args{no_cols} ) { $query = join(', ', - map { + map { my $col = $_; if ( $col =~ m/\+ 0/ ) { my ($real_col) = /^(\S+)/; @@ -4943,6 +4951,10 @@ sub make_row_checksum { my ($real_col) = m/TRIM\(([^\)]+)\)/; $col .= " AS $real_col"; } + elsif ( $col =~ m/CRC32/ ) { + my ($real_col) = m/CRC32\(([^\)]+)\)/; + $col .= " AS $real_col"; + } $col; } @cols) . ', '; diff --git a/lib/ChangeHandler.pm b/lib/ChangeHandler.pm index 33aaa9a6..a3dccbc3 100644 --- a/lib/ChangeHandler.pm +++ b/lib/ChangeHandler.pm @@ -326,13 +326,14 @@ sub make_UPDATE { my $types = $self->{tbl_struct}->{type_for}; return "UPDATE $self->{dst_db_tbl} SET " . join(', ', map { + my $is_hex = ($types->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i; my $is_char = ($types->{$_} || '') =~ m/char|text|enum/i; my $is_float = ($types->{$_} || '') =~ m/float|double/i; $self->{Quoter}->quote($_) . '=' . $self->{Quoter}->quote_val( $row->{$_}, - is_char => $is_char, + is_char => $is_char && !$is_hex, is_float => $is_float, ); } grep { !$in_where{$_} } @cols) @@ -407,11 +408,12 @@ sub make_row { . ') VALUES (' . join(', ', map { - my $is_char = ($type_for->{$_} || '') =~ m/char|text/i; + my $is_hex = ($type_for->{$_} || '') =~ m/^0x[0-9a-fA-F]+$/i; + my $is_char = ($type_for->{$_} || '') =~ m/char|text|enum/i; my $is_float = ($type_for->{$_} || '') =~ m/float|double/i; $q->quote_val( $row->{$_}, - is_char => $is_char, + is_char => $is_char && !$is_hex, is_float => $is_float, ) } @cols) @@ -431,18 +433,22 @@ sub make_row { sub make_where_clause { my ( $self, $row, $cols ) = @_; my @clauses = map { + my $col = $_; + $col = $self->{Quoter}->quote($col); + my $val = $row->{$_}; my $sep = defined $val ? '=' : ' IS '; - my $is_char = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/char|text/i; + my $is_char = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/char|text|enum/i; my $is_float = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/float|double/i; - $self->{Quoter}->quote($_) . $sep . $self->{Quoter}->quote_val($val, + my $is_crc32 = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/binary|text|blob/i; + $col = "CRC32($col)" if ($is_crc32); + $col . $sep . $self->{Quoter}->quote_val($val, is_char => $is_char, is_float => $is_float); } @$cols; return join(' AND ', @clauses); } - # Sub: get_changes # Get a summary of changes made. # diff --git a/lib/TableChecksum.pm b/lib/TableChecksum.pm index e60f0b31..58b6a812 100644 --- a/lib/TableChecksum.pm +++ b/lib/TableChecksum.pm @@ -308,6 +308,9 @@ sub make_row_checksum { elsif ( $args{trim} && $type =~ m/varchar/ ) { $result = "TRIM($result)"; } + elsif ( $type =~ m/binary|text|blob/ ) { + $result = "CRC32($result)"; + } $result; } grep { @@ -335,6 +338,10 @@ sub make_row_checksum { my ($real_col) = m/TRIM\(([^\)]+)\)/; $col .= " AS $real_col"; } + elsif ( $col =~ m/CRC32/ ) { + my ($real_col) = m/CRC32\(([^\)]+)\)/; + $col .= " AS $real_col"; + } $col; } @cols) . ', '; diff --git a/t/pt-table-sync/bidirectional.t b/t/pt-table-sync/bidirectional.t index 84760371..9faf854e 100644 --- a/t/pt-table-sync/bidirectional.t +++ b/t/pt-table-sync/bidirectional.t @@ -43,7 +43,7 @@ else { my $output; my $cnf = '/tmp/12345/my.sandbox.cnf'; -my @args = ('-F', $cnf, 'h=127.1,P=12345', 'P=2900', qw(-d bidi --bidirectional)); +my @args = ('-F', $cnf, 'h=127.1,P=12345', 'P=2900', qw(-d bidi --bidirectional --no-version-check)); $sb->wipe_clean($c1_dbh); $sb->wipe_clean($r1_dbh); diff --git a/t/pt-table-sync/issue_96.t_ b/t/pt-table-sync/issue_96.t similarity index 96% rename from t/pt-table-sync/issue_96.t_ rename to t/pt-table-sync/issue_96.t index 8786f8d0..cc2f455b 100644 --- a/t/pt-table-sync/issue_96.t_ +++ b/t/pt-table-sync/issue_96.t @@ -24,7 +24,8 @@ if ( !$master_dbh ) { plan skip_all => 'Cannot connect to sandbox master'; } else { - plan tests => 2; + plan skip_all => 'This test is freezing'; + #plan tests => 2; } $sb->wipe_clean($master_dbh); diff --git a/t/pt-table-sync/pt-2016.t b/t/pt-table-sync/pt-2016.t new file mode 100644 index 00000000..49d4a1df --- /dev/null +++ b/t/pt-table-sync/pt-2016.t @@ -0,0 +1,84 @@ +#!/usr/bin/env perl + +BEGIN { + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; +}; + +binmode(STDIN, ':utf8') or die "Can't binmode(STDIN, ':utf8'): $OS_ERROR"; +binmode(STDOUT, ':utf8') or die "Can't binmode(STDOUT, ':utf8'): $OS_ERROR"; + +use strict; +use utf8; +use Encode qw(decode encode); +use warnings FATAL => 'all'; +use English qw(-no_match_vars); +use Test::More; + +use PerconaTest; +use Sandbox; +require "$trunk/bin/pt-table-sync"; + +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $master_dbh = $sb->get_dbh_for('master'); +my $slave1_dbh = $sb->get_dbh_for('slave1'); +my $slave2_dbh = $sb->get_dbh_for('slave2'); + +if ( !$master_dbh ) { + plan skip_all => 'Cannot connect to sandbox master'; +} +elsif ( !$slave1_dbh ) { + plan skip_all => 'Cannot connect to sandbox slave1'; +} +elsif ( !$slave1_dbh ) { + plan skip_all => 'Cannot connect to sandbox slave2'; +} +else { + plan tests => 3; +} + + +my ($output, $status); +my @args = ('h=127.0.0.1,P=12346,u=msandbox,p=msandbox,D=test,t=test2', '--sync-to-master', + '--chunk-size=1', '--hex-blob', '--execute'); + +# use lib/samples dir since the main change is in DSNParser +$sb->load_file('master', "t/pt-table-sync/samples/pt-2016.sql"); + +$sb->wait_for_slaves(); + +$slave1_dbh->do("UPDATE test.test2 SET col3='bbb'"); +$slave1_dbh->do("FLUSH TABLES"); + +# 1 +($output, $status) = full_output( + sub { pt_table_sync::main(@args) }, +); + +is( + $status, + 2, # exit_status = 2 -> there were differences + "PT-2016 table-sync CRC32 in key - Exit status", +); + +# 2 +my $want = { + col1 => 1, + col2 => 'aaa', + col3 => 'aaa' +}; +my $row = $slave1_dbh->selectrow_hashref("SELECT col1, col2, col3 FROM test.test2"); +is_deeply( + $row, + $want, + "PT-2016 table-sync CRC32 in key - Master was updated", +) or diag("Want '".($want||"")."', got '".($row->{col3}||"")."'"); + +# ############################################################################# +# Done. +# ############################################################################# +$sb->wipe_clean($master_dbh); +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); +exit; diff --git a/t/pt-table-sync/samples/pt-2016.sql b/t/pt-table-sync/samples/pt-2016.sql new file mode 100644 index 00000000..2e4d2d7c --- /dev/null +++ b/t/pt-table-sync/samples/pt-2016.sql @@ -0,0 +1,12 @@ +DROP DATABASE IF EXISTS test; + +CREATE DATABASE test; + +CREATE TABLE test.test2 ( + col1 INT NOT NULL, + col2 TEXT NOT NULL, + col3 VARCHAR(5) NOT NULL, + UNIQUE KEY (col1, col2(3)) +); + +INSERT INTO test.test2 VALUES(1,'aaa','aaa');