percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-11 13:40:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename new TableChecksum to RowChecksum and revert TableChecksum to r108. This will avoid completely break TableSync*.pm and pt-table-sync.

Browse Source
This commit is contained in:
Daniel Nichter
2011-09-10 13:46:55 -06:00
parent 968d10ac37
commit 84f7c47ff4
6 changed files with 1716 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
t/lib/NibbleIterator.t
View File

@@ -20,6 +20,7 @@ use OptionParser;
use MySQLDump;
use TableParser;
use TableNibbler;
use TableChecksum;
use NibbleIterator;
use PerconaTest;
@@ -38,7 +39,7 @@ if ( !$dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
else {
plan tests => 12;
plan tests => 17;
}
my $q = new Quoter();
@@ -46,6 +47,7 @@ my $tp = new TableParser(Quoter=>$q);
my $du = new MySQLDump();
my $nb = new TableNibbler(TableParser=>$tp, Quoter=>$q);
my $o = new OptionParser(description => 'NibbleIterator');
my $tc = new TableChecksum(OptionParser => $o, Quoter=>$q);
$o->get_specs("$trunk/bin/pt-table-checksum");
@@ -81,6 +83,7 @@ sub make_nibble_iter {
dbh => $dbh,
tbl => $schema->get_table($args{db}, $args{tbl}),
callbacks => $args{callbacks},
select => $args{select},
%common_modules,
);
@@ -289,7 +292,7 @@ done
# Nibble a larger table by numeric pk id
# ############################################################################
SKIP: {
skip "Sakila database is not loaded", 1
skip "Sakila database is not loaded", 6
unless @{ $dbh->selectall_arrayref('show databases like "sakila"') };
$ni = make_nibble_iter(
@@ -305,6 +308,58 @@ SKIP: {
16049,
"Nibble sakila.payment (16049 rows)"
);
my $tbl = {
db => 'sakila',
tbl => 'country',
tbl_struct => $tp->parse(
$du->get_create_table($dbh, $q, 'sakila', 'country')),
};
my $chunk_checksum = $tc->make_chunk_checksum(
dbh => $dbh,
tbl => $tbl,
);
$ni = make_nibble_iter(
db => 'sakila',
tbl => 'country',
argv => [qw(--databases sakila --tables country --chunk-size 25)],
select => $chunk_checksum,
);
my $row = $ni->next();
is_deeply(
$row,
[25, 'da79784d'],
"SELECT chunk checksum 1 FROM sakila.country"
) or print STDERR Dumper($row);
$row = $ni->next();
is_deeply(
$row,
[25, 'e860c4f9'],
"SELECT chunk checksum 2 FROM sakila.country"
) or print STDERR Dumper($row);
$row = $ni->next();
is_deeply(
$row,
[25, 'eb651f58'],
"SELECT chunk checksum 3 FROM sakila.country"
) or print STDERR Dumper($row);
$row = $ni->next();
is_deeply(
$row,
[25, '2d87d588'],
"SELECT chunk checksum 4 FROM sakila.country"
) or print STDERR Dumper($row);
$row = $ni->next();
is_deeply(
$row,
[9, 'beb4a180'],
"SELECT chunk checksum 5 FROM sakila.country"
) or print STDERR Dumper($row);
}
# #############################################################################

417
t/lib/RowChecksum.t Normal file
View File

@@ -0,0 +1,417 @@
#!/usr/bin/perl
BEGIN {
die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
};
use strict;
use warnings FATAL => 'all';
use English qw(-no_match_vars);
use Test::More;
use RowChecksum;
use TableParser;
use Quoter;
use MySQLDump;
use DSNParser;
use OptionParser;
use Sandbox;
use PerconaTest;
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $dbh = $sb->get_dbh_for('master');
if ( !$dbh ) {
plan skip_all => "Cannot connect to sandbox master";
}
else {
plan tests => 28;
}
$sb->create_dbs($dbh, ['test']);
my $q = new Quoter();
my $tp = new TableParser(Quoter => $q);
my $du = new MySQLDump();
my $o = new OptionParser(description => 'NibbleIterator');
$o->get_specs("$trunk/bin/pt-table-checksum");
my $c = new RowChecksum(
OptionParser => $o,
Quoter => $q,
);
# ############################################################################
# _make_xor_slices
# ############################################################################
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 1,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 1), 16, 10) "
. "AS UNSIGNED)), 10, 16), 1, '0')",
'FOO XOR slices 1 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 16,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0')",
'FOO XOR slices 16 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 17,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
. "LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 17, 1), 16, 10) "
. "AS UNSIGNED)), 10, 16), 1, '0')",
'FOO XOR slices 17 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 32,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
. "LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 17, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0')",
'FOO XOR slices 32 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 32,
opt_slice => 0,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(\@crc := FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
. "LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(\@crc, 17, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0')",
'XOR slice optimized in slice 0',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 32,
opt_slice => 1,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(\@crc, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
. "LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(\@crc := FOO, 17, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0')",
'XOR slice optimized in slice 1',
);
# ############################################################################
# make_row_checksum
# ############################################################################
my $tbl = {
db => 'sakila',
tbl => 'film',
tbl_struct => $tp->parse(load_file('t/lib/samples/sakila.film.sql')),
};
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
),
q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, `last_update` + 0 AS `last_update`, }
. q{SHA1(CONCAT_WS('#', }
. q{`film_id`, `title`, `description`, `release_year`, `language_id`, }
. q{`original_language_id`, `rental_duration`, `rental_rate`, `length`, }
. q{`replacement_cost`, `rating`, `special_features`, `last_update` + 0, }
. q{CONCAT(ISNULL(`description`), ISNULL(`release_year`), }
. q{ISNULL(`original_language_id`), ISNULL(`length`), }
. q{ISNULL(`rating`), ISNULL(`special_features`))))},
'SHA1 query for sakila.film',
);
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'FNV_64',
),
q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, `last_update` + 0 AS `last_update`, }
. q{FNV_64(}
. q{`film_id`, `title`, `description`, `release_year`, `language_id`, }
. q{`original_language_id`, `rental_duration`, `rental_rate`, `length`, }
. q{`replacement_cost`, `rating`, `special_features`, `last_update` + 0)},
'FNV_64 query for sakila.film',
);
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id)],
),
q{`film_id`, SHA1(`film_id`)},
'SHA1 query for sakila.film with only one column',
);
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(FILM_ID)],
),
q{`film_id`, SHA1(`film_id`)},
'Column names are case-insensitive',
);
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id title)],
sep => '%',
),
q{`film_id`, `title`, SHA1(CONCAT_WS('%', `film_id`, `title`))},
'Separator',
);
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id title)],
sep => "'%'",
),
q{`film_id`, `title`, SHA1(CONCAT_WS('%', `film_id`, `title`))},
'Bad separator',
);
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id title)],
sep => "'''",
),
q{`film_id`, `title`, SHA1(CONCAT_WS('#', `film_id`, `title`))},
'Really bad separator',
);
# sakila.rental
$tbl = {
db => 'sakila',
tbl => 'rental',
tbl_struct => $tp->parse(load_file('t/lib/samples/sakila.rental.float.sql')),
};
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
),
q{`rental_id`, `foo`, SHA1(CONCAT_WS('#', `rental_id`, `foo`))},
'FLOAT column is like any other',
);
is(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
float_precision => 5,
),
q{`rental_id`, ROUND(`foo`, 5), SHA1(CONCAT_WS('#', `rental_id`, ROUND(`foo`, 5)))},
'FLOAT column is rounded to 5 places',
);
# sakila.film
$tbl = {
db => 'sakila',
tbl => 'film',
tbl_struct => $tp->parse(load_file('t/lib/samples/sakila.film.sql')),
};
like(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
trim => 1,
),
qr{TRIM\(`title`\)},
'VARCHAR column is trimmed',
);
# ############################################################################
# make_chunk_checksum
# ############################################################################
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'SHA1',
crc_width=> 40,
cols => [qw(film_id)],
crc_type => 'varchar',
),
q{COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONCAT(LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 1, }
. q{16), 16, 10) AS UNSIGNED)), 10, 16), 16, '0'), }
. q{LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 17, 16), 16, }
. q{10) AS UNSIGNED)), 10, 16), 16, '0'), }
. q{LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 33, 8), 16, }
. q{10) AS UNSIGNED)), 10, 16), 8, '0'))), 0) AS crc},
'sakila.film SHA1',
);
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'FNV_64',
crc_width=> 99,
cols => [qw(film_id)],
crc_type => 'bigint',
),
q{COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(FNV_64(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc},
'sakila.film FNV_64',
);
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'FNV_64',
crc_width=> 99,
cols => [qw(film_id)],
buffer => 1,
crc_type => 'bigint',
),
q{COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(FNV_64(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc},
'sakila.film FNV_64',
);
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'CRC32',
crc_width=> 99,
cols => [qw(film_id)],
buffer => 1,
crc_type => 'int',
),
q{COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc},
'sakila.film CRC32',
);
# #############################################################################
# Sandbox tests.
# #############################################################################
like(
$c->_get_hash_func(
dbh => $dbh,
),
qr/CRC32|FNV_64|MD5/,
'CRC32, FNV_64 or MD5 is default',
);
like(
$c->_get_hash_func(
dbh => $dbh,
func => 'SHA99',
),
qr/CRC32|FNV_64|MD5/,
'SHA99 does not exist so I get CRC32 or friends',
);
@ARGV = qw(--function MD5);
$o->get_opts();
is(
$c->_get_hash_func(
dbh => $dbh,
func => 'MD5',
),
'MD5',
'MD5 requested and MD5 granted',
);
@ARGV = qw();
$o->get_opts();
is(
$c->_optimize_xor(
dbh => $dbh,
func => 'SHA1',
),
'2',
'SHA1 slice is 2',
);
is(
$c->_optimize_xor(
dbh => $dbh,
func => 'MD5',
),
'1',
'MD5 slice is 1',
);
is(
$c->_get_crc_type(
dbh => $dbh,
func => 'CRC32',
),
'int',
'CRC32 type'
);
is(
$c->_get_crc_type(
dbh => $dbh,
func => 'MD5',
),
'varchar',
'MD5 type'
);
# #############################################################################
# Issue 94: Enhance mk-table-checksum, add a --ignorecols option
# #############################################################################
$sb->load_file('master', 't/lib/samples/issue_94.sql');
$tbl = {
db => 'test',
tbl => 'issue_94',
tbl_struct => $tp->parse($du->get_create_table($dbh, $q, 'test', 'issue_94')),
};
my $query = $c->make_chunk_checksum(
tbl => $tbl,
func => 'CRC32',
crc_width => 16,
crc_type => 'int',
opt_slice => undef,
cols => undef,
sep => '#',
replicate => undef,
precision => undef,
trim => undef,
ignorecols => {'c'=>1},
);
is(
$query,
"COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `a`, `b`)) AS UNSIGNED)), 10, 16)), 0) AS crc",
'Ignores specified columns'
);
# ############################################################################
# Done.
# ############################################################################
$sb->wipe_clean($dbh);
exit;

627
t/lib/TableChecksum.t
View File

@@ -12,11 +12,11 @@ use English qw(-no_match_vars);
use Test::More;
use TableChecksum;
use VersionParser;
use TableParser;
use Quoter;
use MySQLDump;
use DSNParser;
use OptionParser;
use Sandbox;
use PerconaTest;
@@ -28,49 +28,165 @@ if ( !$dbh ) {
plan skip_all => "Cannot connect to sandbox master";
}
else {
plan tests => 28;
plan tests => 51;
}
$sb->create_dbs($dbh, ['test']);
my $q = new Quoter();
my $tp = new TableParser(Quoter => $q);
my $vp = new VersionParser();
my $du = new MySQLDump();
my $o = new OptionParser(description => 'NibbleIterator');
$o->get_specs("$trunk/bin/pt-table-checksum");
my $c = new TableChecksum(Quoter=>$q, VersionParser=>$vp);
my $c = new TableChecksum(
OptionParser => $o,
Quoter => $q,
my $t;
my %args = map { $_ => undef }
qw(db tbl tbl_struct algorithm function crc_wid crc_type opt_slice);
throws_ok (
sub { $c->best_algorithm( %args, algorithm => 'foo', ) },
qr/Invalid checksum algorithm/,
'Algorithm=foo',
);
# ############################################################################
# _make_xor_slices
# ############################################################################
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 1,
# Inject the VersionParser with some bogus versions. Later I'll just pass the
# string version number instead of a real DBH, so the version parsing will
# return the value I want.
foreach my $ver( qw(4.0.0 4.1.1) ) {
$vp->{$ver} = $vp->parse($ver);
}
is (
$c->best_algorithm(
algorithm => 'CHECKSUM',
dbh => '4.1.1',
),
'CHECKSUM',
'Prefers CHECKSUM',
);
is (
$c->best_algorithm(
dbh => '4.1.1',
),
'CHECKSUM',
'Default is CHECKSUM',
);
is (
$c->best_algorithm(
algorithm => 'CHECKSUM',
dbh => '4.1.1',
where => 1,
),
'BIT_XOR',
'CHECKSUM eliminated by where',
);
is (
$c->best_algorithm(
algorithm => 'CHECKSUM',
dbh => '4.1.1',
chunk => 1,
),
'BIT_XOR',
'CHECKSUM eliminated by chunk',
);
is (
$c->best_algorithm(
algorithm => 'CHECKSUM',
dbh => '4.1.1',
replicate => 1,
),
'BIT_XOR',
'CHECKSUM eliminated by replicate',
);
is (
$c->best_algorithm(
dbh => '4.1.1',
count => 1,
),
'BIT_XOR',
'Default CHECKSUM eliminated by count',
);
is (
$c->best_algorithm(
algorithm => 'CHECKSUM',
dbh => '4.1.1',
count => 1,
),
'CHECKSUM',
'Explicit CHECKSUM not eliminated by count',
);
is (
$c->best_algorithm(
algorithm => 'CHECKSUM',
dbh => '4.0.0',
),
'ACCUM',
'CHECKSUM and BIT_XOR eliminated by version',
);
is (
$c->best_algorithm(
algorithm => 'BIT_XOR',
dbh => '4.1.1',
),
'BIT_XOR',
'BIT_XOR as requested',
);
is (
$c->best_algorithm(
algorithm => 'BIT_XOR',
dbh => '4.0.0',
),
'ACCUM',
'BIT_XOR eliminated by version',
);
is (
$c->best_algorithm(
algorithm => 'ACCUM',
dbh => '4.1.1',
),
'ACCUM',
'ACCUM as requested',
);
ok($c->is_hash_algorithm('ACCUM'), 'ACCUM is hash');
ok($c->is_hash_algorithm('BIT_XOR'), 'BIT_XOR is hash');
ok(!$c->is_hash_algorithm('CHECKSUM'), 'CHECKSUM is not hash');
is (
$c->make_xor_slices(
query => 'FOO',
crc_wid => 1,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 1), 16, 10) "
. "AS UNSIGNED)), 10, 16), 1, '0')",
'FOO XOR slices 1 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 16,
is (
$c->make_xor_slices(
query => 'FOO',
crc_wid => 16,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0')",
'FOO XOR slices 16 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 17,
is (
$c->make_xor_slices(
query => 'FOO',
crc_wid => 17,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
@@ -79,10 +195,10 @@ is(
'FOO XOR slices 17 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 32,
is (
$c->make_xor_slices(
query => 'FOO',
crc_wid => 32,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
@@ -91,11 +207,11 @@ is(
'FOO XOR slices 32 wide',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 32,
opt_slice => 0,
is (
$c->make_xor_slices(
query => 'FOO',
crc_wid => 32,
opt_slice => 0,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(\@crc := FOO, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
@@ -104,11 +220,11 @@ is(
'XOR slice optimized in slice 0',
);
is(
$c->_make_xor_slices(
row_checksum => 'FOO',
crc_width => 32,
opt_slice => 1,
is (
$c->make_xor_slices(
query => 'FOO',
crc_wid => 32,
opt_slice => 1,
),
"LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(\@crc, 1, 16), 16, 10) "
. "AS UNSIGNED)), 10, 16), 16, '0'), "
@@ -117,19 +233,12 @@ is(
'XOR slice optimized in slice 1',
);
# ############################################################################
# make_row_checksum
# ############################################################################
my $tbl = {
db => 'sakila',
tbl => 'film',
tbl_struct => $tp->parse(load_file('t/lib/samples/sakila.film.sql')),
};
$t = $tp->parse(load_file('t/lib/samples/sakila.film.sql'));
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
function => 'SHA1',
tbl_struct => $t,
),
q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, `last_update` + 0 AS `last_update`, }
. q{SHA1(CONCAT_WS('#', }
@@ -142,10 +251,10 @@ is(
'SHA1 query for sakila.film',
);
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'FNV_64',
function => 'FNV_64',
tbl_struct => $t,
),
q{`film_id`, `title`, `description`, `release_year`, `language_id`, `original_language_id`, `rental_duration`, `rental_rate`, `length`, `replacement_cost`, `rating`, `special_features`, `last_update` + 0 AS `last_update`, }
. q{FNV_64(}
@@ -155,169 +264,329 @@ is(
'FNV_64 query for sakila.film',
);
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id)],
function => 'SHA1',
tbl_struct => $t,
cols => [qw(film_id)],
),
q{`film_id`, SHA1(`film_id`)},
'SHA1 query for sakila.film with only one column',
);
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(FILM_ID)],
function => 'SHA1',
tbl_struct => $t,
cols => [qw(FILM_ID)],
),
q{`film_id`, SHA1(`film_id`)},
'Column names are case-insensitive',
);
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id title)],
sep => '%',
function => 'SHA1',
tbl_struct => $t,
cols => [qw(film_id title)],
sep => '%',
),
q{`film_id`, `title`, SHA1(CONCAT_WS('%', `film_id`, `title`))},
'Separator',
);
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id title)],
sep => "'%'",
function => 'SHA1',
tbl_struct => $t,
cols => [qw(film_id title)],
sep => "'%'",
),
q{`film_id`, `title`, SHA1(CONCAT_WS('%', `film_id`, `title`))},
'Bad separator',
);
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
cols => [qw(film_id title)],
sep => "'''",
function => 'SHA1',
tbl_struct => $t,
cols => [qw(film_id title)],
sep => "'''",
),
q{`film_id`, `title`, SHA1(CONCAT_WS('#', `film_id`, `title`))},
'Really bad separator',
);
# sakila.rental
$tbl = {
db => 'sakila',
tbl => 'rental',
tbl_struct => $tp->parse(load_file('t/lib/samples/sakila.rental.float.sql')),
};
is(
$t = $tp->parse(load_file('t/lib/samples/sakila.rental.float.sql'));
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
function => 'SHA1',
tbl_struct => $t,
),
q{`rental_id`, `foo`, SHA1(CONCAT_WS('#', `rental_id`, `foo`))},
'FLOAT column is like any other',
);
is(
is (
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
function => 'SHA1',
tbl_struct => $t,
float_precision => 5,
),
q{`rental_id`, ROUND(`foo`, 5), SHA1(CONCAT_WS('#', `rental_id`, ROUND(`foo`, 5)))},
'FLOAT column is rounded to 5 places',
);
# sakila.film
$tbl = {
db => 'sakila',
tbl => 'film',
tbl_struct => $tp->parse(load_file('t/lib/samples/sakila.film.sql')),
};
$t = $tp->parse(load_file('t/lib/samples/sakila.film.sql'));
like(
$c->make_row_checksum(
tbl => $tbl,
func => 'SHA1',
trim => 1,
function => 'SHA1',
tbl_struct => $t,
trim => 1,
),
qr{TRIM\(`title`\)},
'VARCHAR column is trimmed',
);
# ############################################################################
# make_chunk_checksum
# ############################################################################
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'SHA1',
crc_width=> 40,
cols => [qw(film_id)],
crc_type => 'varchar',
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'CHECKSUM',
function => 'SHA1',
crc_wid => 40,
crc_type => 'varchar',
),
q{COUNT(*) AS cnt, }
'CHECKSUM TABLE `sakila`.`film`',
'Sakila.film CHECKSUM',
);
throws_ok (
sub { $c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'BIT_XOR',
crc_wid => 40,
cols => [qw(film_id)],
crc_type => 'varchar',
function => 'SHA1',
algorithm => 'CHECKSUM TABLE',
)
},
qr/missing checksum algorithm/,
'Complains about bad algorithm',
);
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'BIT_XOR',
function => 'SHA1',
crc_wid => 40,
cols => [qw(film_id)],
crc_type => 'varchar',
),
q{SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONCAT(LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 1, }
. q{16), 16, 10) AS UNSIGNED)), 10, 16), 16, '0'), }
. q{LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 17, 16), 16, }
. q{10) AS UNSIGNED)), 10, 16), 16, '0'), }
. q{LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 33, 8), 16, }
. q{10) AS UNSIGNED)), 10, 16), 8, '0'))), 0) AS crc},
'sakila.film SHA1',
. q{10) AS UNSIGNED)), 10, 16), 8, '0'))), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film SHA1 BIT_XOR',
);
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'FNV_64',
crc_width=> 99,
cols => [qw(film_id)],
crc_type => 'bigint',
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'BIT_XOR',
function => 'FNV_64',
crc_wid => 99,
cols => [qw(film_id)],
crc_type => 'bigint',
),
q{COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(FNV_64(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc},
'sakila.film FNV_64',
q{SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(FNV_64(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film FNV_64 BIT_XOR',
);
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'FNV_64',
crc_width=> 99,
cols => [qw(film_id)],
buffer => 1,
crc_type => 'bigint',
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'BIT_XOR',
function => 'FNV_64',
crc_wid => 99,
cols => [qw(film_id)],
buffer => 1,
crc_type => 'bigint',
),
q{COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(FNV_64(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc},
'sakila.film FNV_64',
q{SELECT SQL_BUFFER_RESULT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(FNV_64(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film FNV_64 BIT_XOR',
);
is(
$c->make_chunk_checksum(
tbl => $tbl,
func => 'CRC32',
crc_width=> 99,
cols => [qw(film_id)],
buffer => 1,
crc_type => 'int',
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'BIT_XOR',
function => 'CRC32',
crc_wid => 99,
cols => [qw(film_id)],
buffer => 1,
crc_type => 'int',
),
q{COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc},
'sakila.film CRC32',
q{SELECT SQL_BUFFER_RESULT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(`film_id`) AS UNSIGNED)), 10, 16)), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film CRC32 BIT_XOR',
);
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'BIT_XOR',
function => 'SHA1',
crc_wid => 40,
cols => [qw(film_id)],
replicate => 'test.checksum',
crc_type => 'varchar',
),
q{REPLACE /*PROGRESS_COMMENT*/ INTO test.checksum }
. q{(db, tbl, chunk, boundaries, this_cnt, this_crc) }
. q{SELECT ?, ?, /*CHUNK_NUM*/ ?, COUNT(*) AS cnt, }
. q{COALESCE(LOWER(CONCAT(LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 1, }
. q{16), 16, 10) AS UNSIGNED)), 10, 16), 16, '0'), }
. q{LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 17, 16), 16, }
. q{10) AS UNSIGNED)), 10, 16), 16, '0'), }
. q{LPAD(CONV(BIT_XOR(CAST(CONV(SUBSTRING(SHA1(`film_id`), 33, 8), 16, }
. q{10) AS UNSIGNED)), 10, 16), 8, '0'))), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film SHA1 BIT_XOR with replication',
);
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'ACCUM',
function => 'SHA1',
crc_wid => 40,
crc_type => 'varchar',
),
q{SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, }
. q{COALESCE(RIGHT(MAX(@crc := CONCAT(LPAD(@cnt := @cnt + 1, 16, '0'), }
. q{SHA1(CONCAT(@crc, SHA1(CONCAT_WS('#', }
. q{`film_id`, `title`, `description`, `release_year`, `language_id`, }
. q{`original_language_id`, `rental_duration`, `rental_rate`, `length`, }
. q{`replacement_cost`, `rating`, `special_features`, `last_update` + 0, }
. q{CONCAT(ISNULL(`description`), ISNULL(`release_year`), }
. q{ISNULL(`original_language_id`), ISNULL(`length`), }
. q{ISNULL(`rating`), ISNULL(`special_features`)))))))), 40), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film SHA1 ACCUM',
);
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'ACCUM',
function => 'FNV_64',
crc_wid => 16,
crc_type => 'bigint',
),
q{SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, }
. q{COALESCE(RIGHT(MAX(@crc := CONCAT(LPAD(@cnt := @cnt + 1, 16, '0'), }
. q{CONV(CAST(FNV_64(CONCAT(@crc, FNV_64(}
. q{`film_id`, `title`, `description`, `release_year`, `language_id`, }
. q{`original_language_id`, `rental_duration`, `rental_rate`, `length`, }
. q{`replacement_cost`, `rating`, `special_features`, `last_update` + 0}
. q{))) AS UNSIGNED), 10, 16))), 16), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film FNV_64 ACCUM',
);
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'ACCUM',
function => 'CRC32',
crc_wid => 16,
crc_type => 'int',
cols => [qw(film_id)],
),
q{SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, }
. q{COALESCE(RIGHT(MAX(@crc := CONCAT(LPAD(@cnt := @cnt + 1, 16, '0'), }
. q{CONV(CAST(CRC32(CONCAT(@crc, CRC32(`film_id`}
. q{))) AS UNSIGNED), 10, 16))), 16), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film CRC32 ACCUM',
);
is (
$c->make_checksum_query(
%args,
db => 'sakila',
tbl => 'film',
tbl_struct => $t,
algorithm => 'ACCUM',
function => 'SHA1',
crc_wid => 40,
replicate => 'test.checksum',
crc_type => 'varchar',
),
q{REPLACE /*PROGRESS_COMMENT*/ INTO test.checksum }
. q{(db, tbl, chunk, boundaries, this_cnt, this_crc) }
. q{SELECT ?, ?, /*CHUNK_NUM*/ ?, COUNT(*) AS cnt, }
. q{COALESCE(RIGHT(MAX(@crc := CONCAT(LPAD(@cnt := @cnt + 1, 16, '0'), }
. q{SHA1(CONCAT(@crc, SHA1(CONCAT_WS('#', }
. q{`film_id`, `title`, `description`, `release_year`, `language_id`, }
. q{`original_language_id`, `rental_duration`, `rental_rate`, `length`, }
. q{`replacement_cost`, `rating`, `special_features`, `last_update` + 0, }
. q{CONCAT(ISNULL(`description`), ISNULL(`release_year`), }
. q{ISNULL(`original_language_id`), ISNULL(`length`), }
. q{ISNULL(`rating`), ISNULL(`special_features`)))))))), 40), 0) AS crc }
. q{FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/},
'Sakila.film SHA1 ACCUM with replication',
);
is ( $c->crc32('hello world'), 222957957, 'CRC32 of hello world');
# #############################################################################
# Sandbox tests.
# #############################################################################
like(
$c->_get_hash_func(
$c->choose_hash_func(
dbh => $dbh,
),
qr/CRC32|FNV_64|MD5/,
@@ -325,76 +594,65 @@ like(
);
like(
$c->_get_hash_func(
dbh => $dbh,
func => 'SHA99',
$c->choose_hash_func(
dbh => $dbh,
function => 'SHA99',
),
qr/CRC32|FNV_64|MD5/,
'SHA99 does not exist so I get CRC32 or friends',
);
@ARGV = qw(--function MD5);
$o->get_opts();
is(
$c->_get_hash_func(
dbh => $dbh,
func => 'MD5',
$c->choose_hash_func(
dbh => $dbh,
function => 'MD5',
),
'MD5',
'MD5 requested and MD5 granted',
);
@ARGV = qw();
$o->get_opts();
is(
$c->_optimize_xor(
dbh => $dbh,
func => 'SHA1',
$c->optimize_xor(
dbh => $dbh,
function => 'SHA1',
),
'2',
'SHA1 slice is 2',
);
is(
$c->_optimize_xor(
dbh => $dbh,
func => 'MD5',
$c->optimize_xor(
dbh => $dbh,
function => 'MD5',
),
'1',
'MD5 slice is 1',
);
is(
$c->_get_crc_type(
dbh => $dbh,
func => 'CRC32',
),
'int',
'CRC32 type'
is_deeply(
[$c->get_crc_type($dbh, 'CRC32')],
[qw(int 10)],
'Type and length of CRC32'
);
is(
$c->_get_crc_type(
dbh => $dbh,
func => 'MD5',
),
'varchar',
'MD5 type'
is_deeply(
[$c->get_crc_type($dbh, 'MD5')],
[qw(varchar 32)],
'Type and length of MD5'
);
# #############################################################################
# Issue 94: Enhance mk-table-checksum, add a --ignorecols option
# #############################################################################
$sb->load_file('master', 't/lib/samples/issue_94.sql');
$tbl = {
$t= $tp->parse( $du->get_create_table($dbh, $q, 'test', 'issue_94') );
my $query = $c->make_checksum_query(
db => 'test',
tbl => 'issue_94',
tbl_struct => $tp->parse($du->get_create_table($dbh, $q, 'test', 'issue_94')),
};
my $query = $c->make_chunk_checksum(
tbl => $tbl,
func => 'CRC32',
crc_width => 16,
tbl => 'issue_47',
tbl_struct => $t,
algorithm => 'ACCUM',
function => 'CRC32',
crc_wid => 16,
crc_type => 'int',
opt_slice => undef,
cols => undef,
@@ -404,14 +662,9 @@ my $query = $c->make_chunk_checksum(
trim => undef,
ignorecols => {'c'=>1},
);
is(
$query,
"COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `a`, `b`)) AS UNSIGNED)), 10, 16)), 0) AS crc",
'Ignores specified columns'
);
is($query,
'SELECT /*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, COALESCE(RIGHT(MAX(@crc := CONCAT(LPAD(@cnt := @cnt + 1, 16, \'0\'), CONV(CAST(CRC32(CONCAT(@crc, CRC32(CONCAT_WS(\'#\', `a`, `b`)))) AS UNSIGNED), 10, 16))), 16), 0) AS crc FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/',
'Ignores specified columns');
# ############################################################################
# Done.
# ############################################################################
$sb->wipe_clean($dbh);
exit;