Make --explain cummulative. Update char_chunking.t. Reword checksum query comments.

This commit is contained in:
Daniel Nichter
2011-10-12 11:22:13 -06:00
parent 271cd850d6
commit 083fb1b8ca
6 changed files with 232 additions and 56 deletions

View File

@@ -3483,7 +3483,7 @@ sub new {
: join(', ', map { $q->quote($_) } @cols))
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
. ($args{where} ? " AND ($args{where})" : '')
. " /*one nibble*/";
. " /*checksum table*/";
MKDEBUG && _d('One nibble statement:', $nibble_sql);
my $explain_nibble_sql
@@ -3492,7 +3492,7 @@ sub new {
: join(', ', map { $q->quote($_) } @cols))
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
. ($args{where} ? " AND ($args{where})" : '')
. " /*explain one nibble*/";
. " /*explain checksum table*/";
MKDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql);
$self = {
@@ -3546,7 +3546,7 @@ sub new {
. ($args{where} ? " AND ($args{where})" : '')
. " ORDER BY $order_by"
. " LIMIT ?, 2"
. " /*upper boundary*/";
. " /*next chunk boundary*/";
MKDEBUG && _d('Upper boundary statement:', $ub_sql);
my $nibble_sql
@@ -3558,7 +3558,7 @@ sub new {
. " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($args{where} ? " AND ($args{where})" : '')
. " ORDER BY $order_by"
. " /*nibble*/";
. " /*checksum chunk*/";
MKDEBUG && _d('Nibble statement:', $nibble_sql);
my $explain_nibble_sql
@@ -3570,7 +3570,7 @@ sub new {
. " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($args{where} ? " AND ($args{where})" : '')
. " ORDER BY $order_by"
. " /*explain nibble*/";
. " /*explain checksum chunk*/";
MKDEBUG && _d('Explain nibble statement:', $explain_nibble_sql);
my $limit = $chunk_size - 1;
@@ -3741,6 +3741,11 @@ sub sql {
return $self->{sql};
}
sub more_boundaries {
my ($self) = @_;
return !$self->{no_more_boundaries};
}
sub _find_best_index {
my (%args) = @_;
my @required_args = qw(tbl TableParser dbh Quoter);
@@ -5650,21 +5655,24 @@ sub main {
my $nibble_iter = $args{NibbleIterator};
my $oktonibble = 1;
if ( $o->get('explain') ) { # dry run (--explain)
# We're not going to checksum the table, just report the
# statements that we would use to checksum it.
if ( $o->get('explain') ) {
# --explain level 1: print the checksum and next boundary
# statements.
print "--\n",
"-- Statements for table $tbl->{db}.$tbl->{tbl}:\n",
"-- $tbl->{db}.$tbl->{tbl}\n",
"--\n\n";
my $statements = $nibble_iter->statements();
foreach my $sth ( sort keys %$statements ) {
next if $sth =~ m/^explain/;
if ( $statements->{$sth} ) {
print $statements->{$sth}->{Statement}, "\n\n";
}
}
if ( $o->get('explain') < 2 ) {
$oktonibble = 0; # don't nibble table; next table
}
}
elsif ( $last_chunk ) { # resuming
my $next_lb = next_lower_boundary(
%args,
@@ -5745,6 +5753,22 @@ sub main {
# Count every chunk, even if it's ultimately skipped, etc.
$tbl->{checksum_results}->{n_chunks}++;
# --explain level 2: print chunk,lower boundary values,upper
# boundary values.
if ( $o->get('explain') > 1 ) {
my $lb_quoted = join(',', @{$boundary->{lower}});
my $ub_quoted = join(',', @{$boundary->{upper}});
my $chunk = $nibble_iter->nibble_number();
printf "%d %s %s\n",
$chunk,
(defined $lb_quoted ? $lb_quoted : '1=1'),
(defined $ub_quoted ? $ub_quoted : '1=1');
if ( !$nibble_iter->more_boundaries() ) {
print "\n"; # blank line between this table and the next table
}
return 0; # next boundary
}
# If the table is being chunk (i.e., it's not small enough to be
# consumed by one nibble), then check index usage and chunk size.
if ( !$nibble_iter->one_nibble() ) {
@@ -5799,6 +5823,9 @@ sub main {
my $tbl = $args{tbl};
my $nibble_iter = $args{NibbleIterator};
# Don't need to do anything here if we're just --explain'ing.
return if $o->get('explain');
# Chunk/nibble number that we just inserted or skipped.
my $chunk = $nibble_iter->nibble_number();
@@ -5889,6 +5916,9 @@ sub main {
my $nibble_iter = $args{NibbleIterator};
my $max_chunk = $nibble_iter->nibble_number();
# Don't need to do anything here if we're just --explain'ing.
return if $o->get('explain');
# Wait for all slaves to run all checksum chunks,
# then check for differences.
if ( $o->get('replicate-check') && scalar @$slaves ) {
@@ -6979,7 +7009,7 @@ Do only this comma-separated list of storage engines.
=item --explain
group: Output
cumulative: yes; default: 0; group: Output
Show, but do not execute, checksum queries (disables L<"--[no]empty-replicate-table">).

View File

@@ -82,7 +82,7 @@ sub new {
: join(', ', map { $q->quote($_) } @cols))
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
. ($args{where} ? " AND ($args{where})" : '')
. " /*one nibble*/";
. " /*checksum table*/";
MKDEBUG && _d('One nibble statement:', $nibble_sql);
my $explain_nibble_sql
@@ -91,7 +91,7 @@ sub new {
: join(', ', map { $q->quote($_) } @cols))
. " FROM " . $q->quote(@{$tbl}{qw(db tbl)})
. ($args{where} ? " AND ($args{where})" : '')
. " /*explain one nibble*/";
. " /*explain checksum table*/";
MKDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql);
$self = {
@@ -158,7 +158,7 @@ sub new {
. ($args{where} ? " AND ($args{where})" : '')
. " ORDER BY $order_by"
. " LIMIT ?, 2"
. " /*upper boundary*/";
. " /*next chunk boundary*/";
MKDEBUG && _d('Upper boundary statement:', $ub_sql);
# This statement does the actual nibbling work; its rows are returned
@@ -172,7 +172,7 @@ sub new {
. " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($args{where} ? " AND ($args{where})" : '')
. " ORDER BY $order_by"
. " /*nibble*/";
. " /*checksum chunk*/";
MKDEBUG && _d('Nibble statement:', $nibble_sql);
my $explain_nibble_sql
@@ -184,7 +184,7 @@ sub new {
. " AND " . $asc->{boundaries}->{'<='} # upper boundary
. ($args{where} ? " AND ($args{where})" : '')
. " ORDER BY $order_by"
. " /*explain nibble*/";
. " /*explain checksum chunk*/";
MKDEBUG && _d('Explain nibble statement:', $explain_nibble_sql);
my $limit = $chunk_size - 1;
@@ -364,6 +364,11 @@ sub sql {
return $self->{sql};
}
sub more_boundaries {
my ($self) = @_;
return !$self->{no_more_boundaries};
}
sub _find_best_index {
my (%args) = @_;
my @required_args = qw(tbl TableParser dbh Quoter);

View File

@@ -6,3 +6,149 @@ CREATE TABLE `ascii` (
UNIQUE KEY `c` (`c`)
) ENGINE=MyISAM AUTO_INCREMENT=143 DEFAULT CHARSET=latin1;
INSERT INTO `ascii` VALUES (1,''),(2,'M'),(3,'007'),(4,'Penelope'),(5,'Nick'),(6,'Ed'),(7,'Jennifer'),(8,'johnny'),(9,'Bette'),(10,'Grace'),(11,'matthew'),(12,'joe'),(13,'christian'),(14,'zero'),(15,'karl'),(16,'uma'),(17,'vivien'),(18,'Cuba'),(19,'Fred'),(20,'Helen'),(21,'Dan'),(22,'Bob'),(23,'<[MEGATRON]>'),(24,'Lucille'),(25,'Kirsten'),(26,'Elvis'),(27,'Sandra'),(28,'Cameron'),(29,'Kevin'),(30,'Rip'),(31,'Julia'),(32,'Jean-Luc'),(33,'Woody'),(34,'Alec'),(35,'Sissy'),(36,'tim'),(37,'milla'),(38,'audrey'),(39,'judy'),(40,'burt'),(41,'val'),(42,'tom'),(43,'goldie'),(44,'jodie'),(45,'kirk'),(46,'reese'),(47,'parker'),(48,'Frances'),(49,'Anne'),(50,'Natalie'),(51,'Gary'),(52,'Carmen'),(53,'@the ball park'),(54,'Mena'),(55,'Fay'),(56,'Jude'),(57,'Dustin'),(58,'Henry'),(59,'jayne'),(60,'Ray'),(61,'Angela'),(62,'Mary'),(63,'Jessica'),(64,'kenneth'),(65,'Michelle'),(66,'adam'),(67,'Sean'),(68,'Angelina'),(69,'Cary'),(70,'Groucho'),(71,'Mae'),(72,'Ralph'),(73,'Scarlett'),(74,'Ben'),(75,'bling$$$'),(76,'James'),(77,'Minnie'),(78,'Greg'),(79,'spencer'),(80,'charlize'),(81,'christopher'),(82,'Ellen'),(83,'Daryl'),(84,'Gene'),(85,'Meg'),(86,'Chris'),(87,'jim'),(88,'susan'),(89,'walter'),(90,'sidney'),(91,'gina'),(92,'warren'),(93,'Sylvester'),(94,'Russell'),(95,'Morgan'),(96,'Harrison'),(97,'Renee'),(98,'Liza'),(99,'Salma'),(100,'Julianne'),(101,'Albert'),(102,'Cate'),(103,'Greta'),(104,'jane'),(105,'richard'),(106,'rita'),(107,'ewan'),(108,'whoopi'),(109,'jada'),(110,'River'),(111,'Kim'),(112,'Emily'),(113,'Geoffrey'),(114,'meryl'),(115,'Ian'),(116,'Laura'),(117,'Harvey'),(118,'Oprah'),(119,'Humphrey'),(120,'Al'),(121,'laurence'),(122,'will'),(123,'olympia'),(124,'alan'),(125,'michael'),(126,'william'),(127,'jon'),(128,'Mr. Peanut'),(129,'Mr. Rogers'),(130,'lisa'),(131,'Jeff'),(132,'Debbie'),(133,'Rock'),(134,'Gregory'),(135,'John'),(136,'Bela'),(137,'Thora'),(138,'Zesus'),(139,'Zesus!'),(140,'Zesus!!'),(141,'ZESUS!!!'),(142,'001');
--
-- Here's how MySQL sorts these values on my machine:
--
-- | 1 | |
-- | 2 | 001 |
-- | 3 | 007 |
-- | 4 | <[MEGATRON]> |
-- | 5 | @the ball park |
-- | 6 | adam |
-- | 7 | Al |
-- | 8 | alan |
-- | 9 | Albert |
-- | 10 | Alec |
-- | 11 | Angela |
-- | 12 | Angelina |
-- | 13 | Anne |
-- | 14 | audrey |
-- | 15 | Bela |
-- | 16 | Ben |
-- | 17 | Bette |
-- | 18 | bling$$$ |
-- | 19 | Bob |
-- | 20 | burt |
-- | 21 | Cameron |
-- | 22 | Carmen |
-- | 23 | Cary |
-- | 24 | Cate |
-- | 25 | charlize |
-- | 26 | Chris |
-- | 27 | christian |
-- | 28 | christopher |
-- | 29 | Cuba |
-- | 30 | Dan |
-- | 31 | Daryl |
-- | 32 | Debbie |
-- | 33 | Dustin |
-- | 34 | Ed |
-- | 35 | Ellen |
-- | 36 | Elvis |
-- | 37 | Emily |
-- | 38 | ewan |
-- | 39 | Fay |
-- | 40 | Frances |
-- | 41 | Fred |
-- | 42 | Gary |
-- | 43 | Gene |
-- | 44 | Geoffrey |
-- | 45 | gina |
-- | 46 | goldie |
-- | 47 | Grace |
-- | 48 | Greg |
-- | 49 | Gregory |
-- | 50 | Greta |
-- | 51 | Groucho |
-- | 52 | Harrison |
-- | 53 | Harvey |
-- | 54 | Helen |
-- | 55 | Henry |
-- | 56 | Humphrey |
-- | 57 | Ian |
-- | 58 | jada |
-- | 59 | James |
-- | 60 | jane |
-- | 61 | jayne |
-- | 62 | Jean-Luc |
-- | 63 | Jeff |
-- | 64 | Jennifer |
-- | 65 | Jessica |
-- | 66 | jim |
-- | 67 | jodie |
-- | 68 | joe |
-- | 69 | John |
-- | 70 | johnny |
-- | 71 | jon |
-- | 72 | Jude |
-- | 73 | judy |
-- | 74 | Julia |
-- | 75 | Julianne |
-- | 76 | karl |
-- | 77 | kenneth |
-- | 78 | Kevin |
-- | 79 | Kim |
-- | 80 | kirk |
-- | 81 | Kirsten |
-- | 82 | Laura |
-- | 83 | laurence |
-- | 84 | lisa |
-- | 85 | Liza |
-- | 86 | Lucille |
-- | 87 | M |
-- | 88 | Mae |
-- | 89 | Mary |
-- | 90 | matthew |
-- | 91 | Meg |
-- | 92 | Mena |
-- | 93 | meryl |
-- | 94 | michael |
-- | 95 | Michelle |
-- | 96 | milla |
-- | 97 | Minnie |
-- | 98 | Morgan |
-- | 99 | Mr. Peanut |
-- | 100 | Mr. Rogers |
-- | 101 | Natalie |
-- | 102 | Nick |
-- | 103 | olympia |
-- | 104 | Oprah |
-- | 105 | parker |
-- | 106 | Penelope |
-- | 107 | Ralph |
-- | 108 | Ray |
-- | 109 | reese |
-- | 110 | Renee |
-- | 111 | richard |
-- | 112 | Rip |
-- | 113 | rita |
-- | 114 | River |
-- | 115 | Rock |
-- | 116 | Russell |
-- | 117 | Salma |
-- | 118 | Sandra |
-- | 119 | Scarlett |
-- | 120 | Sean |
-- | 121 | sidney |
-- | 122 | Sissy |
-- | 123 | spencer |
-- | 124 | susan |
-- | 125 | Sylvester |
-- | 126 | Thora |
-- | 127 | tim |
-- | 128 | tom |
-- | 129 | uma |
-- | 130 | val |
-- | 131 | vivien |
-- | 132 | walter |
-- | 133 | warren |
-- | 134 | whoopi |
-- | 135 | will |
-- | 136 | william |
-- | 137 | Woody |
-- | 138 | zero |
-- | 139 | Zesus |
-- | 140 | Zesus! |
-- | 141 | Zesus!! |
-- | 142 | ZESUS!!! |

View File

@@ -15,29 +15,30 @@ use PerconaTest;
use Sandbox;
require "$trunk/bin/pt-table-checksum";
my $vp = new VersionParser();
my $dp = new DSNParser(opts=>$dsn_opts);
my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
my $dbh = $sb->get_dbh_for('master');
my $master_dbh = $sb->get_dbh_for('master');
if ( !$dbh ) {
if ( !$master_dbh ) {
plan skip_all => 'Cannot connect to sandbox master';
}
else {
plan tests => 3;
plan tests => 2;
}
my $output;
my $cnf ='/tmp/12345/my.sandbox.cnf';
my @args = ("F=$cnf", qw(--lock-wait-timeout 3 --chunk-time 0 --chunk-size-limit 0 --tables test.ascii));
# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
# so we need to specify --lock-wait-timeout=3 else the tool will die.
my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
my @args = ($master_dsn, qw(--lock-wait-timeout 3));
$sb->create_dbs($dbh, ['test']);
$sb->create_dbs($master_dbh, ['test']);
$sb->load_file('master', "t/lib/samples/char-chunking/ascii.sql", 'test');
ok(
no_diff(
sub { pt_table_checksum::main(@args,
qw(--chunk-size 20 --explain)) },
qw(--tables test.ascii --chunk-index c --chunk-size 20),
qw(--explain --explain)) },
"t/pt-table-checksum/samples/char-chunk-ascii-explain.txt",
),
"Char chunk ascii, explain"
@@ -46,23 +47,16 @@ ok(
ok(
no_diff(
sub { pt_table_checksum::main(@args,
qw(--chunk-size 20)) },
qw(--tables test.ascii --chunk-index c --chunk-size 20),
qw(--chunk-time 0)) },
"t/pt-table-checksum/samples/char-chunk-ascii.txt",
post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $5 " " $6 " " $8}\'',
),
"Char chunk ascii, chunk size 20"
);
ok(
no_diff(
sub { pt_table_checksum::main(@args,
qw(--chunk-size 20 --chunk-size-limit 3)) },
"t/pt-table-checksum/samples/char-chunk-ascii-oversize.txt",
),
"Char chunk ascii, chunk size 20, with oversize"
);
# #############################################################################
# Done.
# #############################################################################
$sb->wipe_clean($dbh);
$sb->wipe_clean($master_dbh);
exit;

View File

@@ -1,9 +1,17 @@
test ascii SELECT /*test.ascii:1/8*/ 0 AS chunk_num, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `c`)) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `test`.`ascii` FORCE INDEX (`c`) WHERE (`c` < ')')
test ascii `c` < ')'
test ascii `c` >= ')' AND `c` < '1'
test ascii `c` >= '1' AND `c` < '9'
test ascii `c` >= '9' AND `c` < 'A'
test ascii `c` >= 'A' AND `c` < 'I'
test ascii `c` >= 'I' AND `c` < 'Q'
test ascii `c` >= 'Q' AND `c` < 'Y'
test ascii `c` >= 'Y'
--
-- test.ascii
--
REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `c`)) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `test`.`ascii` FORCE INDEX(`c`) WHERE ((`c` >= ?)) AND ((`c` <= ?)) ORDER BY `c` /*checksum chunk*/
SELECT /*!40001 SQL_NO_CACHE */ `c` FROM `test`.`ascii` FORCE INDEX(`c`) WHERE ((`c` >= ?)) ORDER BY `c` LIMIT ?, 2 /*next chunk boundary*/
1 burt
2 Cameron Frances
3 Fred jane
4 jayne kirk
5 Kirsten Mr. Rogers
6 Natalie Sean
7 sidney Zesus!
8 Zesus!! ZESUS!!!

View File

@@ -1,9 +1,2 @@
DATABASE TABLE CHUNK HOST ENGINE COUNT CHECKSUM TIME WAIT STAT LAG
test ascii 0 127.1 MyISAM 1 25e3643f 0 0 NULL NULL
test ascii 1 127.1 MyISAM 2 1eb24fbb 0 0 NULL NULL
test ascii 2 127.1 MyISAM 0 NULL 0 0 NULL NULL
test ascii 3 127.1 MyISAM 2 fb3ab74d 0 0 NULL NULL
test ascii 4 127.1 MyISAM OVERSIZE NULL NULL NULL NULL NULL
test ascii 5 127.1 MyISAM OVERSIZE NULL NULL NULL NULL NULL
test ascii 6 127.1 MyISAM 31 77958122 0 0 NULL NULL
test ascii 7 127.1 MyISAM 5 3c0ef0c2 0 0 NULL NULL
ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE
0 0 142 8 0 test.ascii