From 083fb1b8ca5e4657a68f40a8a8d3f6e0d88f0229 Mon Sep 17 00:00:00 2001 From: Daniel Nichter Date: Wed, 12 Oct 2011 11:22:13 -0600 Subject: [PATCH] Make --explain cummulative. Update char_chunking.t. Reword checksum query comments. --- bin/pt-table-checksum | 52 +++++-- lib/NibbleIterator.pm | 15 +- t/lib/samples/char-chunking/ascii.sql | 146 ++++++++++++++++++ t/pt-table-checksum/char_chunking.t | 38 ++--- .../samples/char-chunk-ascii-explain.txt | 26 ++-- .../samples/char-chunk-ascii.txt | 11 +- 6 files changed, 232 insertions(+), 56 deletions(-) diff --git a/bin/pt-table-checksum b/bin/pt-table-checksum index 039f1018..97bb5945 100755 --- a/bin/pt-table-checksum +++ b/bin/pt-table-checksum @@ -3483,7 +3483,7 @@ sub new { : join(', ', map { $q->quote($_) } @cols)) . " FROM " . $q->quote(@{$tbl}{qw(db tbl)}) . ($args{where} ? " AND ($args{where})" : '') - . " /*one nibble*/"; + . " /*checksum table*/"; MKDEBUG && _d('One nibble statement:', $nibble_sql); my $explain_nibble_sql @@ -3492,7 +3492,7 @@ sub new { : join(', ', map { $q->quote($_) } @cols)) . " FROM " . $q->quote(@{$tbl}{qw(db tbl)}) . ($args{where} ? " AND ($args{where})" : '') - . " /*explain one nibble*/"; + . " /*explain checksum table*/"; MKDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); $self = { @@ -3546,7 +3546,7 @@ sub new { . ($args{where} ? " AND ($args{where})" : '') . " ORDER BY $order_by" . " LIMIT ?, 2" - . " /*upper boundary*/"; + . " /*next chunk boundary*/"; MKDEBUG && _d('Upper boundary statement:', $ub_sql); my $nibble_sql @@ -3558,7 +3558,7 @@ sub new { . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($args{where} ? " AND ($args{where})" : '') . " ORDER BY $order_by" - . " /*nibble*/"; + . " /*checksum chunk*/"; MKDEBUG && _d('Nibble statement:', $nibble_sql); my $explain_nibble_sql @@ -3570,7 +3570,7 @@ sub new { . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($args{where} ? " AND ($args{where})" : '') . " ORDER BY $order_by" - . " /*explain nibble*/"; + . " /*explain checksum chunk*/"; MKDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); my $limit = $chunk_size - 1; @@ -3741,6 +3741,11 @@ sub sql { return $self->{sql}; } +sub more_boundaries { + my ($self) = @_; + return !$self->{no_more_boundaries}; +} + sub _find_best_index { my (%args) = @_; my @required_args = qw(tbl TableParser dbh Quoter); @@ -5650,20 +5655,23 @@ sub main { my $nibble_iter = $args{NibbleIterator}; my $oktonibble = 1; - if ( $o->get('explain') ) { # dry run (--explain) - # We're not going to checksum the table, just report the - # statements that we would use to checksum it. + if ( $o->get('explain') ) { + # --explain level 1: print the checksum and next boundary + # statements. print "--\n", - "-- Statements for table $tbl->{db}.$tbl->{tbl}:\n", + "-- $tbl->{db}.$tbl->{tbl}\n", "--\n\n"; my $statements = $nibble_iter->statements(); foreach my $sth ( sort keys %$statements ) { + next if $sth =~ m/^explain/; if ( $statements->{$sth} ) { print $statements->{$sth}->{Statement}, "\n\n"; } } - $oktonibble = 0; # don't nibble table; next table + if ( $o->get('explain') < 2 ) { + $oktonibble = 0; # don't nibble table; next table + } } elsif ( $last_chunk ) { # resuming my $next_lb = next_lower_boundary( @@ -5745,6 +5753,22 @@ sub main { # Count every chunk, even if it's ultimately skipped, etc. $tbl->{checksum_results}->{n_chunks}++; + # --explain level 2: print chunk,lower boundary values,upper + # boundary values. + if ( $o->get('explain') > 1 ) { + my $lb_quoted = join(',', @{$boundary->{lower}}); + my $ub_quoted = join(',', @{$boundary->{upper}}); + my $chunk = $nibble_iter->nibble_number(); + printf "%d %s %s\n", + $chunk, + (defined $lb_quoted ? $lb_quoted : '1=1'), + (defined $ub_quoted ? $ub_quoted : '1=1'); + if ( !$nibble_iter->more_boundaries() ) { + print "\n"; # blank line between this table and the next table + } + return 0; # next boundary + } + # If the table is being chunk (i.e., it's not small enough to be # consumed by one nibble), then check index usage and chunk size. if ( !$nibble_iter->one_nibble() ) { @@ -5799,6 +5823,9 @@ sub main { my $tbl = $args{tbl}; my $nibble_iter = $args{NibbleIterator}; + # Don't need to do anything here if we're just --explain'ing. + return if $o->get('explain'); + # Chunk/nibble number that we just inserted or skipped. my $chunk = $nibble_iter->nibble_number(); @@ -5889,6 +5916,9 @@ sub main { my $nibble_iter = $args{NibbleIterator}; my $max_chunk = $nibble_iter->nibble_number(); + # Don't need to do anything here if we're just --explain'ing. + return if $o->get('explain'); + # Wait for all slaves to run all checksum chunks, # then check for differences. if ( $o->get('replicate-check') && scalar @$slaves ) { @@ -6979,7 +7009,7 @@ Do only this comma-separated list of storage engines. =item --explain -group: Output +cumulative: yes; default: 0; group: Output Show, but do not execute, checksum queries (disables L<"--[no]empty-replicate-table">). diff --git a/lib/NibbleIterator.pm b/lib/NibbleIterator.pm index 5377b7f5..5a0e1ba4 100644 --- a/lib/NibbleIterator.pm +++ b/lib/NibbleIterator.pm @@ -82,7 +82,7 @@ sub new { : join(', ', map { $q->quote($_) } @cols)) . " FROM " . $q->quote(@{$tbl}{qw(db tbl)}) . ($args{where} ? " AND ($args{where})" : '') - . " /*one nibble*/"; + . " /*checksum table*/"; MKDEBUG && _d('One nibble statement:', $nibble_sql); my $explain_nibble_sql @@ -91,7 +91,7 @@ sub new { : join(', ', map { $q->quote($_) } @cols)) . " FROM " . $q->quote(@{$tbl}{qw(db tbl)}) . ($args{where} ? " AND ($args{where})" : '') - . " /*explain one nibble*/"; + . " /*explain checksum table*/"; MKDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); $self = { @@ -158,7 +158,7 @@ sub new { . ($args{where} ? " AND ($args{where})" : '') . " ORDER BY $order_by" . " LIMIT ?, 2" - . " /*upper boundary*/"; + . " /*next chunk boundary*/"; MKDEBUG && _d('Upper boundary statement:', $ub_sql); # This statement does the actual nibbling work; its rows are returned @@ -172,7 +172,7 @@ sub new { . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($args{where} ? " AND ($args{where})" : '') . " ORDER BY $order_by" - . " /*nibble*/"; + . " /*checksum chunk*/"; MKDEBUG && _d('Nibble statement:', $nibble_sql); my $explain_nibble_sql @@ -184,7 +184,7 @@ sub new { . " AND " . $asc->{boundaries}->{'<='} # upper boundary . ($args{where} ? " AND ($args{where})" : '') . " ORDER BY $order_by" - . " /*explain nibble*/"; + . " /*explain checksum chunk*/"; MKDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); my $limit = $chunk_size - 1; @@ -364,6 +364,11 @@ sub sql { return $self->{sql}; } +sub more_boundaries { + my ($self) = @_; + return !$self->{no_more_boundaries}; +} + sub _find_best_index { my (%args) = @_; my @required_args = qw(tbl TableParser dbh Quoter); diff --git a/t/lib/samples/char-chunking/ascii.sql b/t/lib/samples/char-chunking/ascii.sql index e8aaebf0..a14e4ba0 100644 --- a/t/lib/samples/char-chunking/ascii.sql +++ b/t/lib/samples/char-chunking/ascii.sql @@ -6,3 +6,149 @@ CREATE TABLE `ascii` ( UNIQUE KEY `c` (`c`) ) ENGINE=MyISAM AUTO_INCREMENT=143 DEFAULT CHARSET=latin1; INSERT INTO `ascii` VALUES (1,''),(2,'M'),(3,'007'),(4,'Penelope'),(5,'Nick'),(6,'Ed'),(7,'Jennifer'),(8,'johnny'),(9,'Bette'),(10,'Grace'),(11,'matthew'),(12,'joe'),(13,'christian'),(14,'zero'),(15,'karl'),(16,'uma'),(17,'vivien'),(18,'Cuba'),(19,'Fred'),(20,'Helen'),(21,'Dan'),(22,'Bob'),(23,'<[MEGATRON]>'),(24,'Lucille'),(25,'Kirsten'),(26,'Elvis'),(27,'Sandra'),(28,'Cameron'),(29,'Kevin'),(30,'Rip'),(31,'Julia'),(32,'Jean-Luc'),(33,'Woody'),(34,'Alec'),(35,'Sissy'),(36,'tim'),(37,'milla'),(38,'audrey'),(39,'judy'),(40,'burt'),(41,'val'),(42,'tom'),(43,'goldie'),(44,'jodie'),(45,'kirk'),(46,'reese'),(47,'parker'),(48,'Frances'),(49,'Anne'),(50,'Natalie'),(51,'Gary'),(52,'Carmen'),(53,'@the ball park'),(54,'Mena'),(55,'Fay'),(56,'Jude'),(57,'Dustin'),(58,'Henry'),(59,'jayne'),(60,'Ray'),(61,'Angela'),(62,'Mary'),(63,'Jessica'),(64,'kenneth'),(65,'Michelle'),(66,'adam'),(67,'Sean'),(68,'Angelina'),(69,'Cary'),(70,'Groucho'),(71,'Mae'),(72,'Ralph'),(73,'Scarlett'),(74,'Ben'),(75,'bling$$$'),(76,'James'),(77,'Minnie'),(78,'Greg'),(79,'spencer'),(80,'charlize'),(81,'christopher'),(82,'Ellen'),(83,'Daryl'),(84,'Gene'),(85,'Meg'),(86,'Chris'),(87,'jim'),(88,'susan'),(89,'walter'),(90,'sidney'),(91,'gina'),(92,'warren'),(93,'Sylvester'),(94,'Russell'),(95,'Morgan'),(96,'Harrison'),(97,'Renee'),(98,'Liza'),(99,'Salma'),(100,'Julianne'),(101,'Albert'),(102,'Cate'),(103,'Greta'),(104,'jane'),(105,'richard'),(106,'rita'),(107,'ewan'),(108,'whoopi'),(109,'jada'),(110,'River'),(111,'Kim'),(112,'Emily'),(113,'Geoffrey'),(114,'meryl'),(115,'Ian'),(116,'Laura'),(117,'Harvey'),(118,'Oprah'),(119,'Humphrey'),(120,'Al'),(121,'laurence'),(122,'will'),(123,'olympia'),(124,'alan'),(125,'michael'),(126,'william'),(127,'jon'),(128,'Mr. Peanut'),(129,'Mr. Rogers'),(130,'lisa'),(131,'Jeff'),(132,'Debbie'),(133,'Rock'),(134,'Gregory'),(135,'John'),(136,'Bela'),(137,'Thora'),(138,'Zesus'),(139,'Zesus!'),(140,'Zesus!!'),(141,'ZESUS!!!'),(142,'001'); + +-- +-- Here's how MySQL sorts these values on my machine: +-- +-- | 1 | | +-- | 2 | 001 | +-- | 3 | 007 | +-- | 4 | <[MEGATRON]> | +-- | 5 | @the ball park | +-- | 6 | adam | +-- | 7 | Al | +-- | 8 | alan | +-- | 9 | Albert | +-- | 10 | Alec | +-- | 11 | Angela | +-- | 12 | Angelina | +-- | 13 | Anne | +-- | 14 | audrey | +-- | 15 | Bela | +-- | 16 | Ben | +-- | 17 | Bette | +-- | 18 | bling$$$ | +-- | 19 | Bob | +-- | 20 | burt | +-- | 21 | Cameron | +-- | 22 | Carmen | +-- | 23 | Cary | +-- | 24 | Cate | +-- | 25 | charlize | +-- | 26 | Chris | +-- | 27 | christian | +-- | 28 | christopher | +-- | 29 | Cuba | +-- | 30 | Dan | +-- | 31 | Daryl | +-- | 32 | Debbie | +-- | 33 | Dustin | +-- | 34 | Ed | +-- | 35 | Ellen | +-- | 36 | Elvis | +-- | 37 | Emily | +-- | 38 | ewan | +-- | 39 | Fay | +-- | 40 | Frances | +-- | 41 | Fred | +-- | 42 | Gary | +-- | 43 | Gene | +-- | 44 | Geoffrey | +-- | 45 | gina | +-- | 46 | goldie | +-- | 47 | Grace | +-- | 48 | Greg | +-- | 49 | Gregory | +-- | 50 | Greta | +-- | 51 | Groucho | +-- | 52 | Harrison | +-- | 53 | Harvey | +-- | 54 | Helen | +-- | 55 | Henry | +-- | 56 | Humphrey | +-- | 57 | Ian | +-- | 58 | jada | +-- | 59 | James | +-- | 60 | jane | +-- | 61 | jayne | +-- | 62 | Jean-Luc | +-- | 63 | Jeff | +-- | 64 | Jennifer | +-- | 65 | Jessica | +-- | 66 | jim | +-- | 67 | jodie | +-- | 68 | joe | +-- | 69 | John | +-- | 70 | johnny | +-- | 71 | jon | +-- | 72 | Jude | +-- | 73 | judy | +-- | 74 | Julia | +-- | 75 | Julianne | +-- | 76 | karl | +-- | 77 | kenneth | +-- | 78 | Kevin | +-- | 79 | Kim | +-- | 80 | kirk | +-- | 81 | Kirsten | +-- | 82 | Laura | +-- | 83 | laurence | +-- | 84 | lisa | +-- | 85 | Liza | +-- | 86 | Lucille | +-- | 87 | M | +-- | 88 | Mae | +-- | 89 | Mary | +-- | 90 | matthew | +-- | 91 | Meg | +-- | 92 | Mena | +-- | 93 | meryl | +-- | 94 | michael | +-- | 95 | Michelle | +-- | 96 | milla | +-- | 97 | Minnie | +-- | 98 | Morgan | +-- | 99 | Mr. Peanut | +-- | 100 | Mr. Rogers | +-- | 101 | Natalie | +-- | 102 | Nick | +-- | 103 | olympia | +-- | 104 | Oprah | +-- | 105 | parker | +-- | 106 | Penelope | +-- | 107 | Ralph | +-- | 108 | Ray | +-- | 109 | reese | +-- | 110 | Renee | +-- | 111 | richard | +-- | 112 | Rip | +-- | 113 | rita | +-- | 114 | River | +-- | 115 | Rock | +-- | 116 | Russell | +-- | 117 | Salma | +-- | 118 | Sandra | +-- | 119 | Scarlett | +-- | 120 | Sean | +-- | 121 | sidney | +-- | 122 | Sissy | +-- | 123 | spencer | +-- | 124 | susan | +-- | 125 | Sylvester | +-- | 126 | Thora | +-- | 127 | tim | +-- | 128 | tom | +-- | 129 | uma | +-- | 130 | val | +-- | 131 | vivien | +-- | 132 | walter | +-- | 133 | warren | +-- | 134 | whoopi | +-- | 135 | will | +-- | 136 | william | +-- | 137 | Woody | +-- | 138 | zero | +-- | 139 | Zesus | +-- | 140 | Zesus! | +-- | 141 | Zesus!! | +-- | 142 | ZESUS!!! | diff --git a/t/pt-table-checksum/char_chunking.t b/t/pt-table-checksum/char_chunking.t index 37a86bcb..d8d416ea 100644 --- a/t/pt-table-checksum/char_chunking.t +++ b/t/pt-table-checksum/char_chunking.t @@ -15,29 +15,30 @@ use PerconaTest; use Sandbox; require "$trunk/bin/pt-table-checksum"; -my $vp = new VersionParser(); -my $dp = new DSNParser(opts=>$dsn_opts); -my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); -my $dbh = $sb->get_dbh_for('master'); +my $dp = new DSNParser(opts=>$dsn_opts); +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); +my $master_dbh = $sb->get_dbh_for('master'); -if ( !$dbh ) { +if ( !$master_dbh ) { plan skip_all => 'Cannot connect to sandbox master'; } else { - plan tests => 3; + plan tests => 2; } -my $output; -my $cnf ='/tmp/12345/my.sandbox.cnf'; -my @args = ("F=$cnf", qw(--lock-wait-timeout 3 --chunk-time 0 --chunk-size-limit 0 --tables test.ascii)); +# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic +# so we need to specify --lock-wait-timeout=3 else the tool will die. +my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox'; +my @args = ($master_dsn, qw(--lock-wait-timeout 3)); -$sb->create_dbs($dbh, ['test']); +$sb->create_dbs($master_dbh, ['test']); $sb->load_file('master', "t/lib/samples/char-chunking/ascii.sql", 'test'); ok( no_diff( sub { pt_table_checksum::main(@args, - qw(--chunk-size 20 --explain)) }, + qw(--tables test.ascii --chunk-index c --chunk-size 20), + qw(--explain --explain)) }, "t/pt-table-checksum/samples/char-chunk-ascii-explain.txt", ), "Char chunk ascii, explain" @@ -46,23 +47,16 @@ ok( ok( no_diff( sub { pt_table_checksum::main(@args, - qw(--chunk-size 20)) }, + qw(--tables test.ascii --chunk-index c --chunk-size 20), + qw(--chunk-time 0)) }, "t/pt-table-checksum/samples/char-chunk-ascii.txt", + post_pipe => 'awk \'{print $2 " " $3 " " $4 " " $5 " " $6 " " $8}\'', ), "Char chunk ascii, chunk size 20" ); -ok( - no_diff( - sub { pt_table_checksum::main(@args, - qw(--chunk-size 20 --chunk-size-limit 3)) }, - "t/pt-table-checksum/samples/char-chunk-ascii-oversize.txt", - ), - "Char chunk ascii, chunk size 20, with oversize" -); - # ############################################################################# # Done. # ############################################################################# -$sb->wipe_clean($dbh); +$sb->wipe_clean($master_dbh); exit; diff --git a/t/pt-table-checksum/samples/char-chunk-ascii-explain.txt b/t/pt-table-checksum/samples/char-chunk-ascii-explain.txt index 9677d3ee..9facc008 100644 --- a/t/pt-table-checksum/samples/char-chunk-ascii-explain.txt +++ b/t/pt-table-checksum/samples/char-chunk-ascii-explain.txt @@ -1,9 +1,17 @@ -test ascii SELECT /*test.ascii:1/8*/ 0 AS chunk_num, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `c`)) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `test`.`ascii` FORCE INDEX (`c`) WHERE (`c` < ')') -test ascii `c` < ')' -test ascii `c` >= ')' AND `c` < '1' -test ascii `c` >= '1' AND `c` < '9' -test ascii `c` >= '9' AND `c` < 'A' -test ascii `c` >= 'A' AND `c` < 'I' -test ascii `c` >= 'I' AND `c` < 'Q' -test ascii `c` >= 'Q' AND `c` < 'Y' -test ascii `c` >= 'Y' +-- +-- test.ascii +-- + +REPLACE INTO `percona`.`checksums` (db, tbl, chunk, chunk_index, lower_boundary, upper_boundary, this_cnt, this_crc) SELECT ?, ?, ?, ?, ?, ?, COUNT(*) AS cnt, COALESCE(LOWER(CONV(BIT_XOR(CAST(CRC32(CONCAT_WS('#', `i`, `c`)) AS UNSIGNED)), 10, 16)), 0) AS crc FROM `test`.`ascii` FORCE INDEX(`c`) WHERE ((`c` >= ?)) AND ((`c` <= ?)) ORDER BY `c` /*checksum chunk*/ + +SELECT /*!40001 SQL_NO_CACHE */ `c` FROM `test`.`ascii` FORCE INDEX(`c`) WHERE ((`c` >= ?)) ORDER BY `c` LIMIT ?, 2 /*next chunk boundary*/ + +1 burt +2 Cameron Frances +3 Fred jane +4 jayne kirk +5 Kirsten Mr. Rogers +6 Natalie Sean +7 sidney Zesus! +8 Zesus!! ZESUS!!! + diff --git a/t/pt-table-checksum/samples/char-chunk-ascii.txt b/t/pt-table-checksum/samples/char-chunk-ascii.txt index 5d4b82b1..cdeb536a 100644 --- a/t/pt-table-checksum/samples/char-chunk-ascii.txt +++ b/t/pt-table-checksum/samples/char-chunk-ascii.txt @@ -1,9 +1,2 @@ -DATABASE TABLE CHUNK HOST ENGINE COUNT CHECKSUM TIME WAIT STAT LAG -test ascii 0 127.1 MyISAM 1 25e3643f 0 0 NULL NULL -test ascii 1 127.1 MyISAM 2 1eb24fbb 0 0 NULL NULL -test ascii 2 127.1 MyISAM 0 NULL 0 0 NULL NULL -test ascii 3 127.1 MyISAM 2 fb3ab74d 0 0 NULL NULL -test ascii 4 127.1 MyISAM OVERSIZE NULL NULL NULL NULL NULL -test ascii 5 127.1 MyISAM OVERSIZE NULL NULL NULL NULL NULL -test ascii 6 127.1 MyISAM 31 77958122 0 0 NULL NULL -test ascii 7 127.1 MyISAM 5 3c0ef0c2 0 0 NULL NULL +ERRORS DIFFS ROWS CHUNKS SKIPPED TABLE +0 0 142 8 0 test.ascii