From f95d448e035b9aafa4f21667da229bb1a11b9ecb Mon Sep 17 00:00:00 2001 From: Carlos Salguero Date: Wed, 13 Jun 2018 11:25:47 -0300 Subject: [PATCH] PT-1554 Updated pt-query-digest to use a proper checksum Now, the checksum field is a CHAR(32) instead of a BIGINT. Previously it was failing on some MySQL versions since the truncated int it was storing is not consistent. Now, by having an MD5 stored as a CHAR(32) the checksum is consistent across MySQL and OS versions. This is a breaking change and pt-query-digest history and review tables are not backwards compatible. --- bin/pt-query-digest | 10 +- t/pt-query-digest/history.t | 265 +++++++++++---------- t/pt-query-digest/review.t | 32 +-- t/pt-query-digest/samples/slow006_AR_1.txt | 4 +- t/pt-query-digest/samples/slow006_AR_2.txt | 4 +- t/pt-query-digest/samples/slow006_AR_4.txt | 4 +- t/pt-query-digest/samples/slow006_AR_5.txt | 4 +- 7 files changed, 163 insertions(+), 160 deletions(-) diff --git a/bin/pt-query-digest b/bin/pt-query-digest index ef358598..9280a690 100755 --- a/bin/pt-query-digest +++ b/bin/pt-query-digest @@ -9250,7 +9250,7 @@ sub new { my $sql = <<" SQL"; INSERT INTO $args{db_tbl} (checksum, fingerprint, sample, first_seen, last_seen) - VALUES(CONV(?, 16, 10), ?, ?, COALESCE(?, $now), COALESCE(?, $now)) + VALUES(?, ?, ?, COALESCE(?, $now), COALESCE(?, $now)) ON DUPLICATE KEY UPDATE first_seen = IF( first_seen IS NULL, @@ -9267,8 +9267,8 @@ sub new { my @review_cols = grep { !$skip_cols{$_} } @{$args{tbl_struct}->{cols}}; $sql = "SELECT " . join(', ', map { $args{quoter}->quote($_) } @review_cols) - . ", CONV(checksum, 10, 16) AS checksum_conv FROM $args{db_tbl}" - . " WHERE checksum=CONV(?, 16, 10)"; + . ", checksum AS checksum_conv FROM $args{db_tbl}" + . " WHERE checksum=?"; PTDEBUG && _d('SQL to select from review table:', $sql); my $select_sth = $args{dbh}->prepare($sql); @@ -9398,7 +9398,7 @@ sub set_history_options { my $sql = "REPLACE INTO $args{table}(" . join(', ', map { Quoter->quote($_) } ('checksum', 'sample', @cols)) - . ') VALUES (CONV(?, 16, 10), ?' + . ') VALUES (?, ?' . (@cols ? ', ' : '') # issue 1265 . join(', ', map { $_ eq 'ts_min' || $_ eq 'ts_max' @@ -15767,7 +15767,7 @@ The following table definition is used for L<"--[no]create-history-table">: MAGIC_create_history_table CREATE TABLE IF NOT EXISTS query_history ( - checksum BIGINT UNSIGNED NOT NULL, + checksum CHAR(32) NOT NULL, sample TEXT NOT NULL, ts_min DATETIME, ts_max DATETIME, diff --git a/t/pt-query-digest/history.t b/t/pt-query-digest/history.t index eb1ec119..81ccce02 100644 --- a/t/pt-query-digest/history.t +++ b/t/pt-query-digest/history.t @@ -57,6 +57,7 @@ $output = run_with("slow006.txt", '--create-history-table', my ($table) = $dbh->selectrow_array( "show tables from test like 'query_review_history'"); +# 1 is($table, 'query_review_history', '--create-history-table'); my $res = $dbh->selectall_arrayref('SELECT lock_time_median, lock_time_stddev, @@ -74,40 +75,9 @@ FORMAT(rows_examined_pct_95, 6) AS rows_examined_pct_95, rows_sent_max, FORMAT(query_time_median, 6) AS query_time_median, lock_time_sum FROM test.query_review_history', { Slice => {} } ); - my $expected = [ +my $expected = [ { - checksum => '18446744073709551616.00000000', - lock_time_max => '0.00000000', - lock_time_median => '0.00000000', - lock_time_min => '0.00000000', - lock_time_pct_95 => '0.00000000', - lock_time_stddev => '0.00000000', - lock_time_sum => '0.00000000', - query_time_max => '0.00001200', - query_time_median => '0.00001200', - query_time_min => '0.00001200', - query_time_pct_95 => '0.00001200', - query_time_stddev => '0.00000000', - query_time_sum => '0.00003600', - rows_examined_max => '0.00000000', - rows_examined_median => '0.00000000', - rows_examined_min => '0.00000000', - rows_examined_pct_95 => '0.00000000', - rows_examined_stddev => '0.00000000', - rows_examined_sum => '0.00000000', - rows_sent_max => '0.00000000', - rows_sent_median => '0.00000000', - rows_sent_min => '0.00000000', - rows_sent_pct_95 => '0.00000000', - rows_sent_stddev => '0.00000000', - rows_sent_sum => '0.00000000', - sample => 'SELECT col FROM foo_tbl', - ts_cnt => '3.00000000', - ts_max => '2007-12-18 11:49:30', - ts_min => '2007-12-18 11:48:27' - }, - { - checksum => '18446744073709551616.00000000', + checksum => '3E4FB43148C4B07CD4CD74934382A184', lock_time_max => '0.00000000', lock_time_median => '0.00000000', lock_time_min => '0.00000000', @@ -136,6 +106,37 @@ test.query_review_history', { Slice => {} } ); ts_cnt => '3.00000000', ts_max => '2007-12-18 11:49:07', ts_min => '2007-12-18 11:48:57' + }, + { + checksum => '538CA093E701E0CBA20C29AF174CE545', + lock_time_max => '0.00000000', + lock_time_median => '0.00000000', + lock_time_min => '0.00000000', + lock_time_pct_95 => '0.00000000', + lock_time_stddev => '0.00000000', + lock_time_sum => '0.00000000', + query_time_max => '0.00001200', + query_time_median => '0.00001200', + query_time_min => '0.00001200', + query_time_pct_95 => '0.00001200', + query_time_stddev => '0.00000000', + query_time_sum => '0.00003600', + rows_examined_max => '0.00000000', + rows_examined_median => '0.00000000', + rows_examined_min => '0.00000000', + rows_examined_pct_95 => '0.00000000', + rows_examined_stddev => '0.00000000', + rows_examined_sum => '0.00000000', + rows_sent_max => '0.00000000', + rows_sent_median => '0.00000000', + rows_sent_min => '0.00000000', + rows_sent_pct_95 => '0.00000000', + rows_sent_stddev => '0.00000000', + rows_sent_sum => '0.00000000', + sample => 'SELECT col FROM foo_tbl', + ts_cnt => '3.00000000', + ts_max => '2007-12-18 11:49:30', + ts_min => '2007-12-18 11:48:27' } ]; @@ -147,7 +148,6 @@ is_deeply( 'Adds/updates queries to query review history table', ); - run_with("slow006.txt", '--create-history-table', '--history', "$dsn"); @@ -167,106 +167,107 @@ run_with("slow002.txt", $res = $dbh->selectall_arrayref( 'SELECT * FROM test.query_review_history', { Slice => {} } ); -$expected = [ - { - sample => "UPDATE foo.bar -SET biz = '91848182522'", - checksum => '18446744073709551616', - filesort_on_disk_cnt => '2', - filesort_on_disk_sum => '0', - tmp_table_on_disk_cnt => '2', - tmp_table_on_disk_sum => '0', - filesort_cnt => '2', - filesort_sum => '0', - full_join_cnt => '2', - full_join_sum => '0', - full_scan_cnt => '2', - full_scan_sum => '0', - innodb_io_r_bytes_max => 0, - innodb_io_r_bytes_median => 0, - innodb_io_r_bytes_min => 0, - innodb_io_r_bytes_pct_95 => 0, - innodb_io_r_bytes_stddev => 0, - innodb_io_r_ops_max => 0, - innodb_io_r_ops_median => 0, - innodb_io_r_ops_min => 0, - innodb_io_r_ops_pct_95 => 0, - innodb_io_r_ops_stddev => 0, - innodb_io_r_wait_max => 0, - innodb_io_r_wait_median => 0, - innodb_io_r_wait_min => 0, - innodb_io_r_wait_pct_95 => 0, - innodb_io_r_wait_stddev => 0, - innodb_pages_distinct_max => 18, - innodb_pages_distinct_median => 18, - innodb_pages_distinct_min => 18, - innodb_pages_distinct_pct_95 => 18, - innodb_pages_distinct_stddev => 0, - innodb_queue_wait_max => 0, - innodb_queue_wait_median => 0, - innodb_queue_wait_min => 0, - innodb_queue_wait_pct_95 => 0, - innodb_queue_wait_stddev => 0, - innodb_rec_lock_wait_max => 0, - innodb_rec_lock_wait_median => 0, - innodb_rec_lock_wait_min => 0, - innodb_rec_lock_wait_pct_95 => 0, - innodb_rec_lock_wait_stddev => 0, - lock_time_max => '0.000027', - lock_time_median => '0.000027', - lock_time_min => '0.000027', - lock_time_pct_95 => '0.000027', - lock_time_stddev => '0', - lock_time_sum => '0.000054', - merge_passes_max => '0', - merge_passes_median => '0', - merge_passes_min => '0', - merge_passes_pct_95 => '0', - merge_passes_stddev => '0', - merge_passes_sum => '0', - qc_hit_cnt => '2', - qc_hit_sum => '0', - query_time_max => '0.00053', - query_time_median => '0.00053', - query_time_min => '0.00053', - query_time_pct_95 => '0.00053', - query_time_stddev => '0', - query_time_sum => '0.00106', - rows_affected_max => undef, - rows_affected_median => undef, - rows_affected_min => undef, - rows_affected_pct_95 => undef, - rows_affected_stddev => undef, - rows_affected_sum => undef, - rows_examined_max => 0, - rows_examined_median => '0', - rows_examined_min => 0, - rows_examined_pct_95 => '0', - rows_examined_stddev => '0', - rows_examined_sum => 0, - rows_read_max => undef, - rows_read_median => undef, - rows_read_min => undef, - rows_read_pct_95 => undef, - rows_read_stddev => undef, - rows_read_sum => undef, - rows_sent_max => '0', - rows_sent_median => '0', - rows_sent_min => '0', - rows_sent_pct_95 => '0', - rows_sent_stddev => '0', - rows_sent_sum => '0', - tmp_table_cnt => '2', - tmp_table_sum => '0', - ts_cnt => 2, - ts_max => '2007-12-18 11:48:27', - ts_min => '2007-12-18 11:48:27', - }, - ]; +$expected = [ + { + checksum => '32F63B9B2CE5A9B1B211BA2B8D6D065C', + filesort_cnt => '2', + filesort_on_disk_cnt => '2', + filesort_on_disk_sum => '0', + filesort_sum => '0', + full_join_cnt => '2', + full_join_sum => '0', + full_scan_cnt => '2', + full_scan_sum => '0', + innodb_io_r_bytes_max => '0', + innodb_io_r_bytes_median => '0', + innodb_io_r_bytes_min => '0', + innodb_io_r_bytes_pct_95 => '0', + innodb_io_r_bytes_stddev => '0', + innodb_io_r_ops_max => '0', + innodb_io_r_ops_median => '0', + innodb_io_r_ops_min => '0', + innodb_io_r_ops_pct_95 => '0', + innodb_io_r_ops_stddev => '0', + innodb_io_r_wait_max => '0', + innodb_io_r_wait_median => '0', + innodb_io_r_wait_min => '0', + innodb_io_r_wait_pct_95 => '0', + innodb_io_r_wait_stddev => '0', + innodb_pages_distinct_max => '18', + innodb_pages_distinct_median => '18', + innodb_pages_distinct_min => '18', + innodb_pages_distinct_pct_95 => '18', + innodb_pages_distinct_stddev => '0', + innodb_queue_wait_max => '0', + innodb_queue_wait_median => '0', + innodb_queue_wait_min => '0', + innodb_queue_wait_pct_95 => '0', + innodb_queue_wait_stddev => '0', + innodb_rec_lock_wait_max => '0', + innodb_rec_lock_wait_median => '0', + innodb_rec_lock_wait_min => '0', + innodb_rec_lock_wait_pct_95 => '0', + innodb_rec_lock_wait_stddev => '0', + lock_time_max => '2.7e-05', + lock_time_median => '2.7e-05', + lock_time_min => '2.7e-05', + lock_time_pct_95 => '2.7e-05', + lock_time_stddev => '0', + lock_time_sum => '5.4e-05', + merge_passes_max => '0', + merge_passes_median => '0', + merge_passes_min => '0', + merge_passes_pct_95 => '0', + merge_passes_stddev => '0', + merge_passes_sum => '0', + qc_hit_cnt => '2', + qc_hit_sum => '0', + query_time_max => '0.00053', + query_time_median => '0.00053', + query_time_min => '0.00053', + query_time_pct_95 => '0.00053', + query_time_stddev => '0', + query_time_sum => '0.00106', + rows_affected_max => undef, + rows_affected_median => undef, + rows_affected_min => undef, + rows_affected_pct_95 => undef, + rows_affected_stddev => undef, + rows_affected_sum => undef, + rows_examined_max => '0', + rows_examined_median => '0', + rows_examined_min => '0', + rows_examined_pct_95 => '0', + rows_examined_stddev => '0', + rows_examined_sum => '0', + rows_read_max => undef, + rows_read_median => undef, + rows_read_min => undef, + rows_read_pct_95 => undef, + rows_read_stddev => undef, + rows_read_sum => undef, + rows_sent_max => '0', + rows_sent_median => '0', + rows_sent_min => '0', + rows_sent_pct_95 => '0', + rows_sent_stddev => '0', + rows_sent_sum => '0', + sample => 'UPDATE foo.bar +SET biz = \'91848182522\'', + tmp_table_cnt => '2', + tmp_table_on_disk_cnt => '2', + tmp_table_on_disk_sum => '0', + tmp_table_sum => '0', + ts_cnt => '2', + ts_max => '2007-12-18 11:48:27', + ts_min => '2007-12-18 11:48:27' + } +]; normalize_numbers($res); normalize_numbers($expected); +# 4 is_deeply( $res, $expected, @@ -281,8 +282,8 @@ $dbh->do('use test'); $dbh->do('drop table test.query_review_history'); # mqd says "The table must have at least the following columns:" -my $min_tbl = "CREATE TABLE query_review_history ( - checksum BIGINT UNSIGNED NOT NULL, +my $min_tbl = "CREATE TABLE test.query_review_history ( + checksum CHAR(32) NOT NULL PRIMARY KEY, sample TEXT NOT NULL )"; $dbh->do($min_tbl); diff --git a/t/pt-query-digest/review.t b/t/pt-query-digest/review.t index 5bf0e813..05c8ad9d 100644 --- a/t/pt-query-digest/review.t +++ b/t/pt-query-digest/review.t @@ -65,7 +65,17 @@ my $res = $dbh->selectall_arrayref( 'SELECT * FROM test.query_review', my $expected = [ { - checksum => '18446744073709551616.00000000', + checksum => '3E4FB43148C4B07CD4CD74934382A184', + comments => undef, + fingerprint => 'select col from bar_tbl', + first_seen => '2007-12-18 11:48:57', + last_seen => '2007-12-18 11:49:07', + reviewed_by => undef, + reviewed_on => undef, + sample => 'SELECT col FROM bar_tbl' + }, + { + checksum => '538CA093E701E0CBA20C29AF174CE545', comments => undef, fingerprint => 'select col from foo_tbl', first_seen => '2007-12-18 11:48:27', @@ -73,17 +83,7 @@ my $expected = [ reviewed_by => undef, reviewed_on => undef, sample => 'SELECT col FROM foo_tbl' - }, - # { - # checksum => '15334040482108055552.00000000', - # comments => undef, - # fingerprint => 'select col from bar_tbl', - # first_seen => '2007-12-18 11:48:57', - # last_seen => '2007-12-18 11:49:07', - # reviewed_by => undef, - # reviewed_on => undef, - # sample => 'SELECT col FROM bar_tbl' - # } + } ]; normalize_numbers($res); @@ -100,6 +100,7 @@ is_deeply( # their respective query review info added to the report. $output = run_with("slow006.txt", '--review', "$dsn,D=test,t=query_review" ); +# 3 ok( no_diff($output, "t/pt-query-digest/samples/slow006_AR_1.txt", cmd_output => 1), 'Analyze-review pass 1 reports not-reviewed queries' @@ -109,9 +110,10 @@ ok( # not be reported. $dbh->do('UPDATE test.query_review SET reviewed_by="daniel", reviewed_on="2008-12-24 12:00:00", comments="foo_tbl is ok, so are cranberries" - WHERE checksum=11676753765851784517'); + WHERE checksum="11676753765851784517"'); $output = run_with("slow006.txt", '--review', "$dsn,D=test,t=query_review"); +# 4 ok( no_diff($output, "t/pt-query-digest/samples/slow006_AR_2.txt", cmd_output => 1), 'Analyze-review pass 2 does not report the reviewed query' @@ -130,7 +132,7 @@ ok( # Test that reported review info gets all meta-columns dynamically. $dbh->do('ALTER TABLE test.query_review ADD COLUMN foo INT'); $dbh->do('UPDATE test.query_review - SET foo=42 WHERE checksum=15334040482108055940'); + SET foo=42 WHERE checksum="15334040482108055940"'); $output = run_with("slow006.txt", '--review', "$dsn,D=test,t=query_review"); @@ -193,7 +195,7 @@ $output = run_with("slow006.txt", '--no-report', $res = $dbh->selectall_arrayref('SELECT * FROM test.query_review'); is( $res->[0]->[1], - 'select col from foo_tbl', + 'select col from bar_tbl', "--review works with --no-report" ); is( diff --git a/t/pt-query-digest/samples/slow006_AR_1.txt b/t/pt-query-digest/samples/slow006_AR_1.txt index 2a8d50f5..adf86418 100644 --- a/t/pt-query-digest/samples/slow006_AR_1.txt +++ b/t/pt-query-digest/samples/slow006_AR_1.txt @@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G # 1s # 10s+ # Review information -# first_seen: 2007-12-18 11:48:27 -# last_seen: 2007-12-18 11:49:30 +# first_seen: 2007-12-18 11:48:57 +# last_seen: 2007-12-18 11:49:07 # reviewed_by: # reviewed_on: # comments: diff --git a/t/pt-query-digest/samples/slow006_AR_2.txt b/t/pt-query-digest/samples/slow006_AR_2.txt index 2a8d50f5..adf86418 100644 --- a/t/pt-query-digest/samples/slow006_AR_2.txt +++ b/t/pt-query-digest/samples/slow006_AR_2.txt @@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G # 1s # 10s+ # Review information -# first_seen: 2007-12-18 11:48:27 -# last_seen: 2007-12-18 11:49:30 +# first_seen: 2007-12-18 11:48:57 +# last_seen: 2007-12-18 11:49:07 # reviewed_by: # reviewed_on: # comments: diff --git a/t/pt-query-digest/samples/slow006_AR_4.txt b/t/pt-query-digest/samples/slow006_AR_4.txt index 2a8d50f5..adf86418 100644 --- a/t/pt-query-digest/samples/slow006_AR_4.txt +++ b/t/pt-query-digest/samples/slow006_AR_4.txt @@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G # 1s # 10s+ # Review information -# first_seen: 2007-12-18 11:48:27 -# last_seen: 2007-12-18 11:49:30 +# first_seen: 2007-12-18 11:48:57 +# last_seen: 2007-12-18 11:49:07 # reviewed_by: # reviewed_on: # comments: diff --git a/t/pt-query-digest/samples/slow006_AR_5.txt b/t/pt-query-digest/samples/slow006_AR_5.txt index eb50c7fa..6abebf21 100644 --- a/t/pt-query-digest/samples/slow006_AR_5.txt +++ b/t/pt-query-digest/samples/slow006_AR_5.txt @@ -65,8 +65,8 @@ SELECT col FROM foo_tbl\G # 1s # 10s+ # Review information -# first_seen: 2007-12-18 11:48:27 -# last_seen: 2007-12-18 11:49:30 +# first_seen: 2007-12-18 11:48:57 +# last_seen: 2007-12-18 11:49:07 # reviewed_by: # reviewed_on: # comments: