percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-17 17:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

PT-1554 Updated pt-query-digest to use a proper checksum

Browse Source 
Now, the checksum field is a CHAR(32) instead of a BIGINT.

Previously it was failing on some MySQL versions since the truncated int
it was storing is not consistent. Now, by having an MD5 stored as a
CHAR(32) the checksum is consistent across MySQL and OS versions.

This is a breaking change and pt-query-digest history and review tables
are not backwards compatible.
This commit is contained in:
Carlos Salguero
2018-06-13 11:25:47 -03:00
parent 81582f3ce3
commit f95d448e03
7 changed files with 163 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bin/pt-query-digest
View File

@@ -9250,7 +9250,7 @@ sub new {
my $sql = <<" SQL"; my $sql = <<" SQL";
INSERT INTO $args{db_tbl} INSERT INTO $args{db_tbl}
(checksum, fingerprint, sample, first_seen, last_seen) (checksum, fingerprint, sample, first_seen, last_seen)
VALUES(CONV(?, 16, 10), ?, ?, COALESCE(?, $now), COALESCE(?, $now)) VALUES(?, ?, ?, COALESCE(?, $now), COALESCE(?, $now))
ON DUPLICATE KEY UPDATE ON DUPLICATE KEY UPDATE
first_seen = IF( first_seen = IF(
first_seen IS NULL, first_seen IS NULL,
@@ -9267,8 +9267,8 @@ sub new {
my @review_cols = grep { !$skip_cols{$_} } @{$args{tbl_struct}->{cols}}; my @review_cols = grep { !$skip_cols{$_} } @{$args{tbl_struct}->{cols}};
$sql = "SELECT " $sql = "SELECT "
. join(', ', map { $args{quoter}->quote($_) } @review_cols) . join(', ', map { $args{quoter}->quote($_) } @review_cols)
. ", CONV(checksum, 10, 16) AS checksum_conv FROM $args{db_tbl}" . ", checksum AS checksum_conv FROM $args{db_tbl}"
. " WHERE checksum=CONV(?, 16, 10)"; . " WHERE checksum=?";
PTDEBUG && _d('SQL to select from review table:', $sql); PTDEBUG && _d('SQL to select from review table:', $sql);
my $select_sth = $args{dbh}->prepare($sql); my $select_sth = $args{dbh}->prepare($sql);
@@ -9398,7 +9398,7 @@ sub set_history_options {
my $sql = "REPLACE INTO $args{table}(" my $sql = "REPLACE INTO $args{table}("
. join(', ', . join(', ',
map { Quoter->quote($_) } ('checksum', 'sample', @cols)) map { Quoter->quote($_) } ('checksum', 'sample', @cols))
. ') VALUES (CONV(?, 16, 10), ?' . ') VALUES (?, ?'
. (@cols ? ', ' : '') # issue 1265 . (@cols ? ', ' : '') # issue 1265
. join(', ', map { . join(', ', map {
$_ eq 'ts_min' || $_ eq 'ts_max' $_ eq 'ts_min' || $_ eq 'ts_max'
@@ -15767,7 +15767,7 @@ The following table definition is used for L<"--[no]create-history-table">:
MAGIC_create_history_table MAGIC_create_history_table
CREATE TABLE IF NOT EXISTS query_history ( CREATE TABLE IF NOT EXISTS query_history (
checksum BIGINT UNSIGNED NOT NULL, checksum CHAR(32) NOT NULL,
sample TEXT NOT NULL, sample TEXT NOT NULL,
ts_min DATETIME, ts_min DATETIME,
ts_max DATETIME, ts_max DATETIME,

165
t/pt-query-digest/history.t
View File

@@ -57,6 +57,7 @@ $output = run_with("slow006.txt", '--create-history-table',
my ($table) = $dbh->selectrow_array( my ($table) = $dbh->selectrow_array(
"show tables from test like 'query_review_history'"); "show tables from test like 'query_review_history'");
# 1
is($table, 'query_review_history', '--create-history-table'); is($table, 'query_review_history', '--create-history-table');
my $res = $dbh->selectall_arrayref('SELECT lock_time_median, lock_time_stddev, my $res = $dbh->selectall_arrayref('SELECT lock_time_median, lock_time_stddev,
@@ -76,38 +77,7 @@ test.query_review_history', { Slice => {} } );
my $expected = [ my $expected = [
{ {
checksum => '18446744073709551616.00000000', checksum => '3E4FB43148C4B07CD4CD74934382A184',
lock_time_max => '0.00000000',
lock_time_median => '0.00000000',
lock_time_min => '0.00000000',
lock_time_pct_95 => '0.00000000',
lock_time_stddev => '0.00000000',
lock_time_sum => '0.00000000',
query_time_max => '0.00001200',
query_time_median => '0.00001200',
query_time_min => '0.00001200',
query_time_pct_95 => '0.00001200',
query_time_stddev => '0.00000000',
query_time_sum => '0.00003600',
rows_examined_max => '0.00000000',
rows_examined_median => '0.00000000',
rows_examined_min => '0.00000000',
rows_examined_pct_95 => '0.00000000',
rows_examined_stddev => '0.00000000',
rows_examined_sum => '0.00000000',
rows_sent_max => '0.00000000',
rows_sent_median => '0.00000000',
rows_sent_min => '0.00000000',
rows_sent_pct_95 => '0.00000000',
rows_sent_stddev => '0.00000000',
rows_sent_sum => '0.00000000',
sample => 'SELECT col FROM foo_tbl',
ts_cnt => '3.00000000',
ts_max => '2007-12-18 11:49:30',
ts_min => '2007-12-18 11:48:27'
},
{
checksum => '18446744073709551616.00000000',
lock_time_max => '0.00000000', lock_time_max => '0.00000000',
lock_time_median => '0.00000000', lock_time_median => '0.00000000',
lock_time_min => '0.00000000', lock_time_min => '0.00000000',
@@ -136,6 +106,37 @@ test.query_review_history', { Slice => {} } );
ts_cnt => '3.00000000', ts_cnt => '3.00000000',
ts_max => '2007-12-18 11:49:07', ts_max => '2007-12-18 11:49:07',
ts_min => '2007-12-18 11:48:57' ts_min => '2007-12-18 11:48:57'
},
{
checksum => '538CA093E701E0CBA20C29AF174CE545',
lock_time_max => '0.00000000',
lock_time_median => '0.00000000',
lock_time_min => '0.00000000',
lock_time_pct_95 => '0.00000000',
lock_time_stddev => '0.00000000',
lock_time_sum => '0.00000000',
query_time_max => '0.00001200',
query_time_median => '0.00001200',
query_time_min => '0.00001200',
query_time_pct_95 => '0.00001200',
query_time_stddev => '0.00000000',
query_time_sum => '0.00003600',
rows_examined_max => '0.00000000',
rows_examined_median => '0.00000000',
rows_examined_min => '0.00000000',
rows_examined_pct_95 => '0.00000000',
rows_examined_stddev => '0.00000000',
rows_examined_sum => '0.00000000',
rows_sent_max => '0.00000000',
rows_sent_median => '0.00000000',
rows_sent_min => '0.00000000',
rows_sent_pct_95 => '0.00000000',
rows_sent_stddev => '0.00000000',
rows_sent_sum => '0.00000000',
sample => 'SELECT col FROM foo_tbl',
ts_cnt => '3.00000000',
ts_max => '2007-12-18 11:49:30',
ts_min => '2007-12-18 11:48:27'
} }
]; ];
@@ -147,7 +148,6 @@ is_deeply(
'Adds/updates queries to query review history table', 'Adds/updates queries to query review history table',
); );
run_with("slow006.txt", '--create-history-table', run_with("slow006.txt", '--create-history-table',
'--history', "$dsn"); '--history', "$dsn");
@@ -169,55 +169,51 @@ $res = $dbh->selectall_arrayref( 'SELECT * FROM test.query_review_history',
$expected = [ $expected = [
{ {
sample => "UPDATE foo.bar checksum => '32F63B9B2CE5A9B1B211BA2B8D6D065C',
SET biz = '91848182522'", filesort_cnt => '2',
checksum => '18446744073709551616',
filesort_on_disk_cnt => '2', filesort_on_disk_cnt => '2',
filesort_on_disk_sum => '0', filesort_on_disk_sum => '0',
tmp_table_on_disk_cnt => '2',
tmp_table_on_disk_sum => '0',
filesort_cnt => '2',
filesort_sum => '0', filesort_sum => '0',
full_join_cnt => '2', full_join_cnt => '2',
full_join_sum => '0', full_join_sum => '0',
full_scan_cnt => '2', full_scan_cnt => '2',
full_scan_sum => '0', full_scan_sum => '0',
innodb_io_r_bytes_max => 0, innodb_io_r_bytes_max => '0',
innodb_io_r_bytes_median => 0, innodb_io_r_bytes_median => '0',
innodb_io_r_bytes_min => 0, innodb_io_r_bytes_min => '0',
innodb_io_r_bytes_pct_95 => 0, innodb_io_r_bytes_pct_95 => '0',
innodb_io_r_bytes_stddev => 0, innodb_io_r_bytes_stddev => '0',
innodb_io_r_ops_max => 0, innodb_io_r_ops_max => '0',
innodb_io_r_ops_median => 0, innodb_io_r_ops_median => '0',
innodb_io_r_ops_min => 0, innodb_io_r_ops_min => '0',
innodb_io_r_ops_pct_95 => 0, innodb_io_r_ops_pct_95 => '0',
innodb_io_r_ops_stddev => 0, innodb_io_r_ops_stddev => '0',
innodb_io_r_wait_max => 0, innodb_io_r_wait_max => '0',
innodb_io_r_wait_median => 0, innodb_io_r_wait_median => '0',
innodb_io_r_wait_min => 0, innodb_io_r_wait_min => '0',
innodb_io_r_wait_pct_95 => 0, innodb_io_r_wait_pct_95 => '0',
innodb_io_r_wait_stddev => 0, innodb_io_r_wait_stddev => '0',
innodb_pages_distinct_max => 18, innodb_pages_distinct_max => '18',
innodb_pages_distinct_median => 18, innodb_pages_distinct_median => '18',
innodb_pages_distinct_min => 18, innodb_pages_distinct_min => '18',
innodb_pages_distinct_pct_95 => 18, innodb_pages_distinct_pct_95 => '18',
innodb_pages_distinct_stddev => 0, innodb_pages_distinct_stddev => '0',
innodb_queue_wait_max => 0, innodb_queue_wait_max => '0',
innodb_queue_wait_median => 0, innodb_queue_wait_median => '0',
innodb_queue_wait_min => 0, innodb_queue_wait_min => '0',
innodb_queue_wait_pct_95 => 0, innodb_queue_wait_pct_95 => '0',
innodb_queue_wait_stddev => 0, innodb_queue_wait_stddev => '0',
innodb_rec_lock_wait_max => 0, innodb_rec_lock_wait_max => '0',
innodb_rec_lock_wait_median => 0, innodb_rec_lock_wait_median => '0',
innodb_rec_lock_wait_min => 0, innodb_rec_lock_wait_min => '0',
innodb_rec_lock_wait_pct_95 => 0, innodb_rec_lock_wait_pct_95 => '0',
innodb_rec_lock_wait_stddev => 0, innodb_rec_lock_wait_stddev => '0',
lock_time_max => '0.000027', lock_time_max => '2.7e-05',
lock_time_median => '0.000027', lock_time_median => '2.7e-05',
lock_time_min => '0.000027', lock_time_min => '2.7e-05',
lock_time_pct_95 => '0.000027', lock_time_pct_95 => '2.7e-05',
lock_time_stddev => '0', lock_time_stddev => '0',
lock_time_sum => '0.000054', lock_time_sum => '5.4e-05',
merge_passes_max => '0', merge_passes_max => '0',
merge_passes_median => '0', merge_passes_median => '0',
merge_passes_min => '0', merge_passes_min => '0',
@@ -238,12 +234,12 @@ SET biz = '91848182522'",
rows_affected_pct_95 => undef, rows_affected_pct_95 => undef,
rows_affected_stddev => undef, rows_affected_stddev => undef,
rows_affected_sum => undef, rows_affected_sum => undef,
rows_examined_max => 0, rows_examined_max => '0',
rows_examined_median => '0', rows_examined_median => '0',
rows_examined_min => 0, rows_examined_min => '0',
rows_examined_pct_95 => '0', rows_examined_pct_95 => '0',
rows_examined_stddev => '0', rows_examined_stddev => '0',
rows_examined_sum => 0, rows_examined_sum => '0',
rows_read_max => undef, rows_read_max => undef,
rows_read_median => undef, rows_read_median => undef,
rows_read_min => undef, rows_read_min => undef,
@@ -256,17 +252,22 @@ SET biz = '91848182522'",
rows_sent_pct_95 => '0', rows_sent_pct_95 => '0',
rows_sent_stddev => '0', rows_sent_stddev => '0',
rows_sent_sum => '0', rows_sent_sum => '0',
sample => 'UPDATE foo.bar
SET biz = \'91848182522\'',
tmp_table_cnt => '2', tmp_table_cnt => '2',
tmp_table_on_disk_cnt => '2',
tmp_table_on_disk_sum => '0',
tmp_table_sum => '0', tmp_table_sum => '0',
ts_cnt => 2, ts_cnt => '2',
ts_max => '2007-12-18 11:48:27', ts_max => '2007-12-18 11:48:27',
ts_min => '2007-12-18 11:48:27', ts_min => '2007-12-18 11:48:27'
}, }
]; ];
normalize_numbers($res); normalize_numbers($res);
normalize_numbers($expected); normalize_numbers($expected);
# 4
is_deeply( is_deeply(
$res, $res,
$expected, $expected,
@@ -281,8 +282,8 @@ $dbh->do('use test');
$dbh->do('drop table test.query_review_history'); $dbh->do('drop table test.query_review_history');
# mqd says "The table must have at least the following columns:" # mqd says "The table must have at least the following columns:"
my $min_tbl = "CREATE TABLE query_review_history ( my $min_tbl = "CREATE TABLE test.query_review_history (
checksum BIGINT UNSIGNED NOT NULL, checksum CHAR(32) NOT NULL PRIMARY KEY,
sample TEXT NOT NULL sample TEXT NOT NULL
)"; )";
$dbh->do($min_tbl); $dbh->do($min_tbl);

32
t/pt-query-digest/review.t
View File

@@ -65,7 +65,17 @@ my $res = $dbh->selectall_arrayref( 'SELECT * FROM test.query_review',
my $expected = [ my $expected = [
{ {
checksum => '18446744073709551616.00000000', checksum => '3E4FB43148C4B07CD4CD74934382A184',
comments => undef,
fingerprint => 'select col from bar_tbl',
first_seen => '2007-12-18 11:48:57',
last_seen => '2007-12-18 11:49:07',
reviewed_by => undef,
reviewed_on => undef,
sample => 'SELECT col FROM bar_tbl'
},
{
checksum => '538CA093E701E0CBA20C29AF174CE545',
comments => undef, comments => undef,
fingerprint => 'select col from foo_tbl', fingerprint => 'select col from foo_tbl',
first_seen => '2007-12-18 11:48:27', first_seen => '2007-12-18 11:48:27',
@@ -73,17 +83,7 @@ my $expected = [
reviewed_by => undef, reviewed_by => undef,
reviewed_on => undef, reviewed_on => undef,
sample => 'SELECT col FROM foo_tbl' sample => 'SELECT col FROM foo_tbl'
}, }
# {
# checksum => '15334040482108055552.00000000',
# comments => undef,
# fingerprint => 'select col from bar_tbl',
# first_seen => '2007-12-18 11:48:57',
# last_seen => '2007-12-18 11:49:07',
# reviewed_by => undef,
# reviewed_on => undef,
# sample => 'SELECT col FROM bar_tbl'
# }
]; ];
normalize_numbers($res); normalize_numbers($res);
@@ -100,6 +100,7 @@ is_deeply(
# their respective query review info added to the report. # their respective query review info added to the report.
$output = run_with("slow006.txt", $output = run_with("slow006.txt",
'--review', "$dsn,D=test,t=query_review" ); '--review', "$dsn,D=test,t=query_review" );
# 3
ok( ok(
no_diff($output, "t/pt-query-digest/samples/slow006_AR_1.txt", cmd_output => 1), no_diff($output, "t/pt-query-digest/samples/slow006_AR_1.txt", cmd_output => 1),
'Analyze-review pass 1 reports not-reviewed queries' 'Analyze-review pass 1 reports not-reviewed queries'
@@ -109,9 +110,10 @@ ok(
# not be reported. # not be reported.
$dbh->do('UPDATE test.query_review $dbh->do('UPDATE test.query_review
SET reviewed_by="daniel", reviewed_on="2008-12-24 12:00:00", comments="foo_tbl is ok, so are cranberries" SET reviewed_by="daniel", reviewed_on="2008-12-24 12:00:00", comments="foo_tbl is ok, so are cranberries"
WHERE checksum=11676753765851784517'); WHERE checksum="11676753765851784517"');
$output = run_with("slow006.txt", $output = run_with("slow006.txt",
'--review', "$dsn,D=test,t=query_review"); '--review', "$dsn,D=test,t=query_review");
# 4
ok( ok(
no_diff($output, "t/pt-query-digest/samples/slow006_AR_2.txt", cmd_output => 1), no_diff($output, "t/pt-query-digest/samples/slow006_AR_2.txt", cmd_output => 1),
'Analyze-review pass 2 does not report the reviewed query' 'Analyze-review pass 2 does not report the reviewed query'
@@ -130,7 +132,7 @@ ok(
# Test that reported review info gets all meta-columns dynamically. # Test that reported review info gets all meta-columns dynamically.
$dbh->do('ALTER TABLE test.query_review ADD COLUMN foo INT'); $dbh->do('ALTER TABLE test.query_review ADD COLUMN foo INT');
$dbh->do('UPDATE test.query_review $dbh->do('UPDATE test.query_review
SET foo=42 WHERE checksum=15334040482108055940'); SET foo=42 WHERE checksum="15334040482108055940"');
$output = run_with("slow006.txt", $output = run_with("slow006.txt",
'--review', "$dsn,D=test,t=query_review"); '--review', "$dsn,D=test,t=query_review");
@@ -193,7 +195,7 @@ $output = run_with("slow006.txt", '--no-report',
$res = $dbh->selectall_arrayref('SELECT * FROM test.query_review'); $res = $dbh->selectall_arrayref('SELECT * FROM test.query_review');
is( is(
$res->[0]->[1], $res->[0]->[1],
'select col from foo_tbl', 'select col from bar_tbl',
"--review works with --no-report" "--review works with --no-report"
); );
is( is(

4
t/pt-query-digest/samples/slow006_AR_1.txt
View File

@@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G
# 1s # 1s
# 10s+ # 10s+
# Review information # Review information
# first_seen: 2007-12-18 11:48:27 # first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:30 # last_seen: 2007-12-18 11:49:07
# reviewed_by: # reviewed_by:
# reviewed_on: # reviewed_on:
# comments: # comments:

4
t/pt-query-digest/samples/slow006_AR_2.txt
View File

@@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G
# 1s # 1s
# 10s+ # 10s+
# Review information # Review information
# first_seen: 2007-12-18 11:48:27 # first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:30 # last_seen: 2007-12-18 11:49:07
# reviewed_by: # reviewed_by:
# reviewed_on: # reviewed_on:
# comments: # comments:

4
t/pt-query-digest/samples/slow006_AR_4.txt
View File

@@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G
# 1s # 1s
# 10s+ # 10s+
# Review information # Review information
# first_seen: 2007-12-18 11:48:27 # first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:30 # last_seen: 2007-12-18 11:49:07
# reviewed_by: # reviewed_by:
# reviewed_on: # reviewed_on:
# comments: # comments:

4
t/pt-query-digest/samples/slow006_AR_5.txt
View File

@@ -65,8 +65,8 @@ SELECT col FROM foo_tbl\G
# 1s # 1s
# 10s+ # 10s+
# Review information # Review information
# first_seen: 2007-12-18 11:48:27 # first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:30 # last_seen: 2007-12-18 11:49:07
# reviewed_by: # reviewed_by:
# reviewed_on: # reviewed_on:
# comments: # comments: