percona/percona-toolkit
1
0
Fork 0
mirror of https://github.com/percona/percona-toolkit.git synced 2025-09-17 17:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

PT-1554 Updated pt-query-digest to use a proper checksum

Browse Source 
Now, the checksum field is a CHAR(32) instead of a BIGINT.

Previously it was failing on some MySQL versions since the truncated int
it was storing is not consistent. Now, by having an MD5 stored as a
CHAR(32) the checksum is consistent across MySQL and OS versions.

This is a breaking change and pt-query-digest history and review tables
are not backwards compatible.
This commit is contained in:
Carlos Salguero
2018-06-13 11:25:47 -03:00
parent 81582f3ce3
commit f95d448e03
7 changed files with 163 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bin/pt-query-digest
View File

@@ -9250,7 +9250,7 @@ sub new {
my $sql = <<" SQL";
INSERT INTO $args{db_tbl}
(checksum, fingerprint, sample, first_seen, last_seen)
VALUES(CONV(?, 16, 10), ?, ?, COALESCE(?, $now), COALESCE(?, $now))
VALUES(?, ?, ?, COALESCE(?, $now), COALESCE(?, $now))
ON DUPLICATE KEY UPDATE
first_seen = IF(
first_seen IS NULL,
@@ -9267,8 +9267,8 @@ sub new {
my @review_cols = grep { !$skip_cols{$_} } @{$args{tbl_struct}->{cols}};
$sql = "SELECT "
. join(', ', map { $args{quoter}->quote($_) } @review_cols)
. ", CONV(checksum, 10, 16) AS checksum_conv FROM $args{db_tbl}"
. " WHERE checksum=CONV(?, 16, 10)";
. ", checksum AS checksum_conv FROM $args{db_tbl}"
. " WHERE checksum=?";
PTDEBUG && _d('SQL to select from review table:', $sql);
my $select_sth = $args{dbh}->prepare($sql);
@@ -9398,7 +9398,7 @@ sub set_history_options {
my $sql = "REPLACE INTO $args{table}("
. join(', ',
map { Quoter->quote($_) } ('checksum', 'sample', @cols))
. ') VALUES (CONV(?, 16, 10), ?'
. ') VALUES (?, ?'
. (@cols ? ', ' : '') # issue 1265
. join(', ', map {
$_ eq 'ts_min' || $_ eq 'ts_max'
@@ -15767,7 +15767,7 @@ The following table definition is used for L<"--[no]create-history-table">:
MAGIC_create_history_table
CREATE TABLE IF NOT EXISTS query_history (
checksum BIGINT UNSIGNED NOT NULL,
checksum CHAR(32) NOT NULL,
sample TEXT NOT NULL,
ts_min DATETIME,
ts_max DATETIME,

265
t/pt-query-digest/history.t
View File

@@ -57,6 +57,7 @@ $output = run_with("slow006.txt", '--create-history-table',
my ($table) = $dbh->selectrow_array(
"show tables from test like 'query_review_history'");
# 1
is($table, 'query_review_history', '--create-history-table');
my $res = $dbh->selectall_arrayref('SELECT lock_time_median, lock_time_stddev,
@@ -74,40 +75,9 @@ FORMAT(rows_examined_pct_95, 6) AS rows_examined_pct_95, rows_sent_max,
FORMAT(query_time_median, 6) AS query_time_median, lock_time_sum FROM
test.query_review_history', { Slice => {} } );
my $expected = [
my $expected = [
{
checksum => '18446744073709551616.00000000',
lock_time_max => '0.00000000',
lock_time_median => '0.00000000',
lock_time_min => '0.00000000',
lock_time_pct_95 => '0.00000000',
lock_time_stddev => '0.00000000',
lock_time_sum => '0.00000000',
query_time_max => '0.00001200',
query_time_median => '0.00001200',
query_time_min => '0.00001200',
query_time_pct_95 => '0.00001200',
query_time_stddev => '0.00000000',
query_time_sum => '0.00003600',
rows_examined_max => '0.00000000',
rows_examined_median => '0.00000000',
rows_examined_min => '0.00000000',
rows_examined_pct_95 => '0.00000000',
rows_examined_stddev => '0.00000000',
rows_examined_sum => '0.00000000',
rows_sent_max => '0.00000000',
rows_sent_median => '0.00000000',
rows_sent_min => '0.00000000',
rows_sent_pct_95 => '0.00000000',
rows_sent_stddev => '0.00000000',
rows_sent_sum => '0.00000000',
sample => 'SELECT col FROM foo_tbl',
ts_cnt => '3.00000000',
ts_max => '2007-12-18 11:49:30',
ts_min => '2007-12-18 11:48:27'
},
{
checksum => '18446744073709551616.00000000',
checksum => '3E4FB43148C4B07CD4CD74934382A184',
lock_time_max => '0.00000000',
lock_time_median => '0.00000000',
lock_time_min => '0.00000000',
@@ -136,6 +106,37 @@ test.query_review_history', { Slice => {} } );
ts_cnt => '3.00000000',
ts_max => '2007-12-18 11:49:07',
ts_min => '2007-12-18 11:48:57'
},
{
checksum => '538CA093E701E0CBA20C29AF174CE545',
lock_time_max => '0.00000000',
lock_time_median => '0.00000000',
lock_time_min => '0.00000000',
lock_time_pct_95 => '0.00000000',
lock_time_stddev => '0.00000000',
lock_time_sum => '0.00000000',
query_time_max => '0.00001200',
query_time_median => '0.00001200',
query_time_min => '0.00001200',
query_time_pct_95 => '0.00001200',
query_time_stddev => '0.00000000',
query_time_sum => '0.00003600',
rows_examined_max => '0.00000000',
rows_examined_median => '0.00000000',
rows_examined_min => '0.00000000',
rows_examined_pct_95 => '0.00000000',
rows_examined_stddev => '0.00000000',
rows_examined_sum => '0.00000000',
rows_sent_max => '0.00000000',
rows_sent_median => '0.00000000',
rows_sent_min => '0.00000000',
rows_sent_pct_95 => '0.00000000',
rows_sent_stddev => '0.00000000',
rows_sent_sum => '0.00000000',
sample => 'SELECT col FROM foo_tbl',
ts_cnt => '3.00000000',
ts_max => '2007-12-18 11:49:30',
ts_min => '2007-12-18 11:48:27'
}
];
@@ -147,7 +148,6 @@ is_deeply(
'Adds/updates queries to query review history table',
);
run_with("slow006.txt", '--create-history-table',
'--history', "$dsn");
@@ -167,106 +167,107 @@ run_with("slow002.txt",
$res = $dbh->selectall_arrayref( 'SELECT * FROM test.query_review_history',
{ Slice => {} } );
$expected = [
{
sample => "UPDATE foo.bar
SET biz = '91848182522'",
checksum => '18446744073709551616',
filesort_on_disk_cnt => '2',
filesort_on_disk_sum => '0',
tmp_table_on_disk_cnt => '2',
tmp_table_on_disk_sum => '0',
filesort_cnt => '2',
filesort_sum => '0',
full_join_cnt => '2',
full_join_sum => '0',
full_scan_cnt => '2',
full_scan_sum => '0',
innodb_io_r_bytes_max => 0,
innodb_io_r_bytes_median => 0,
innodb_io_r_bytes_min => 0,
innodb_io_r_bytes_pct_95 => 0,
innodb_io_r_bytes_stddev => 0,
innodb_io_r_ops_max => 0,
innodb_io_r_ops_median => 0,
innodb_io_r_ops_min => 0,
innodb_io_r_ops_pct_95 => 0,
innodb_io_r_ops_stddev => 0,
innodb_io_r_wait_max => 0,
innodb_io_r_wait_median => 0,
innodb_io_r_wait_min => 0,
innodb_io_r_wait_pct_95 => 0,
innodb_io_r_wait_stddev => 0,
innodb_pages_distinct_max => 18,
innodb_pages_distinct_median => 18,
innodb_pages_distinct_min => 18,
innodb_pages_distinct_pct_95 => 18,
innodb_pages_distinct_stddev => 0,
innodb_queue_wait_max => 0,
innodb_queue_wait_median => 0,
innodb_queue_wait_min => 0,
innodb_queue_wait_pct_95 => 0,
innodb_queue_wait_stddev => 0,
innodb_rec_lock_wait_max => 0,
innodb_rec_lock_wait_median => 0,
innodb_rec_lock_wait_min => 0,
innodb_rec_lock_wait_pct_95 => 0,
innodb_rec_lock_wait_stddev => 0,
lock_time_max => '0.000027',
lock_time_median => '0.000027',
lock_time_min => '0.000027',
lock_time_pct_95 => '0.000027',
lock_time_stddev => '0',
lock_time_sum => '0.000054',
merge_passes_max => '0',
merge_passes_median => '0',
merge_passes_min => '0',
merge_passes_pct_95 => '0',
merge_passes_stddev => '0',
merge_passes_sum => '0',
qc_hit_cnt => '2',
qc_hit_sum => '0',
query_time_max => '0.00053',
query_time_median => '0.00053',
query_time_min => '0.00053',
query_time_pct_95 => '0.00053',
query_time_stddev => '0',
query_time_sum => '0.00106',
rows_affected_max => undef,
rows_affected_median => undef,
rows_affected_min => undef,
rows_affected_pct_95 => undef,
rows_affected_stddev => undef,
rows_affected_sum => undef,
rows_examined_max => 0,
rows_examined_median => '0',
rows_examined_min => 0,
rows_examined_pct_95 => '0',
rows_examined_stddev => '0',
rows_examined_sum => 0,
rows_read_max => undef,
rows_read_median => undef,
rows_read_min => undef,
rows_read_pct_95 => undef,
rows_read_stddev => undef,
rows_read_sum => undef,
rows_sent_max => '0',
rows_sent_median => '0',
rows_sent_min => '0',
rows_sent_pct_95 => '0',
rows_sent_stddev => '0',
rows_sent_sum => '0',
tmp_table_cnt => '2',
tmp_table_sum => '0',
ts_cnt => 2,
ts_max => '2007-12-18 11:48:27',
ts_min => '2007-12-18 11:48:27',
},
];
$expected = [
{
checksum => '32F63B9B2CE5A9B1B211BA2B8D6D065C',
filesort_cnt => '2',
filesort_on_disk_cnt => '2',
filesort_on_disk_sum => '0',
filesort_sum => '0',
full_join_cnt => '2',
full_join_sum => '0',
full_scan_cnt => '2',
full_scan_sum => '0',
innodb_io_r_bytes_max => '0',
innodb_io_r_bytes_median => '0',
innodb_io_r_bytes_min => '0',
innodb_io_r_bytes_pct_95 => '0',
innodb_io_r_bytes_stddev => '0',
innodb_io_r_ops_max => '0',
innodb_io_r_ops_median => '0',
innodb_io_r_ops_min => '0',
innodb_io_r_ops_pct_95 => '0',
innodb_io_r_ops_stddev => '0',
innodb_io_r_wait_max => '0',
innodb_io_r_wait_median => '0',
innodb_io_r_wait_min => '0',
innodb_io_r_wait_pct_95 => '0',
innodb_io_r_wait_stddev => '0',
innodb_pages_distinct_max => '18',
innodb_pages_distinct_median => '18',
innodb_pages_distinct_min => '18',
innodb_pages_distinct_pct_95 => '18',
innodb_pages_distinct_stddev => '0',
innodb_queue_wait_max => '0',
innodb_queue_wait_median => '0',
innodb_queue_wait_min => '0',
innodb_queue_wait_pct_95 => '0',
innodb_queue_wait_stddev => '0',
innodb_rec_lock_wait_max => '0',
innodb_rec_lock_wait_median => '0',
innodb_rec_lock_wait_min => '0',
innodb_rec_lock_wait_pct_95 => '0',
innodb_rec_lock_wait_stddev => '0',
lock_time_max => '2.7e-05',
lock_time_median => '2.7e-05',
lock_time_min => '2.7e-05',
lock_time_pct_95 => '2.7e-05',
lock_time_stddev => '0',
lock_time_sum => '5.4e-05',
merge_passes_max => '0',
merge_passes_median => '0',
merge_passes_min => '0',
merge_passes_pct_95 => '0',
merge_passes_stddev => '0',
merge_passes_sum => '0',
qc_hit_cnt => '2',
qc_hit_sum => '0',
query_time_max => '0.00053',
query_time_median => '0.00053',
query_time_min => '0.00053',
query_time_pct_95 => '0.00053',
query_time_stddev => '0',
query_time_sum => '0.00106',
rows_affected_max => undef,
rows_affected_median => undef,
rows_affected_min => undef,
rows_affected_pct_95 => undef,
rows_affected_stddev => undef,
rows_affected_sum => undef,
rows_examined_max => '0',
rows_examined_median => '0',
rows_examined_min => '0',
rows_examined_pct_95 => '0',
rows_examined_stddev => '0',
rows_examined_sum => '0',
rows_read_max => undef,
rows_read_median => undef,
rows_read_min => undef,
rows_read_pct_95 => undef,
rows_read_stddev => undef,
rows_read_sum => undef,
rows_sent_max => '0',
rows_sent_median => '0',
rows_sent_min => '0',
rows_sent_pct_95 => '0',
rows_sent_stddev => '0',
rows_sent_sum => '0',
sample => 'UPDATE foo.bar
SET biz = \'91848182522\'',
tmp_table_cnt => '2',
tmp_table_on_disk_cnt => '2',
tmp_table_on_disk_sum => '0',
tmp_table_sum => '0',
ts_cnt => '2',
ts_max => '2007-12-18 11:48:27',
ts_min => '2007-12-18 11:48:27'
}
];
normalize_numbers($res);
normalize_numbers($expected);
# 4
is_deeply(
$res,
$expected,
@@ -281,8 +282,8 @@ $dbh->do('use test');
$dbh->do('drop table test.query_review_history');
# mqd says "The table must have at least the following columns:"
my $min_tbl = "CREATE TABLE query_review_history (
checksum BIGINT UNSIGNED NOT NULL,
my $min_tbl = "CREATE TABLE test.query_review_history (
checksum CHAR(32) NOT NULL PRIMARY KEY,
sample TEXT NOT NULL
)";
$dbh->do($min_tbl);

32
t/pt-query-digest/review.t
View File

@@ -65,7 +65,17 @@ my $res = $dbh->selectall_arrayref( 'SELECT * FROM test.query_review',
my $expected = [
{
checksum => '18446744073709551616.00000000',
checksum => '3E4FB43148C4B07CD4CD74934382A184',
comments => undef,
fingerprint => 'select col from bar_tbl',
first_seen => '2007-12-18 11:48:57',
last_seen => '2007-12-18 11:49:07',
reviewed_by => undef,
reviewed_on => undef,
sample => 'SELECT col FROM bar_tbl'
},
{
checksum => '538CA093E701E0CBA20C29AF174CE545',
comments => undef,
fingerprint => 'select col from foo_tbl',
first_seen => '2007-12-18 11:48:27',
@@ -73,17 +83,7 @@ my $expected = [
reviewed_by => undef,
reviewed_on => undef,
sample => 'SELECT col FROM foo_tbl'
},
# {
# checksum => '15334040482108055552.00000000',
# comments => undef,
# fingerprint => 'select col from bar_tbl',
# first_seen => '2007-12-18 11:48:57',
# last_seen => '2007-12-18 11:49:07',
# reviewed_by => undef,
# reviewed_on => undef,
# sample => 'SELECT col FROM bar_tbl'
# }
}
];
normalize_numbers($res);
@@ -100,6 +100,7 @@ is_deeply(
# their respective query review info added to the report.
$output = run_with("slow006.txt",
'--review', "$dsn,D=test,t=query_review" );
# 3
ok(
no_diff($output, "t/pt-query-digest/samples/slow006_AR_1.txt", cmd_output => 1),
'Analyze-review pass 1 reports not-reviewed queries'
@@ -109,9 +110,10 @@ ok(
# not be reported.
$dbh->do('UPDATE test.query_review
SET reviewed_by="daniel", reviewed_on="2008-12-24 12:00:00", comments="foo_tbl is ok, so are cranberries"
WHERE checksum=11676753765851784517');
WHERE checksum="11676753765851784517"');
$output = run_with("slow006.txt",
'--review', "$dsn,D=test,t=query_review");
# 4
ok(
no_diff($output, "t/pt-query-digest/samples/slow006_AR_2.txt", cmd_output => 1),
'Analyze-review pass 2 does not report the reviewed query'
@@ -130,7 +132,7 @@ ok(
# Test that reported review info gets all meta-columns dynamically.
$dbh->do('ALTER TABLE test.query_review ADD COLUMN foo INT');
$dbh->do('UPDATE test.query_review
SET foo=42 WHERE checksum=15334040482108055940');
SET foo=42 WHERE checksum="15334040482108055940"');
$output = run_with("slow006.txt",
'--review', "$dsn,D=test,t=query_review");
@@ -193,7 +195,7 @@ $output = run_with("slow006.txt", '--no-report',
$res = $dbh->selectall_arrayref('SELECT * FROM test.query_review');
is(
$res->[0]->[1],
'select col from foo_tbl',
'select col from bar_tbl',
"--review works with --no-report"
);
is(

4
t/pt-query-digest/samples/slow006_AR_1.txt
View File

@@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G
# 1s
# 10s+
# Review information
# first_seen: 2007-12-18 11:48:27
# last_seen: 2007-12-18 11:49:30
# first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:07
# reviewed_by:
# reviewed_on:
# comments:

4
t/pt-query-digest/samples/slow006_AR_2.txt
View File

@@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G
# 1s
# 10s+
# Review information
# first_seen: 2007-12-18 11:48:27
# last_seen: 2007-12-18 11:49:30
# first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:07
# reviewed_by:
# reviewed_on:
# comments:

4
t/pt-query-digest/samples/slow006_AR_4.txt
View File

@@ -64,8 +64,8 @@ SELECT col FROM foo_tbl\G
# 1s
# 10s+
# Review information
# first_seen: 2007-12-18 11:48:27
# last_seen: 2007-12-18 11:49:30
# first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:07
# reviewed_by:
# reviewed_on:
# comments:

4
t/pt-query-digest/samples/slow006_AR_5.txt
View File

@@ -65,8 +65,8 @@ SELECT col FROM foo_tbl\G
# 1s
# 10s+
# Review information
# first_seen: 2007-12-18 11:48:27
# last_seen: 2007-12-18 11:49:30
# first_seen: 2007-12-18 11:48:57
# last_seen: 2007-12-18 11:49:07
# reviewed_by:
# reviewed_on:
# comments: