From 56eea61daef3480cfb16903d320b33f7d3e48da7 Mon Sep 17 00:00:00 2001 From: Sveta Smirnova Date: Wed, 17 Jan 2024 02:26:07 +0300 Subject: [PATCH] Port improved pt-pmp Initial port of https://github.com/Percona-Lab/pt-pmp: - Added bin/pt-eustack-resolver - Ported changes to bin/pt-pmp --- bin/pt-eustack-resolver | 249 ++++++++++++++++++++++++++++++++++++++++ bin/pt-pmp | 215 +++++++++++++++++++++++----------- 2 files changed, 395 insertions(+), 69 deletions(-) create mode 100755 bin/pt-eustack-resolver diff --git a/bin/pt-eustack-resolver b/bin/pt-eustack-resolver new file mode 100755 index 00000000..b751a9e9 --- /dev/null +++ b/bin/pt-eustack-resolver @@ -0,0 +1,249 @@ +#!/usr/bin/env perl + +# This program is part of Percona Toolkit: http://www.percona.com/software/ +# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal +# notices and disclaimers. + +use strict; +use Data::Dumper; + +my $mmap; + +sub load_mapping { + my ($pid)= @_; + + if ($pid =~ /^[0-9]+$/) { + open F, '<', "/proc/$pid/maps" + or die "Failed to open /proc/$pid/maps: $!\n"; + } else { + open F, '<', $pid + or die "Failed to open saved map file '$pid': $!\n"; + } + + my $arr= []; + while () { + next unless m/^([a-f0-9]+)-([a-f0-9]+) ..x. ([a-f0-9]+) [a-f0-9:]+ [0-9]+ +(.*)/; + push @$arr, { S => hex($1), E => hex($2), B => hex($3), F => $4 }; + } + close F; + sort { $a->{S} <=> $b->{S} } @$arr; + $mmap= $arr; +} + +my $syms= { }; + +sub get_image { + my ($addr)= @_; + my $e; + for $e (@$mmap) { + next if $e->{E} <= $addr; + last if $e->{S} > $addr; + # Found, look up. + return $e->{F}; + } + return ""; +} + +die "Usage: $0 " unless @ARGV == 1; + +my $pid= $ARGV[0]; +load_mapping($pid); + +#for (@$mmap) { +# printf "0x%x - 0x%x (0x%x): %s\n", $_->{S}, $_->{E}, $_->{B}, $_->{F}; +#} + +open (STACK_TRACE, "eu-stack -q -p $pid 2>/dev/null|") or die "open(): $!"; +my @lines= ; +close(STACK_TRACE); + +my $frame_no= 0; +my %addr=(); +my %sf=(); +my $lwp; + +for my $line (@lines) { + if ($line =~ /^TID ([0-9]+):/) + { + $frame_no= 0; + $lwp=$1; + } + elsif ($line =~ /^#[0-9]+?\s*0x([a-f0-9]+)/) + { + push @{$sf{$lwp}},$1; + $addr{$1}=[get_image(hex($1)),""]; + } else { + #print $line; + } +} + +my %inverse; +push @{ $inverse{ $addr{$_}->[0] } }, $_ for keys %addr; + +foreach my $bin (keys %inverse) +{ + my $addrs=join(" ",@{$inverse{$bin}}); + my @resolved=(); + + @resolved=(`eu-addr2line --pretty-print -s -C -f -p $pid $addrs`); + + my $idx=0; + foreach $a (@{$inverse{$bin}}) + { + $addr{$a}->[1]=$resolved[$idx]; + $addr{$a}->[1]=~ s/\n//; + $addr{$a}->[1]=~ s/at \?\?:0/from $addr{$a}->[0]/; + $idx++; + } +} + +foreach $lwp (sort {$a<=>$b} keys %sf) +{ + my $idx=0; + print "Thread $lwp (LWP $lwp):\n"; + foreach $frame_no (@{$sf{$lwp}}) + { + print join(" ","#".$idx, "0x".$frame_no,"in", $addr{$frame_no}->[1]),"\n"; + $idx++; + } + print "\n"; +} + + +# ############################################################################ +# Documentation +# ############################################################################ +=pod + +=head1 NAME + +pt-eustack-resover - Get stack traces for a selected program with eu-stack and resolve symbols + +=head1 SYNOPSIS + +Usage: pt-eustack-resolver + +=head1 RISKS + +Percona Toolkit is mature, proven in the real world, and well tested, +but all database tools can pose a risk to the system and the database +server. Before using this tool, please: + +=over + +=item * Read the tool's documentation + +=item * Review the tool's known L<"BUGS"> + +=item * Test the tool on a non-production server + +=item * Backup your production server and verify the backups + +=back + +=head1 DESCRIPTION + +pt-eustack-resover is the tool that gets stack traces for a selected program +with eu-stack and resolves symbols. This is companion tool for L, called +when option --dumper=pteu is specified. + +eu-stack is a tool from elfutils package that prints a stack for each thread in +a process or core file. eu-stack is faster than gdb and smaller overhead on the +diagnosed process. + +=head1 OUTPUT + +Stack for each thread, formatted similarly to `gdb thread apply all bt` output. + +=head1 ATTENTION + +Using might expose passwords. When debug is enabled, all command line +parameters are shown in the output. + +=head1 SYSTEM REQUIREMENTS + +You need eu-stack from the elfutils package. + +=head1 BUGS + +For a list of known bugs, see L. + +Please report bugs at L. +Include the following information in your bug report: + +=over + +=item * Complete command-line used to run the tool + +=item * Tool L<"--version"> + +=item * MySQL version of all servers involved + +=item * Output from the tool including STDERR + +=item * Input files (log/dump/config files, etc.) + +=back + +If possible, include debugging output by running the tool with C; +see L<"ENVIRONMENT">. + +=head1 DOWNLOADING + +Visit L to download the +latest release of Percona Toolkit. Or, get the latest release from the +command line: + + wget percona.com/get/percona-toolkit.tar.gz + + wget percona.com/get/percona-toolkit.rpm + + wget percona.com/get/percona-toolkit.deb + +You can also get individual tools from the latest release: + + wget percona.com/get/TOOL + +Replace C with the name of any tool. + +=head1 AUTHORS + +Alexey Stroganov + +=head1 ACKNOWLEDGMENTS + +Part of code for symbol resolving derived from resolve-stack-traces.pl script +(https://github.com/knielsen/knielsen-pmp) + +=head1 ABOUT PERCONA TOOLKIT + +This tool is part of Percona Toolkit, a collection of advanced command-line +tools for MySQL developed by Percona. Percona Toolkit was forked from two +projects in June, 2011: Maatkit and Aspersa. Those projects were created by +Baron Schwartz and primarily developed by him and Daniel Nichter. Visit +L to learn about other free, open-source +software from Percona. + +=head1 COPYRIGHT, LICENSE, AND WARRANTY + +This program is copyright 2017-2024 Percona LLC and/or its affiliates. + +THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, version 2; OR the Perl Artistic License. On UNIX and similar +systems, you can issue `man perlgpl' or `man perlartistic' to read these +licenses. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA. + +=head1 VERSION + +pt-eustack-resolver 3.5.7 + +=cut diff --git a/bin/pt-pmp b/bin/pt-pmp index 7884ed9e..12e67965 100755 --- a/bin/pt-pmp +++ b/bin/pt-pmp @@ -567,91 +567,124 @@ set +u # to pass the maxlen argument into this function to make maxlen testable. aggregate_stacktrace() { local maxlen=$1; + local tids=$2; + + if [ "$tids" == "*" ]; then tids="" ; fi + shift - awk " + shift + awk -v tids=$tids -v dumper=$OPT_DUMPER " BEGIN { s = \"\"; + n_tid=split(tids,tids_array,\",\"); } - /^Thread/ { - if ( s != \"\" ) { + /^Thread|^TID/ { + if ( s != \"\" && tid_found ~ 1 ) { print s; } s = \"\"; c = 0; + + tid_found=1; + if (n_tid>0) + { + tid_found=0; + tid=\$NF; + gsub(/)*:/,\"\",tid); + for (i=1;i<=n_tid;i++) { if ( tid ~ tids_array[i]){tid_found=1;} } + } } - /^\#/ { - if ( \$2 ~ /0x/ ) { - if ( \$4 ~/void|const/ ) { - targ = \$5; + /^#/ { + if ( ${maxlen:-0} == 0 || c < ${maxlen:-0} ) { + if ( \$2 ~ /0x/ ) { + if (dumper ~ /^eu/) { + targ = \$3; + tfile= \$NF; + } + else { + if ( \$4 ~/void|const/ ) { + targ = \$5; + } + else { + targ = \$4; + tfile= \$NF; + } + } + + if ( targ ~ /[<\\(]/ ) { + if (dumper ~ /eu/) { + while ( targ ~ /\\(/ ) { + if ( 0 == gsub(/\\(.*\$/, \"\", targ) ) { + break; + } + } + } + if ( dumper ~ /gdb/ || dumper ~ /qs/ ) { + targ = substr(\$0, index(\$0, \" in \") + 4); + if ( targ ~ / from / ) { + targ = substr(targ, 1, index(targ, \" from \") - 1); + } + if ( targ ~ / at / ) { + targ = substr(targ, 1, index(targ, \" at \") - 1); + } + } + # Shorten C++ templates, e.g. in t/samples/stacktrace-004.txt + while ( targ ~ />::/ ) { + if ( 0 == gsub(/<[^<>]*>/, \"\", targ) ) { + break; + } + } + # Further shorten argument lists. + while ( targ ~ /\\(/ ) { + if ( 0 == gsub(/\\([^()]*\\)/, \"\", targ) ) { + break; + } + } + # Remove void and const decorators. + gsub(/ ?(void|const) ?/, \"\", targ); + gsub(/ /, \"\", targ); + } + else if ( targ ~ /\\?\\?/ && \$2 ~ /[1-9]/ ) { + # Substitute ?? by the name of the library. + targ = \$NF; + while ( targ ~ /\\// ) { + targ = substr(targ, index(targ, \"/\") + 1); + } + targ = substr(targ, 1, index(targ, \".\") - 1); + targ = targ \"::??\"; + } } else { - targ = \$4; + targ = \$2; tfile= \$NF; } - if ( targ ~ /[<\\(]/ ) { - targ = substr(\$0, index(\$0, \" in \") + 4); - if ( targ ~ / from / ) { - targ = substr(targ, 1, index(targ, \" from \") - 1); - } - if ( targ ~ / at / ) { - targ = substr(targ, 1, index(targ, \" at \") - 1); - } - # Shorten C++ templates, e.g. in t/samples/stacktrace-004.txt - while ( targ ~ />::/ ) { - if ( 0 == gsub(/<[^<>]*>/, \"\", targ) ) { - break; - } - } - # Further shorten argument lists. - while ( targ ~ /\\(/ ) { - if ( 0 == gsub(/\\([^()]*\\)/, \"\", targ) ) { - break; - } - } - # Remove void and const decorators. - gsub(/ ?(void|const) ?/, \"\", targ); - gsub(/ /, \"\", targ); - } - else if ( targ ~ /\\?\\?/ && \$2 ~ /[1-9]/ ) { - # Substitute ?? by the name of the library. - targ = \$NF; - while ( targ ~ /\\// ) { - targ = substr(targ, index(targ, \"/\") + 1); - } - targ = substr(targ, 1, index(targ, \".\") - 1); - targ = targ \"::??\"; - } - } - else { - targ = \$2; - tfile= \$NF; - } - # get rid of long symbol names such as 'pthread_cond_wait@@GLIBC_2.3.2' - if ( targ ~ /@@/ ) { - fname = substr(targ, 1, index(targ, \"@@\") - 1); - } - else { - fname = targ; - if ( tfile ~ /^\// ) { - last=split(tfile,filen,/\//); - fname = targ \"(\" filen[last] \")\"; + # get rid of long symbol names such as 'pthread_cond_wait@@GLIBC_2.3.2' + if ( targ ~ /@@/ ) { + fname = substr(targ, 1, index(targ, \"@@\") - 1); } else { - fname = targ + fname = targ; + if ( tfile ~ /^\// ) { + last=split(tfile,filen,/\//); + fname = targ \"(\" filen[last] \")\"; + } + else { + fname = targ + } } - } - if ( ${maxlen:-0} == 0 || c < ${maxlen:-0} ) { if (s != \"\" ) { s = s \",\" fname; } else { s = fname; } + c++; } - c++; } END { - print s + if (tid_found ~ 1 ) { + print s + } } " "$@" | sort | uniq -c | sort -r -n -k 1,1 } @@ -673,20 +706,52 @@ main() { fi date for x in $(_seq $OPT_ITERATIONS); do - gdb -ex "set pagination 0" \ - -ex "thread apply all bt" \ - -batch \ - -p $OPT_PID \ - >> "$output_file" - date +'TS %N.%s %F %T' >> "$output_file" - sleep $OPT_INTERVAL + + if [ $OPT_DUMPER == "gdb" ]; then + if [ `_which gdb` ]; then + + readnever="" + if gdb -nx --quiet --batch --readnever > /dev/null 2>&1; then + readnever="--readnever" + fi + + gdb $readnever -ex "set pagination 0" \ + -ex "thread apply all bt" \ + -batch \ + -p $OPT_PID \ + >> "$output_file" + else + die "Can't find gdb binary. Exiting" + fi + elif [ $OPT_DUMPER == "eu" ]; then + if [ `_which eu-stack` ]; then + eu-stack -s -m -p $OPT_PID | sed -e '$!N;s/\n //g;P;D' | sed -e 's/\(0x[[:xdigit:]]*\) -/\1 ??() -/' >> "$output_file" + else + die "Can't find eu-stack binary from elfutils. Exiting" + fi + elif [ $OPT_DUMPER == "pteu" ]; then + if [ `_which eu-stack` ] && [ `_which pt-eustack-resolver` ]; then + pt-eustack-resolver $OPT_PID >> "$output_file" + else + die "Can't find eu-stack binary from elfutils or pt-eustack-resolver. Exiting" + fi + elif [ $OPT_DUMPER == "qs" ]; then + if [ `_which quickstack` ]; then + quickstack -l -f -p $OPT_PID >> "$output_file" + else + die "Can't find quickstack binary from https://github.com/yoshinorim/quickstack. Exiting" + fi + fi + + date +'TS %N.%s %F %T' >> "$output_file" + sleep $OPT_INTERVAL done fi if [ -z "$ARGV" ]; then - aggregate_stacktrace "$OPT_LINES" "$output_file" + aggregate_stacktrace "$OPT_LINES" "${OPT_TIDS}" "$output_file" else - aggregate_stacktrace "$OPT_LINES" $ARGV + aggregate_stacktrace "$OPT_LINES" "${OPT_TIDS}" $ARGV fi } @@ -782,6 +847,18 @@ short form: -b; type: string; default: mysqld Which binary to trace. +=item --dumper + +short form: -d; type: string; default: gdb + +Which dumper use to get stack traces(gdb: gdb, eu: eu-stack, pteu: pt-eustack-resolver, qs: quickstack). + +=item --tids + +short form: -t; type: string; default: * + +Extract traces only for specific tids. + =item --help Show help and exit.