Port improved pt-pmp

Initial port of https://github.com/Percona-Lab/pt-pmp:
- Added bin/pt-eustack-resolver
- Ported changes to bin/pt-pmp

bin/pt-eustack-resolver

@@ -0,0 +1,249 @@
+#!/usr/bin/env perl
+
+# This program is part of Percona Toolkit: http://www.percona.com/software/
+# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal
+# notices and disclaimers.
+
+use strict;
+use Data::Dumper;
+
+my $mmap;
+
+sub load_mapping {
+  my ($pid)= @_;
+
+  if ($pid =~ /^[0-9]+$/) {
+    open F, '<', "/proc/$pid/maps"
+        or die "Failed to open /proc/$pid/maps: $!\n";
+  } else {
+    open F, '<', $pid
+        or die "Failed to open saved map file '$pid': $!\n";
+  }
+
+  my $arr= [];
+  while (<F>) {
+    next unless m/^([a-f0-9]+)-([a-f0-9]+) ..x. ([a-f0-9]+) [a-f0-9:]+ [0-9]+ +(.*)/;
+    push @$arr, { S => hex($1), E => hex($2), B => hex($3), F => $4 };
+  }
+  close F;
+  sort { $a->{S} <=> $b->{S} } @$arr;
+  $mmap= $arr;
+}
+
+my $syms= { };
+
+sub get_image {
+  my ($addr)= @_;
+  my $e;
+  for $e (@$mmap) {
+    next if $e->{E} <= $addr;
+    last if $e->{S} > $addr;
+    # Found, look up.  
+      return $e->{F};
+  }
+  return "";
+}
+
+die "Usage: $0 <pid>" unless @ARGV == 1;
+
+my $pid= $ARGV[0];
+load_mapping($pid);
+
+#for (@$mmap) {
+#  printf "0x%x - 0x%x (0x%x): %s\n", $_->{S}, $_->{E}, $_->{B}, $_->{F};
+#}
+
+open (STACK_TRACE, "eu-stack -q -p $pid 2>/dev/null|") or die "open(): $!";
+my @lines= <STACK_TRACE>;
+close(STACK_TRACE);
+
+my $frame_no= 0;
+my %addr=();
+my %sf=();
+my $lwp;
+
+for my $line (@lines) {
+   if ($line =~ /^TID ([0-9]+):/)
+   {
+     $frame_no= 0;
+     $lwp=$1;
+   }
+   elsif ($line =~ /^#[0-9]+?\s*0x([a-f0-9]+)/) 
+   {
+     push @{$sf{$lwp}},$1;
+     $addr{$1}=[get_image(hex($1)),""];
+   } else {
+     #print $line;
+   }
+}
+
+my %inverse;
+push @{ $inverse{ $addr{$_}->[0] } }, $_ for keys %addr;
+
+foreach my $bin (keys %inverse)
+{
+   my $addrs=join(" ",@{$inverse{$bin}});
+   my @resolved=();
+   
+   @resolved=(`eu-addr2line --pretty-print -s -C -f -p $pid $addrs`);          
+   
+   my $idx=0;
+   foreach $a (@{$inverse{$bin}})
+   {
+      $addr{$a}->[1]=$resolved[$idx];
+      $addr{$a}->[1]=~ s/\n//;
+      $addr{$a}->[1]=~ s/at \?\?:0/from $addr{$a}->[0]/;
+      $idx++;
+   }
+}
+
+foreach $lwp (sort {$a<=>$b} keys %sf)
+{
+  my $idx=0;
+  print "Thread $lwp (LWP $lwp):\n";
+  foreach $frame_no (@{$sf{$lwp}})
+  {
+    print join(" ","#".$idx, "0x".$frame_no,"in", $addr{$frame_no}->[1]),"\n";
+    $idx++;
+  }
+  print "\n";
+}
+    
+    
+# ############################################################################
+# Documentation
+# ############################################################################
+=pod
+
+=head1 NAME
+
+pt-eustack-resover - Get stack traces for a selected program with eu-stack and resolve symbols 
+
+=head1 SYNOPSIS
+
+Usage: pt-eustack-resolver <pid>
+
+=head1 RISKS
+
+Percona Toolkit is mature, proven in the real world, and well tested,
+but all database tools can pose a risk to the system and the database
+server.  Before using this tool, please:
+
+=over
+
+=item * Read the tool's documentation
+
+=item * Review the tool's known L<"BUGS">
+
+=item * Test the tool on a non-production server
+
+=item * Backup your production server and verify the backups
+
+=back
+
+=head1 DESCRIPTION
+
+pt-eustack-resover is the tool that gets stack traces for a selected program 
+with eu-stack and resolves symbols. This is companion tool for L<pt-pmp>, called
+when option --dumper=pteu is specified.
+
+eu-stack is a tool from elfutils package that prints a stack for each thread in 
+a process or core file. eu-stack is faster than gdb and smaller overhead on the
+diagnosed process.
+
+=head1 OUTPUT
+
+Stack for each thread, formatted similarly to `gdb thread apply all bt` output.
+
+=head1 ATTENTION
+
+Using <PTDEBUG> might expose passwords. When debug is enabled, all command line
+parameters are shown in the output.
+
+=head1 SYSTEM REQUIREMENTS
+
+You need eu-stack from the elfutils package.
+
+=head1 BUGS
+
+For a list of known bugs, see L<https://jira.percona.com/projects/PT/issues>.
+
+Please report bugs at L<https://jira.percona.com/projects/PT>.
+Include the following information in your bug report:
+
+=over
+
+=item * Complete command-line used to run the tool
+
+=item * Tool L<"--version">
+
+=item * MySQL version of all servers involved
+
+=item * Output from the tool including STDERR
+
+=item * Input files (log/dump/config files, etc.)
+
+=back
+
+If possible, include debugging output by running the tool with C<PTDEBUG>;
+see L<"ENVIRONMENT">.
+
+=head1 DOWNLOADING
+
+Visit L<http://www.percona.com/software/percona-toolkit/> to download the
+latest release of Percona Toolkit.  Or, get the latest release from the
+command line:
+
+   wget percona.com/get/percona-toolkit.tar.gz
+
+   wget percona.com/get/percona-toolkit.rpm
+
+   wget percona.com/get/percona-toolkit.deb
+
+You can also get individual tools from the latest release:
+
+   wget percona.com/get/TOOL
+
+Replace C<TOOL> with the name of any tool.
+
+=head1 AUTHORS
+
+Alexey Stroganov
+
+=head1 ACKNOWLEDGMENTS
+
+Part of code for symbol resolving derived from resolve-stack-traces.pl script
+(https://github.com/knielsen/knielsen-pmp)
+
+=head1 ABOUT PERCONA TOOLKIT
+      
+This tool is part of Percona Toolkit, a collection of advanced command-line
+tools for MySQL developed by Percona.  Percona Toolkit was forked from two
+projects in June, 2011: Maatkit and Aspersa.  Those projects were created by
+Baron Schwartz and primarily developed by him and Daniel Nichter.  Visit
+L<http://www.percona.com/software/> to learn about other free, open-source
+software from Percona.
+
+=head1 COPYRIGHT, LICENSE, AND WARRANTY
+
+This program is copyright 2017-2024 Percona LLC and/or its affiliates.
+
+THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
+systems, you can issue `man perlgpl' or `man perlartistic' to read these
+licenses.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA  02111-1307  USA.
+   
+=head1 VERSION
+
+pt-eustack-resolver 3.5.7
+
+=cut

bin/pt-pmp

@@ -567,91 +567,124 @@ set +u
 # to pass the maxlen argument into this function to make maxlen testable.
 aggregate_stacktrace() {
    local maxlen=$1;
+   local tids=$2;
+
+   if [ "$tids" == "*" ]; then tids="" ; fi
+
    shift
-   awk "
+   shift
+   awk -v tids=$tids -v dumper=$OPT_DUMPER "
       BEGIN {
          s = \"\";
+         n_tid=split(tids,tids_array,\",\");
       }
-      /^Thread/ {
+      /^Thread|^TID/ {
-         if ( s != \"\" ) {
+         if ( s != \"\" && tid_found ~ 1 ) {
             print s;
          }
          s = \"\";
          c = 0;
+         
+         tid_found=1;
+         if (n_tid>0)
+         {
+           tid_found=0;
+           tid=\$NF;
+           gsub(/)*:/,\"\",tid);
+           for (i=1;i<=n_tid;i++) { if ( tid ~ tids_array[i]){tid_found=1;} }
+         }
       }
-      /^\#/ {
+      /^#/ {
-         if ( \$2 ~ /0x/ ) {
+         if ( ${maxlen:-0} == 0 || c < ${maxlen:-0} ) {
-            if ( \$4 ~/void|const/ ) {
+            if ( \$2 ~ /0x/ ) {
-               targ = \$5;
+               if (dumper ~ /^eu/) {
+                  targ = \$3;
+                  tfile= \$NF;
+               }
+               else {
+                  if ( \$4 ~/void|const/ ) {
+                     targ = \$5;
+                  }
+                  else {
+                     targ = \$4;
+                     tfile= \$NF;
+                  }
+               }
+            
+               if ( targ ~ /[<\\(]/ ) {
+                  if (dumper ~ /eu/) {
+                     while ( targ ~ /\\(/ ) {
+                        if ( 0 == gsub(/\\(.*\$/, \"\", targ) ) {
+                           break;
+                        }
+                     }
+                  }
+                  if ( dumper ~ /gdb/ || dumper ~ /qs/ ) {
+                     targ = substr(\$0, index(\$0, \" in \") + 4);
+                     if ( targ ~ / from / ) {
+                        targ = substr(targ, 1, index(targ, \" from \") - 1);
+                     }
+                     if ( targ ~ / at / ) {
+                        targ = substr(targ, 1, index(targ, \" at \") - 1);
+                     }
+                  }
+                  # Shorten C++ templates, e.g. in t/samples/stacktrace-004.txt
+                  while ( targ ~ />::/ ) {
+                     if ( 0 == gsub(/<[^<>]*>/, \"\", targ) ) {
+                        break;
+                     }
+                  }
+                  # Further shorten argument lists.
+                  while ( targ ~ /\\(/ ) {
+                     if ( 0 == gsub(/\\([^()]*\\)/, \"\", targ) ) {
+                        break;
+                     }
+                  }
+                  # Remove void and const decorators.
+                  gsub(/ ?(void|const) ?/, \"\", targ);
+                  gsub(/ /, \"\", targ);
+               }
+               else if ( targ ~ /\\?\\?/ && \$2 ~ /[1-9]/ ) {
+                  # Substitute ?? by the name of the library.
+                  targ = \$NF;
+                  while ( targ ~ /\\// ) {
+                     targ = substr(targ, index(targ, \"/\") + 1);
+                  }
+                  targ = substr(targ, 1, index(targ, \".\") - 1);
+                  targ = targ \"::??\";
+               }
             }
             else {
-               targ = \$4;
+               targ = \$2;
                tfile= \$NF;
             }
-            if ( targ ~ /[<\\(]/ ) {
+            # get rid of long symbol names such as 'pthread_cond_wait@@GLIBC_2.3.2'
-               targ = substr(\$0, index(\$0, \" in \") + 4);
+            if ( targ ~ /@@/ ) {
-               if ( targ ~ / from / ) {
+               fname = substr(targ, 1, index(targ, \"@@\") - 1);
-                  targ = substr(targ, 1, index(targ, \" from \") - 1);
-               }
-               if ( targ ~ / at / ) {
-                  targ = substr(targ, 1, index(targ, \" at \") - 1);
-               }
-               # Shorten C++ templates, e.g. in t/samples/stacktrace-004.txt
-               while ( targ ~ />::/ ) {
-                  if ( 0 == gsub(/<[^<>]*>/, \"\", targ) ) {
-                     break;
-                  }
-               }
-               # Further shorten argument lists.
-               while ( targ ~ /\\(/ ) {
-                  if ( 0 == gsub(/\\([^()]*\\)/, \"\", targ) ) {
-                     break;
-                  }
-               }
-               # Remove void and const decorators.
-               gsub(/ ?(void|const) ?/, \"\", targ);
-               gsub(/ /, \"\", targ);
-            }
-            else if ( targ ~ /\\?\\?/ && \$2 ~ /[1-9]/ ) {
-               # Substitute ?? by the name of the library.
-               targ = \$NF;
-               while ( targ ~ /\\// ) {
-                  targ = substr(targ, index(targ, \"/\") + 1);
-               }
-               targ = substr(targ, 1, index(targ, \".\") - 1);
-               targ = targ \"::??\";
-            }
-         }
-         else {
-            targ = \$2;
-            tfile= \$NF;
-         }
-         # get rid of long symbol names such as 'pthread_cond_wait@@GLIBC_2.3.2'
-         if ( targ ~ /@@/ ) {
-            fname = substr(targ, 1, index(targ, \"@@\") - 1);
-         }
-         else {
-            fname = targ;
-            if ( tfile ~ /^\// ) {
-               last=split(tfile,filen,/\//);
-               fname = targ \"(\" filen[last] \")\";
             }
             else {
-               fname = targ
+               fname = targ;
+               if ( tfile ~ /^\// ) {
+                  last=split(tfile,filen,/\//);
+                  fname = targ \"(\" filen[last] \")\";
+               }
+               else {
+                  fname = targ
+               }
             }
-         }
-         if ( ${maxlen:-0} == 0 || c < ${maxlen:-0} ) {
             if (s != \"\" ) {
                s = s \",\" fname;
             }
             else {
                s = fname;
             }
+            c++;
          }
-         c++;
       }
       END {
-         print s
+         if (tid_found ~ 1 ) {
+            print s
+         }
       }
    " "$@" | sort | uniq -c | sort -r -n -k 1,1
 }
@@ -673,20 +706,52 @@ main() {
       fi
       date
       for x in $(_seq $OPT_ITERATIONS); do
-         gdb -ex "set pagination 0"    \
+           
-             -ex "thread apply all bt" \
+         if [ $OPT_DUMPER == "gdb" ]; then
-             -batch                    \
+            if [ `_which gdb` ]; then 
-             -p $OPT_PID               \
+
-             >> "$output_file"
+               readnever=""
-         date +'TS %N.%s %F %T' >> "$output_file"
+               if gdb -nx --quiet --batch --readnever > /dev/null 2>&1; then
-         sleep $OPT_INTERVAL
+                  readnever="--readnever"
+               fi
+
+               gdb $readnever -ex "set pagination 0"    \
+                   -ex "thread apply all bt" \
+                   -batch                    \
+                   -p $OPT_PID               \
+                   >> "$output_file"
+           else
+              die "Can't find gdb binary. Exiting"
+           fi
+        elif [ $OPT_DUMPER == "eu" ]; then
+           if [ `_which eu-stack` ]; then
+              eu-stack -s -m -p $OPT_PID | sed -e '$!N;s/\n //g;P;D' | sed -e 's/\(0x[[:xdigit:]]*\) -/\1 ??() -/' >> "$output_file"
+           else
+              die "Can't find eu-stack binary from elfutils. Exiting"
+           fi
+        elif [ $OPT_DUMPER == "pteu" ]; then
+           if [ `_which eu-stack` ] && [ `_which pt-eustack-resolver` ]; then
+              pt-eustack-resolver $OPT_PID  >> "$output_file"
+           else
+              die "Can't find eu-stack binary from elfutils or pt-eustack-resolver. Exiting"
+           fi
+        elif [ $OPT_DUMPER == "qs" ]; then
+           if [ `_which quickstack` ]; then
+              quickstack -l -f -p $OPT_PID >> "$output_file"
+           else
+              die "Can't find quickstack binary from https://github.com/yoshinorim/quickstack. Exiting"
+           fi
+        fi
+             
+        date +'TS %N.%s %F %T' >> "$output_file"
+        sleep $OPT_INTERVAL
       done
    fi
 
    if [ -z "$ARGV" ]; then
-      aggregate_stacktrace "$OPT_LINES" "$output_file"
+      aggregate_stacktrace "$OPT_LINES" "${OPT_TIDS}" "$output_file"
    else
-      aggregate_stacktrace "$OPT_LINES" $ARGV
+      aggregate_stacktrace "$OPT_LINES" "${OPT_TIDS}" $ARGV
    fi
 }
 
@@ -782,6 +847,18 @@ short form: -b; type: string; default: mysqld
 
 Which binary to trace.
 
+=item --dumper
+
+short form: -d; type: string; default: gdb
+
+Which dumper use to get stack traces(gdb: gdb, eu: eu-stack, pteu: pt-eustack-resolver, qs: quickstack).
+
+=item --tids
+
+short form: -t; type: string; default: *
+
+Extract traces only for specific tids.
+
 =item --help
 
 Show help and exit.