From b53a9a76fe8ffe210e878bd05b4610eaf6c958c9 Mon Sep 17 00:00:00 2001 From: Fernando Ipar Date: Tue, 24 Jan 2017 16:04:37 -0300 Subject: [PATCH 1/5] added support for operf if present, and if CMD_OPCONTROL is not set --- bin/pt-stalk | 19 +++++++++++++++---- lib/bash/collect.sh | 20 ++++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index 288e2163..22a6f99f 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -779,6 +779,7 @@ CMD_MPSTAT="${CMD_MPSTAT:-"$(_which mpstat)"}" CMD_MYSQL="${CMD_MYSQL:-"$(_which mysql)"}" CMD_MYSQLADMIN="${CMD_MYSQLADMIN:-"$(_which mysqladmin)"}" CMD_OPCONTROL="${CMD_OPCONTROL:-"$(_which opcontrol)"}" +[ -z "$CMD_OPCONTROL" ] && CMD_OPCONTROL=$(_which operf) CMD_OPREPORT="${CMD_OPREPORT:-"$(_which opreport)"}" CMD_PMAP="${CMD_PMAP:-"$(_which pmap)"}" CMD_STRACE="${CMD_STRACE:-"$(_which strace)"}" @@ -853,9 +854,15 @@ collect() { local have_oprofile="" if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then - if $CMD_OPCONTROL --init; then - $CMD_OPCONTROL --start --no-vmlinux - have_oprofile="yes" + if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then + # use legacy opcontrol + if $CMD_OPCONTROL --init; then + $CMD_OPCONTROL --start --no-vmlinux + have_oprofile="yes" + fi + else + $CMD_OPCONTROL -p `pidof mysqld` & + OPERF_PID=$! fi elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then $CMD_STRACE -T -s 0 -f -p $mysqld_pid -o "$d/$p-strace" & @@ -951,6 +958,7 @@ collect() { log "Loop end: $(date +'TS %s.%N %F %T')" if [ "$have_oprofile" ]; then + if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then $CMD_OPCONTROL --stop $CMD_OPCONTROL --dump @@ -963,6 +971,9 @@ collect() { $CMD_OPCONTROL --save=pt_collect_$p + else + kill -SIGINT $OPERF_PID + fi local mysqld_path=$(_which mysqld); if [ "$mysqld_path" -a -f "$mysqld_path" ]; then $CMD_OPREPORT \ @@ -985,7 +996,7 @@ collect() { kill -s 15 $strace_pid [ "$mysqld_pid" ] && kill -s 18 $mysqld_pid fi - + innodb_status 2 tokudb_status 2 $CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" & diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 85b9c141..2bc1409a 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -34,6 +34,7 @@ CMD_MPSTAT="${CMD_MPSTAT:-"$(_which mpstat)"}" CMD_MYSQL="${CMD_MYSQL:-"$(_which mysql)"}" CMD_MYSQLADMIN="${CMD_MYSQLADMIN:-"$(_which mysqladmin)"}" CMD_OPCONTROL="${CMD_OPCONTROL:-"$(_which opcontrol)"}" +[ -z "$CMD_OPCONTROL" ] && CMD_OPCONTROL=$(_which operf) CMD_OPREPORT="${CMD_OPREPORT:-"$(_which opreport)"}" CMD_PMAP="${CMD_PMAP:-"$(_which pmap)"}" CMD_STRACE="${CMD_STRACE:-"$(_which strace)"}" @@ -128,9 +129,16 @@ collect() { # The --init should be a no-op if it has already been init-ed. local have_oprofile="" if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then - if $CMD_OPCONTROL --init; then - $CMD_OPCONTROL --start --no-vmlinux - have_oprofile="yes" + if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then + # use legacy opcontrol + if $CMD_OPCONTROL --init; then + $CMD_OPCONTROL --start --no-vmlinux + have_oprofile="yes" + fi + else + # use operf, may fail under VirtualBox or old processor models (see http://oprofile.sourceforge.net/doc/perf_events.html) + $CMD_OPCONTROL -p `pidof mysqld` & + OPERF_PID=$! fi elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then # Don't run oprofile and strace at the same time. @@ -245,6 +253,7 @@ collect() { log "Loop end: $(date +'TS %s.%N %F %T')" if [ "$have_oprofile" ]; then + if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then $CMD_OPCONTROL --stop $CMD_OPCONTROL --dump @@ -257,6 +266,9 @@ collect() { $CMD_OPCONTROL --save=pt_collect_$p + else + kill -SIGINT $OPERF_PID + fi # Attempt to generate a report; if this fails, then just tell the user # how to generate the report. local mysqld_path=$(_which mysqld); @@ -282,7 +294,7 @@ collect() { # Sometimes strace leaves threads/processes in T status. [ "$mysqld_pid" ] && kill -s 18 $mysqld_pid fi - + innodb_status 2 tokudb_status 2 $CMD_MYSQL $EXT_ARGV -e "$mutex" >> "$d/$p-mutex-status2" & From 102ac33210f7af099a828a8e1191891ca1a5a34e Mon Sep 17 00:00:00 2001 From: Fernando Ipar Date: Tue, 24 Jan 2017 19:34:02 -0300 Subject: [PATCH 2/5] doing system wide operf --- bin/pt-stalk | 2 +- lib/bash/collect.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index 22a6f99f..44e6b51f 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -861,7 +861,7 @@ collect() { have_oprofile="yes" fi else - $CMD_OPCONTROL -p `pidof mysqld` & + $CMD_OPCONTROL -s & OPERF_PID=$! fi elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index 2bc1409a..f274e480 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -137,7 +137,7 @@ collect() { fi else # use operf, may fail under VirtualBox or old processor models (see http://oprofile.sourceforge.net/doc/perf_events.html) - $CMD_OPCONTROL -p `pidof mysqld` & + $CMD_OPCONTROL -s & OPERF_PID=$! fi elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then From 5ca0e8a898498110969de3514b52319624f8cbad Mon Sep 17 00:00:00 2001 From: Sveta Smirnova Date: Fri, 26 Jan 2024 01:01:23 +0300 Subject: [PATCH 3/5] PR-160 - added support for operf if present, and if CMD_OPCONTROL is not set - Made PR actually working --- bin/pt-stalk | 52 +++++++++++++++++++++++++------------------ lib/bash/collect.sh | 54 +++++++++++++++++++++++++++------------------ 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index c0e795fe..d87af153 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -932,16 +932,20 @@ collect_mysql_data_one() { fi if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then - if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then - # use legacy opcontrol - if $CMD_OPCONTROL --init; then - $CMD_OPCONTROL --start --no-vmlinux - have_oprofile="yes" - fi + if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then + # use legacy opcontrol + if $CMD_OPCONTROL --init; then + $CMD_OPCONTROL --start --no-vmlinux + fi else - $CMD_OPCONTROL -s & - OPERF_PID=$! + local tmpfile="$PT_TMPDIR/oprofile" + mkdir "$d/pt_collect_$p" + $CMD_OPCONTROL -p "$mysqld_pid" -d "$d/pt_collect_$p" > "$tmpfile" & + sleep 1 + OPERF_PID=$(grep -Eo "kill -SIGINT [[:digit:]]+" "$tmpfile" | grep -Eo "[[:digit:]]+") + rm "$tmpfile" fi + have_oprofile="yes" elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then $CMD_STRACE -T -s 0 -f -p $mysqld_pid -o "$d/$p-strace" & local strace_pid=$! @@ -1062,29 +1066,35 @@ collect_system_data_loop() { collect_mysql_data_two() { if [ "$have_oprofile" ]; then - if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then - $CMD_OPCONTROL --stop - $CMD_OPCONTROL --dump + local session="--session-dir=$d/pt_collect_$p" - local oprofiled_pid=$(_pidof oprofiled | awk '{print $1; exit;}') - if [ "$oprofiled_pid" ]; then - kill $oprofiled_pid + if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then + $CMD_OPCONTROL --stop + $CMD_OPCONTROL --dump + + local oprofiled_pid=$(_pidof oprofiled | awk '{print $1; exit;}') + if [ "$oprofiled_pid" ]; then + kill $oprofiled_pid + else + warn "Cannot kill oprofiled because its PID cannot be determined" + fi + + session="session:pt_collect_$p" + + $CMD_OPCONTROL --save="pt_collect_$p" else - warn "Cannot kill oprofiled because its PID cannot be determined" + log "$OPERF_PID" + kill -SIGINT "$OPERF_PID" fi - $CMD_OPCONTROL --save=pt_collect_$p - else - kill -SIGINT $OPERF_PID - fi - local mysqld_path=$(_which mysqld); + local mysqld_path=$(readlink -f "/proc/$mysqld_pid/exe"); if [ "$mysqld_path" -a -f "$mysqld_path" ]; then $CMD_OPREPORT \ --demangle=smart \ --symbols \ --merge tgid \ - session:pt_collect_$p \ + "$session" \ "$mysqld_path" \ > "$d/$p-opreport" else diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index e2e211d2..f28de728 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -200,17 +200,21 @@ collect_mysql_data_one() { # Next, start oprofile gathering data during the whole rest of this process. # The --init should be a no-op if it has already been init-ed. if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then - if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then - # use legacy opcontrol - if $CMD_OPCONTROL --init; then - $CMD_OPCONTROL --start --no-vmlinux - have_oprofile="yes" - fi + if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then + # use legacy opcontrol + if $CMD_OPCONTROL --init; then + $CMD_OPCONTROL --start --no-vmlinux + fi else - # use operf, may fail under VirtualBox or old processor models (see http://oprofile.sourceforge.net/doc/perf_events.html) - $CMD_OPCONTROL -s & - OPERF_PID=$! + local tmpfile="$PT_TMPDIR/oprofile" + mkdir "$d/pt_collect_$p" + # use operf, may fail under VirtualBox or old processor models (see http://oprofile.sourceforge.net/doc/perf_events.html) + $CMD_OPCONTROL -p "$mysqld_pid" -d "$d/pt_collect_$p" > "$tmpfile" & + sleep 1 + OPERF_PID=$(grep -Eo "kill -SIGINT [[:digit:]]+" "$tmpfile" | grep -Eo "[[:digit:]]+") + rm "$tmpfile" fi + have_oprofile="yes" elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then # Don't run oprofile and strace at the same time. $CMD_STRACE -T -s 0 -f -p $mysqld_pid -o "$d/$p-strace" & @@ -348,31 +352,37 @@ collect_system_data_loop() { collect_mysql_data_two() { if [ "$have_oprofile" ]; then - if [ $(echo $CMD_OPCONTROL|grep -c opcontrol) -gt 0 ]; then - $CMD_OPCONTROL --stop - $CMD_OPCONTROL --dump + local session="--session-dir=$d/pt_collect_$p" - local oprofiled_pid=$(_pidof oprofiled | awk '{print $1; exit;}') - if [ "$oprofiled_pid" ]; then - kill $oprofiled_pid + if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then + $CMD_OPCONTROL --stop + $CMD_OPCONTROL --dump + + local oprofiled_pid=$(_pidof oprofiled | awk '{print $1; exit;}') + if [ "$oprofiled_pid" ]; then + kill $oprofiled_pid + else + warn "Cannot kill oprofiled because its PID cannot be determined" + fi + + session="session:pt_collect_$p" + + $CMD_OPCONTROL --save="pt_collect_$p" else - warn "Cannot kill oprofiled because its PID cannot be determined" + log "$OPERF_PID" + kill -SIGINT "$OPERF_PID" fi - $CMD_OPCONTROL --save=pt_collect_$p - else - kill -SIGINT $OPERF_PID - fi # Attempt to generate a report; if this fails, then just tell the user # how to generate the report. - local mysqld_path=$(_which mysqld); + local mysqld_path=$(readlink -f "/proc/$mysqld_pid/exe"); if [ "$mysqld_path" -a -f "$mysqld_path" ]; then $CMD_OPREPORT \ --demangle=smart \ --symbols \ --merge tgid \ - session:pt_collect_$p \ + "$session" \ "$mysqld_path" \ > "$d/$p-opreport" else From 3fa1d8dfed7ac690edf2d21299424ca7ad88c53a Mon Sep 17 00:00:00 2001 From: Sveta Smirnova Date: Fri, 26 Jan 2024 16:39:26 +0300 Subject: [PATCH 4/5] PR-160 - added support for operf if present, and if CMD_OPCONTROL is not set - Added back support for CMD_OPCONTROL --- bin/pt-stalk | 10 ++++++---- lib/bash/collect.sh | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/bin/pt-stalk b/bin/pt-stalk index d87af153..c2085e02 100755 --- a/bin/pt-stalk +++ b/bin/pt-stalk @@ -932,12 +932,14 @@ collect_mysql_data_one() { fi if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then - if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then - # use legacy opcontrol + if [ $(echo $CMD_OPCONTROL | grep -cv operf) -gt 0 ]; then + # use legacy or custom opcontrol if $CMD_OPCONTROL --init; then $CMD_OPCONTROL --start --no-vmlinux + have_oprofile="yes" fi else + have_oprofile="yes" local tmpfile="$PT_TMPDIR/oprofile" mkdir "$d/pt_collect_$p" $CMD_OPCONTROL -p "$mysqld_pid" -d "$d/pt_collect_$p" > "$tmpfile" & @@ -945,7 +947,6 @@ collect_mysql_data_one() { OPERF_PID=$(grep -Eo "kill -SIGINT [[:digit:]]+" "$tmpfile" | grep -Eo "[[:digit:]]+") rm "$tmpfile" fi - have_oprofile="yes" elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then $CMD_STRACE -T -s 0 -f -p $mysqld_pid -o "$d/$p-strace" & local strace_pid=$! @@ -1068,7 +1069,8 @@ collect_mysql_data_two() { if [ "$have_oprofile" ]; then local session="--session-dir=$d/pt_collect_$p" - if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then + if [ $(echo $CMD_OPCONTROL | grep -cv operf) -gt 0 ]; then + # use legacy or custom opcontrol $CMD_OPCONTROL --stop $CMD_OPCONTROL --dump diff --git a/lib/bash/collect.sh b/lib/bash/collect.sh index f28de728..44ce68ea 100644 --- a/lib/bash/collect.sh +++ b/lib/bash/collect.sh @@ -200,12 +200,14 @@ collect_mysql_data_one() { # Next, start oprofile gathering data during the whole rest of this process. # The --init should be a no-op if it has already been init-ed. if [ "$CMD_OPCONTROL" -a "$OPT_COLLECT_OPROFILE" ]; then - if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then - # use legacy opcontrol + if [ $(echo $CMD_OPCONTROL | grep -cv operf) -gt 0 ]; then + # use legacy or custom opcontrol if $CMD_OPCONTROL --init; then $CMD_OPCONTROL --start --no-vmlinux + have_oprofile="yes" fi else + have_oprofile="yes" local tmpfile="$PT_TMPDIR/oprofile" mkdir "$d/pt_collect_$p" # use operf, may fail under VirtualBox or old processor models (see http://oprofile.sourceforge.net/doc/perf_events.html) @@ -214,7 +216,6 @@ collect_mysql_data_one() { OPERF_PID=$(grep -Eo "kill -SIGINT [[:digit:]]+" "$tmpfile" | grep -Eo "[[:digit:]]+") rm "$tmpfile" fi - have_oprofile="yes" elif [ "$CMD_STRACE" -a "$OPT_COLLECT_STRACE" -a "$mysqld_pid" ]; then # Don't run oprofile and strace at the same time. $CMD_STRACE -T -s 0 -f -p $mysqld_pid -o "$d/$p-strace" & @@ -354,7 +355,8 @@ collect_mysql_data_two() { if [ "$have_oprofile" ]; then local session="--session-dir=$d/pt_collect_$p" - if [ $(echo $CMD_OPCONTROL | grep -c opcontrol) -gt 0 ]; then + if [ $(echo $CMD_OPCONTROL | grep -cv operf) -gt 0 ]; then + # use legacy or custom opcontrol $CMD_OPCONTROL --stop $CMD_OPCONTROL --dump From 6330cf0f53df0718698575815186b247e0296cfd Mon Sep 17 00:00:00 2001 From: Sveta Smirnova Date: Fri, 26 Jan 2024 18:14:29 +0300 Subject: [PATCH 5/5] PR-160 - added support for operf if present, and if CMD_OPCONTROL is not set - Added test case --- t/pt-stalk/pt-stalk.t | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/t/pt-stalk/pt-stalk.t b/t/pt-stalk/pt-stalk.t index 19c4dace..d378d351 100644 --- a/t/pt-stalk/pt-stalk.t +++ b/t/pt-stalk/pt-stalk.t @@ -956,6 +956,39 @@ like( "numastat collection has data" ); +# ########################################################################### +# Test if option operf collection works +# ########################################################################### + +# ./bin/pt-stalk --no-stalk --iterations=1 --sleep=1 --dest=tmp/pt-stalk --collect-oprofile -- --user=msandbox --password=msandbox --port=12345 --host=127.0.0.1 ^C +SKIP: { + my $operf = `which operf`; + chomp $operf; + skip "--collect-oprofile tests require operf" unless -x "$operf"; + + cleanup(); + + $retval = system("$trunk/bin/pt-stalk --no-stalk --run-time 10 --sleep 2 --dest $dest --pid $pid_file --iterations 1 --collect-oprofile -- --defaults-file=$cnf >$log_file 2>&1"); + + PerconaTest::wait_until(sub { !-f $pid_file }); + + $output = `ls $dest`; + + like( + $output, + qr/opreport/, + "operf data collected" + ) or diag($output); + + $output = `cat $dest/*-opreport`; + + like( + $output, + qr/(mysqld)/, + "operf collection has data" + ); +} + # ############################################################################# # Done. # #############################################################################