mirror of
https://github.com/brendangregg/perf-tools.git
synced 2025-11-30 23:16:03 +07:00
391 lines
12 KiB
Bash
Executable File
391 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# uprobe - trace a given uprobe definition. User-level dynamic tracing.
|
|
# Written using Linux ftrace. Experimental.
|
|
#
|
|
# This will create, trace, then destroy a given uprobe definition. See
|
|
# Documentation/trace/uprobetrace.txt in the Linux kernel source for the
|
|
# syntax of a uprobe definition, and "uprobe -h" for examples. With this tool,
|
|
# the probe alias is optional (it will default to something meaningful).
|
|
#
|
|
# USAGE: ./uprobe [-FhHsv] [-d secs] [-p pid] [-L tid] {-l target |
|
|
# uprobe_definition [filter]}
|
|
#
|
|
# Run "uprobe -h" for full usage.
|
|
#
|
|
# WARNING: This uses dynamic tracing of user-level functions, using some
|
|
# relatively new kernel code. I have seen this cause target processes to fail,
|
|
# either entering endless spin loops or crashing on illegal instructions. I
|
|
# believe newer kernels (post 4.0) are relatively safer, but use caution. Test
|
|
# in a lab environment, and know what you are doing, before use.
|
|
#
|
|
# Use extreme caution with the raw address mode: eg, "p:libc:0xbf130". uprobe
|
|
# does not check for instruction alignment, so tracing the wrong address (eg,
|
|
# mid-way through a multi-byte instruction) will corrupt the target's memory.
|
|
# Other tracers (eg, perf_events with debuginfo) check alignment.
|
|
#
|
|
# Also beware of widespread tracing that interferes with the operation of the
|
|
# system, eg, tracing libc:malloc, which by-default will trace _all_ processes.
|
|
# Test in a lab environment before use.
|
|
#
|
|
# I wrote this because I kept testing different custom uprobes at the command
|
|
# line, and wanted a way to automate the steps. For generic user-level
|
|
# tracing, use perf_events directly.
|
|
#
|
|
# REQUIREMENTS: FTRACE and UPROBE CONFIG, which you may already have on recent
|
|
# kernel versions, file(1), ldconfig(8), objdump(1), and some version of awk.
|
|
# Also, currently only executes on Linux 4.0+ (see WARNING) unless -F is used.
|
|
#
|
|
# From perf-tools: https://github.com/brendangregg/perf-tools
|
|
#
|
|
# See the uprobe(8) man page (in perf-tools) for more info.
|
|
#
|
|
# COPYRIGHT: Copyright (c) 2015 Brendan Gregg.
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software Foundation,
|
|
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
#
|
|
# (http://www.gnu.org/copyleft/gpl.html)
|
|
#
|
|
# 27-Jul-2015 Brendan Gregg Created this.
|
|
|
|
### default variables
|
|
tracing=/sys/kernel/debug/tracing
|
|
flock=/var/tmp/.ftrace-lock; wroteflock=0
|
|
opt_duration=0; duration=; opt_pid=0; pid=; opt_tid=0; tid=
|
|
opt_filter=0; filter=; opt_view=0; opt_headers=0; opt_stack=0; dmesg=2
|
|
debug=0; opt_force=0; opt_list=0; target=
|
|
PATH=$PATH:/usr/bin:/sbin # ensure we find objdump, ldconfig
|
|
trap ':' INT QUIT TERM PIPE HUP # sends execution to end tracing section
|
|
|
|
function usage {
|
|
cat <<-END >&2
|
|
USAGE: uprobe [-FhHsv] [-d secs] [-p PID] [-L TID] {-l target |
|
|
uprobe_definition [filter]}
|
|
-F # force. trace despite warnings.
|
|
-d seconds # trace duration, and use buffers
|
|
-l target # list functions from this executable
|
|
-p PID # PID to match on events
|
|
-L TID # thread id to match on events
|
|
-v # view format file (don't trace)
|
|
-H # include column headers
|
|
-s # show user stack traces
|
|
-h # this usage message
|
|
|
|
Note that these examples may need modification to match your kernel
|
|
version's function names and platform's register usage.
|
|
eg,
|
|
# trace readline() calls in all running "bash" executables:
|
|
uprobe p:bash:readline
|
|
# trace readline() with explicit executable path:
|
|
uprobe p:/bin/bash:readline
|
|
# trace the return of readline() with return value as a string:
|
|
uprobe 'r:bash:readline +0(\$retval):string'
|
|
# trace sleep() calls in all running libc shared libraries:
|
|
uprobe p:libc:sleep
|
|
# trace sleep() with register %di (x86):
|
|
uprobe 'p:libc:sleep %di'
|
|
# trace this address (use caution: must be instruction aligned):
|
|
uprobe p:libc:0xbf130
|
|
# trace gettimeofday() for PID 1182 only:
|
|
uprobe -p 1182 p:libc:gettimeofday
|
|
# trace the return of fopen() only when it returns NULL:
|
|
uprobe 'r:libc:fopen file=\$retval' 'file == 0'
|
|
|
|
See the man page and example file for more info.
|
|
END
|
|
exit
|
|
}
|
|
|
|
function warn {
|
|
if ! eval "$@"; then
|
|
echo >&2 "WARNING: command failed \"$@\""
|
|
fi
|
|
}
|
|
|
|
function end {
|
|
# disable tracing
|
|
echo 2>/dev/null
|
|
echo "Ending tracing..." 2>/dev/null
|
|
cd $tracing
|
|
warn "echo 0 > events/uprobes/$uname/enable"
|
|
if (( opt_filter )); then
|
|
warn "echo 0 > events/uprobes/$uname/filter"
|
|
fi
|
|
warn "echo -:$uname >> uprobe_events"
|
|
(( opt_stack )) && warn "echo 0 > options/userstacktrace"
|
|
warn "echo > trace"
|
|
(( wroteflock )) && warn "rm $flock"
|
|
}
|
|
|
|
function die {
|
|
echo >&2 "$@"
|
|
exit 1
|
|
}
|
|
|
|
function edie {
|
|
# die with a quiet end()
|
|
echo >&2 "$@"
|
|
exec >/dev/null 2>&1
|
|
end
|
|
exit 1
|
|
}
|
|
|
|
function set_path {
|
|
name=$1
|
|
|
|
path=$(which $name)
|
|
if [[ "$path" == "" ]]; then
|
|
path=$(ldconfig -v 2>/dev/null | awk -v lib=$name '
|
|
$1 ~ /:/ { sub(/:/, "", $1); path = $1 }
|
|
{ sub(/\..*/, "", $1); }
|
|
$1 == lib { print path "/" $3 }')
|
|
if [[ "$path" == "" ]]; then
|
|
die "ERROR: segment \"$name\" ambiguous." \
|
|
"Program or library? Try a full path."
|
|
fi
|
|
fi
|
|
|
|
if [[ ! -x $path ]]; then
|
|
die "ERROR: resolved \"$name\" to \"$path\", but file missing"
|
|
fi
|
|
}
|
|
|
|
function set_addr {
|
|
path=$1
|
|
name=$2
|
|
sym=$3
|
|
|
|
[[ "$path" == "" ]] && die "ERROR: missing symbol path."
|
|
[[ "$sym" == "" ]] && die "ERROR: missing symbol for $path"
|
|
|
|
addr=$(objdump -tT $path | awk -v sym=$sym '
|
|
$NF == sym && $4 == ".text" { print $1; exit }')
|
|
[[ "$addr" == "" ]] && die "ERROR: missing symbol \"$sym\" in $path"
|
|
(( 0x$addr == 0 )) && die "ERROR: failed resolving \"$sym\" in $path." \
|
|
"Maybe it exists in a different target (eg, library)?"
|
|
addr=0x$( printf "%x" 0x$addr ) # strip leading zeros
|
|
|
|
type=$(file $path)
|
|
if [[ "$type" != *shared?object* ]]; then
|
|
# subtract the base mapping address. see Documentation/trace/
|
|
# uprobetracer.txt for background.
|
|
base=$(objdump -x $path | awk '
|
|
$1 == "LOAD" && $3 ~ /^[0x]*$/ { print $5 }')
|
|
[[ "$base" != 0x* ]] && die "ERROR: finding base load addr"\
|
|
"for $path."
|
|
addr=$(( addr - base ))
|
|
(( addr < 0 )) && die "ERROR: transposed address for $sym"\
|
|
"became negative: $addr"
|
|
addr=0x$( printf "%x" $addr)
|
|
fi
|
|
}
|
|
|
|
### process options
|
|
while getopts Fd:hHl:p:L:sv opt
|
|
do
|
|
case $opt in
|
|
F) opt_force=1 ;;
|
|
d) opt_duration=1; duration=$OPTARG ;;
|
|
p) opt_pid=1; pid=$OPTARG ;;
|
|
L) opt_tid=1; tid=$OPTARG ;;
|
|
l) opt_list=1; target=$OPTARG ;;
|
|
H) opt_headers=1 ;;
|
|
s) opt_stack=1 ;;
|
|
v) opt_view=1 ;;
|
|
h|?) usage ;;
|
|
esac
|
|
done
|
|
shift $(( $OPTIND - 1 ))
|
|
uprobe=$1
|
|
shift
|
|
if (( $# )); then
|
|
opt_filter=1
|
|
filter=$1
|
|
fi
|
|
|
|
### handle listing
|
|
[[ "$opt_list" == 1 && "$uprobe" != "" ]] && die "ERROR: -l takes a target only"
|
|
if (( opt_list )); then
|
|
if [[ "$target" != */* ]]; then
|
|
set_path $target
|
|
target=$path
|
|
fi
|
|
objdump -tT $target | awk '$4 == ".text" { print $NF }' | sort | uniq
|
|
exit
|
|
fi
|
|
|
|
### check kernel version
|
|
ver=$(uname -r)
|
|
maj=${ver%%.*}
|
|
if (( opt_force == 0 && $maj < 4 )); then
|
|
cat <<-END >&2
|
|
ERROR: Kernel version >= 4.0 preferred (you have $ver). Aborting.
|
|
|
|
Background: uprobes were first added in 3.5. I've tested them on 3.13,
|
|
and found them unsafe, as they can crash or lock up processes, which can
|
|
effectively lock up the system. On 4.0, uprobes seem much safer. You
|
|
can use -F to force tracing, but you've been warned.
|
|
END
|
|
exit
|
|
fi
|
|
|
|
### check command dependencies
|
|
for cmd in file objdump ldconfig awk; do
|
|
which $cmd > /dev/null
|
|
(( $? != 0 )) && die "ERROR: missing $cmd in \$PATH. $0 needs" \
|
|
"to use this command. Exiting."
|
|
done
|
|
|
|
### option logic
|
|
[[ "$uprobe" == "" ]] && usage
|
|
(( opt_pid + opt_filter + opt_tid > 1 )) && \
|
|
die "ERROR: use at most one of -p, -L, or filter."
|
|
(( opt_duration && opt_view )) && die "ERROR: use either -d or -v."
|
|
if (( opt_pid )); then
|
|
# convert to filter
|
|
opt_filter=1
|
|
# ftrace common_pid is thread id from user's perspective
|
|
for tid in /proc/$pid/task/*; do
|
|
filter="$filter || common_pid == ${tid##*/}"
|
|
done
|
|
filter=${filter:3} # trim leading ' || ' (four characters)
|
|
fi
|
|
if (( opt_tid )); then
|
|
opt_filter=1
|
|
filter="common_pid == $tid"
|
|
fi
|
|
if [[ "$uprobe" != p:* && "$uprobe" != r:* ]]; then
|
|
echo >&2 "ERROR: invalid uprobe definition (should start with p: or r:)"
|
|
usage
|
|
fi
|
|
#
|
|
# Parse the following:
|
|
# p:bash:readline
|
|
# p:my bash:readline
|
|
# p:bash:readline %si
|
|
# r:bash:readline $ret
|
|
# p:my bash:readline %si
|
|
# p:bash:readline si=%si
|
|
# p:my bash:readline si=%si
|
|
# r:bash:readline cmd=+0($retval):string
|
|
# ... and all of the above with /bin/bash instead of bash
|
|
# ... and all of the above with libc:sleep instead of ...
|
|
# ... and all of the above with /lib/x86_64-linux-gnu/libc.so.6:sleep ...
|
|
# ... and all of the above with symbol addresses
|
|
# ... and examples from USAGE message
|
|
# The following code is not as complicated as it looks.
|
|
#
|
|
utype=${uprobe%%:*}
|
|
urest="${uprobe#*:} "
|
|
set -- $urest
|
|
if [[ $1 == *:* ]]; then
|
|
uname=; probe=$1; shift; uargs="$@"
|
|
else
|
|
[[ $2 != *:* ]] && die "ERROR: invalid probe. See usage (-h)."
|
|
uname=$1; probe=$2; shift 2; uargs="$@"
|
|
fi
|
|
path=$probe; path=${path%%:*}
|
|
addr=$probe; addr=${addr##*:}
|
|
|
|
# set seg and fix path (eg, seg=bash, path=/bin/bash)
|
|
if [[ $path == */* ]]; then
|
|
seg=${path##*/}
|
|
seg=${seg%%.*}
|
|
else
|
|
seg=$path
|
|
# determine path, eg, given "zsh" or "libc"
|
|
set_path $path
|
|
fi
|
|
|
|
# fix uname and addr (eg, uname=readline, addr=0x8db60)
|
|
if [[ "$addr" == 0x* ]]; then
|
|
# symbol unknown; default to seg+addr
|
|
[[ "$uname" == "" ]] && uname=${seg}_$addr
|
|
else
|
|
[[ "$uname" == "" ]] && uname=$addr
|
|
set_addr $path $seg $addr
|
|
fi
|
|
|
|
# construct uprobe
|
|
uprobe="$utype:$uname $path:$addr"
|
|
[[ "$uargs" != "" ]] && uprobe="$uprobe $uargs"
|
|
|
|
if (( debug )); then
|
|
echo "uname: \"$uname\", uprobe: \"$uprobe\""
|
|
fi
|
|
|
|
### check permissions
|
|
cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
|
|
debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
|
|
|
|
if (( !opt_view )); then
|
|
if (( opt_duration )); then
|
|
echo "Tracing uprobe $uname for $duration seconds (buffered)..."
|
|
else
|
|
echo "Tracing uprobe $uname ($uprobe). Ctrl-C to end."
|
|
fi
|
|
fi
|
|
|
|
### ftrace lock
|
|
[[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
|
|
echo $$ > $flock || die "ERROR: unable to write $flock."
|
|
wroteflock=1
|
|
|
|
### setup and begin tracing
|
|
echo nop > current_tracer
|
|
if ! echo "$uprobe" >> uprobe_events; then
|
|
echo >&2 "ERROR: adding uprobe \"$uprobe\"."
|
|
if (( dmesg )); then
|
|
echo >&2 "Last $dmesg dmesg entries (might contain reason):"
|
|
dmesg | tail -$dmesg | sed 's/^/ /'
|
|
fi
|
|
edie "Exiting."
|
|
fi
|
|
if (( opt_view )); then
|
|
cat events/uprobes/$uname/format
|
|
edie ""
|
|
fi
|
|
if (( opt_filter )); then
|
|
if ! echo "$filter" > events/uprobes/$uname/filter; then
|
|
edie "ERROR: setting filter or -p. Exiting."
|
|
fi
|
|
fi
|
|
if (( opt_stack )); then
|
|
if ! echo 1 > options/userstacktrace; then
|
|
edie "ERROR: enabling stack traces (-s). Exiting"
|
|
fi
|
|
fi
|
|
if ! echo 1 > events/uprobes/$uname/enable; then
|
|
edie "ERROR: enabling uprobe $uname. Exiting."
|
|
fi
|
|
|
|
### print trace buffer
|
|
warn "echo > trace"
|
|
if (( opt_duration )); then
|
|
sleep $duration
|
|
if (( opt_headers )); then
|
|
cat trace
|
|
else
|
|
grep -v '^#' trace
|
|
fi
|
|
else
|
|
# trace_pipe lack headers, so fetch them from trace
|
|
(( opt_headers )) && cat trace
|
|
cat trace_pipe
|
|
fi
|
|
|
|
### end tracing
|
|
end
|