strace: Tracing System Calls for Debugging

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Trace a command
strace ls

# Trace running process
strace -p 1234

# Trace with timestamps
strace -t ls

# Trace with relative timestamps
strace -r ls

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Only file operations
strace -e trace=file ls

# Only network operations
strace -e trace=network curl example.com

# Only process operations
strace -e trace=process bash -c 'sleep 1'

# Specific syscalls
strace -e open,read,write cat file.txt

# Exclude syscalls
strace -e trace=!mmap ls

1
2
3
4
5
6
7

file      # open, stat, chmod, etc.
process   # fork, exec, exit, etc.
network   # socket, connect, send, etc.
signal    # signal, kill, etc.
ipc       # shmget, semop, etc.
desc      # read, write, close, etc.
memory    # mmap, brk, etc.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Write to file
strace -o output.txt ls

# Append to file
strace -o output.txt -A ls

# With timestamps (wall clock)
strace -t ls

# With microseconds
strace -tt ls

# Relative timestamps
strace -r ls

1
2
3
4
5
6

# Follow forked processes
strace -f bash -c 'ls; echo done'

# Follow forks and write separate files
strace -ff -o trace ls
# Creates trace.1234, trace.1235, etc.

1
2
3
4
5

# Show full strings (default truncates at 32 chars)
strace -s 1000 cat file.txt

# Show full strings for specific calls
strace -e read -s 10000 cat file.txt

1
2
3
4
5
6
7
8
9

# Summary of syscalls
strace -c ls

# Sample output:
# % time     seconds  usecs/call     calls    errors syscall
# ------ ----------- ----------- --------- --------- ----------------
#  45.00    0.000045          45         1           execve
#  25.00    0.000025           3         8           mmap
#  15.00    0.000015           2         6           openat

1
2
3

# Summary with detailed trace
strace -c -S time ls  # Sort by time
strace -c -S calls ls # Sort by call count

1
2

# See what files the program is trying to open
strace -e openat ./myprogram 2>&1 | grep -i "no such file"

1
2

# See all files a program tries to read
strace -e openat nginx -t 2>&1 | grep -E "openat.*O_RDONLY"

1
2
3
4
5

# Watch network connections
strace -e connect curl https://example.com

# See DNS lookups
strace -e socket,connect,sendto,recvfrom dig example.com

1
2
3
4
5
6
7

# Attach to hung process
strace -p $(pgrep hung-process)

# Common findings:
# - Waiting on read() = blocked on input
# - Waiting on futex() = waiting for lock
# - Waiting on poll/select = waiting for I/O

1
2
3
4
5
6
7
8

# Time each syscall
strace -T ls

# Shows time spent in each call:
# openat(AT_FDCWD, ".", ...) = 3 <0.000015>

# Summary to find slow operations
strace -c -S time slow-program

1
2
3
4
5

# See access denials
strace -e openat,access ./program 2>&1 | grep -i denied

# Sample output:
# openat(AT_FDCWD, "/etc/secret", O_RDONLY) = -1 EACCES (Permission denied)

1
2
3
4
5

# See all reads and writes
strace -e read,write -s 100 cat file.txt

# Count I/O operations
strace -c -e read,write dd if=/dev/zero of=/dev/null bs=1M count=100

1
2
3
4
5

# Trace signals
strace -e signal,kill ./program

# See what signal killed a process
strace -e trace=signal -p 1234

1
2
3
4
5
6

# See shared library loading
strace -e openat ./program 2>&1 | grep "\.so"

# Common issues:
# - Library not found
# - Wrong library version loaded

1
2
3
4
5

# Make open() fail with ENOENT
strace -e fault=openat:error=ENOENT ls

# Fail every 3rd call
strace -e fault=read:error=EIO:when=3 cat file.txt

1
2
3
4
5

# Decode socket addresses
strace -yy curl example.com

# Decode file descriptors
strace -y cat file.txt

1
2
3
4
5

# Only show failed syscalls
strace -Z ls /nonexistent

# Show syscalls that return specific value
strace -e status=failed ls /nonexistent

1
2
3
4
5
6
7
8

# ltrace - trace library calls (not syscalls)
ltrace ./program

# perf trace - lower overhead
perf trace ls

# bpftrace - more powerful, requires setup
bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s\n", str(args->filename)); }'

1
2
3
4
5

# Use statistics mode (less overhead)
strace -c -p 1234

# Or use perf trace
perf trace -p 1234