Application Crashes

Application Crashes

Debug application crashes and unexpected behavior.

Quick Diagnostics

# Check if running
pgrep -a myapp
systemctl status myapp

# Recent logs
journalctl -u myapp --since "10 minutes ago"
tail -100 /var/log/myapp/error.log

# Exit code of last run
echo $?

Finding Crash Information

System Logs

# Systemd service logs
journalctl -u myapp -e
journalctl -u myapp --since today

# Kernel messages (segfaults)
dmesg | tail -50
dmesg | grep myapp

# Syslog
grep myapp /var/log/syslog

Application Logs

# Follow logs
tail -f /var/log/myapp/*.log

# Search for errors
grep -i error /var/log/myapp/*.log | tail -50
grep -i exception /var/log/myapp/*.log | tail -50

# Timestamps around crash
grep "2024-01-15 14:3" /var/log/myapp/app.log

Core Dumps

Enable Core Dumps

# Check current limit
ulimit -c

# Enable
ulimit -c unlimited

# Persistent (add to /etc/security/limits.conf)
* soft core unlimited
* hard core unlimited

# Core dump location
cat /proc/sys/kernel/core_pattern

Analyze Core Dump

# Install debugger
sudo apt install gdb

# Analyze with gdb
gdb /path/to/binary /path/to/core

# In gdb
(gdb) bt        # Backtrace
(gdb) bt full   # Detailed backtrace
(gdb) info registers

Stack Traces

Java

# Get thread dump
jstack PID

# Or send signal
kill -3 PID  # Dumps to stdout

# Heap dump
jmap -dump:live,format=b,file=heap.hprof PID

Python

# Enable faulthandler
python -X faulthandler myapp.py

# Or in code
import faulthandler
faulthandler.enable()

# Get traceback of running process
kill -USR1 PID  # If handler installed

Node.js

# Enable core dumps
node --abort-on-uncaught-exception app.js

# Inspect with lldb/gdb
lldb node core.PID

# Get heap snapshot
kill -USR2 PID

Memory Issues

Out of Memory

# Check OOM killer
dmesg | grep -i "out of memory"
journalctl -k | grep -i oom

# Memory usage when crashed
grep -A 5 "Out of memory" /var/log/syslog

# Current memory
ps aux --sort=-rss | head -10

Memory Leaks

# Monitor memory growth
while true; do
  ps -o pid,rss,vsz,comm -p PID
  sleep 60
done

# Valgrind (if applicable)
valgrind --leak-check=full ./myapp

Restart and Recovery

# Systemd restart
sudo systemctl restart myapp

# Check restart count
systemctl show myapp | grep NRestarts

# View restart settings
systemctl cat myapp | grep Restart

# Manual restart with debug
/path/to/myapp --debug 2>&1 | tee debug.log

Common Crash Causes