Log Analysis

Log Analysis

Search and analyze system and application logs effectively.

Quick Reference

# Recent errors
journalctl -p err --since "1 hour ago"

# Follow logs
tail -f /var/log/syslog

# Search logs
grep -i error /var/log/syslog | tail -50

System Logs

journalctl

# All logs
journalctl

# Follow (like tail -f)
journalctl -f

# Since boot
journalctl -b

# By unit/service
journalctl -u nginx
journalctl -u ssh

# By priority
journalctl -p err        # Errors and above
journalctl -p warning    # Warnings and above

# Time range
journalctl --since "2024-01-15 10:00"
journalctl --since "1 hour ago"
journalctl --since yesterday --until today

# Kernel messages
journalctl -k

# JSON output
journalctl -o json-pretty

Traditional Logs

# System log
tail -f /var/log/syslog
tail -f /var/log/messages  # RHEL

# Auth log
tail -f /var/log/auth.log
tail -f /var/log/secure    # RHEL

# Kernel
dmesg
dmesg -T  # Human-readable timestamps

Searching Logs

grep Basics

# Simple search
grep "error" /var/log/syslog

# Case insensitive
grep -i "error" /var/log/syslog

# With context
grep -B 5 -A 5 "error" /var/log/syslog

# Count occurrences
grep -c "error" /var/log/syslog

# Show line numbers
grep -n "error" /var/log/syslog

# Multiple patterns
grep -E "error|warning|critical" /var/log/syslog

# Exclude pattern
grep -v "DEBUG" /var/log/syslog

Advanced Patterns

# IP addresses
grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' /var/log/auth.log

# Timestamps
grep "Jan 15 14:" /var/log/syslog

# Failed logins
grep "Failed password" /var/log/auth.log

# SSH connections
grep "Accepted" /var/log/auth.log

Log Analysis Tools

awk

# Count by field
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# Sum values
awk '{sum += $10} END {print sum}' /var/log/nginx/access.log

# Filter by field
awk '$9 >= 400' /var/log/nginx/access.log

Aggregation

# Count errors per hour
grep "error" /var/log/syslog | cut -d: -f1-2 | sort | uniq -c

# Top IPs in access log
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -10

# Status code distribution
awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

Real-time Monitoring

# Multiple files
tail -f /var/log/nginx/*.log

# Highlight matches
tail -f /var/log/syslog | grep --color=always -E 'error|$'

# With timestamp
tail -f /var/log/syslog | while read line; do echo "$(date): $line"; done

Compressed Logs

# Search compressed
zgrep "error" /var/log/syslog.*.gz

# View compressed
zcat /var/log/syslog.1.gz | tail -100

# Search across all
zcat /var/log/syslog.*.gz | grep "error"

Log Locations

Quick Analysis Script

#!/bin/bash
echo "=== Recent Errors ==="
journalctl -p err --since "1 hour ago" | tail -20

echo "=== Failed Logins ==="
grep "Failed password" /var/log/auth.log | tail -10

echo "=== Disk Usage ==="
df -h | grep -E "^/dev"

echo "=== Memory ==="
free -h