Linux

Bash scripts have a reputation for being fragile, unreadable hacks. They don’t have to be. These patterns will help you write scripts that are maintainable, debuggable, and safe to run in production. The Essentials: Start Every Script Right 1 2 3 4 #!/usr/bin/env bash set -euo pipefail IFS=$'\n\t' What these do: set -e — Exit on any error set -u — Error on undefined variables set -o pipefail — Catch errors in pipelines IFS=$'\n\t' — Safer word splitting (no spaces) Script Template 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 #!/usr/bin/env bash set -euo pipefail # Script metadata readonly SCRIPT_NAME="$(basename "$0")" readonly SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" readonly SCRIPT_VERSION="1.0.0" # Default values VERBOSE=false DRY_RUN=false CONFIG_FILE="" # Colors (if terminal supports them) if [[ -t 1 ]]; then readonly RED='\033[0;31m' readonly GREEN='\033[0;32m' readonly YELLOW='\033[0;33m' readonly NC='\033[0m' # No Color else readonly RED='' GREEN='' YELLOW='' NC='' fi # Logging functions log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } log_debug() { [[ "$VERBOSE" == true ]] && echo -e "[DEBUG] $*" || true; } # Cleanup function cleanup() { local exit_code=$? # Add cleanup tasks here log_debug "Cleaning up..." exit $exit_code } trap cleanup EXIT # Usage usage() { cat <<EOF Usage: $SCRIPT_NAME [OPTIONS] <argument> Description of what this script does. Options: -h, --help Show this help message -v, --verbose Enable verbose output -n, --dry-run Show what would be done -c, --config FILE Path to config file --version Show version Examples: $SCRIPT_NAME -v input.txt $SCRIPT_NAME --config /etc/myapp.conf data/ EOF } # Parse arguments parse_args() { while [[ $# -gt 0 ]]; do case $1 in -h|--help) usage exit 0 ;; -v|--verbose) VERBOSE=true shift ;; -n|--dry-run) DRY_RUN=true shift ;; -c|--config) CONFIG_FILE="$2" shift 2 ;; --version) echo "$SCRIPT_NAME version $SCRIPT_VERSION" exit 0 ;; -*) log_error "Unknown option: $1" usage exit 1 ;; *) ARGS+=("$1") shift ;; esac done } # Validation validate() { if [[ ${#ARGS[@]} -eq 0 ]]; then log_error "Missing required argument" usage exit 1 fi if [[ -n "$CONFIG_FILE" && ! -f "$CONFIG_FILE" ]]; then log_error "Config file not found: $CONFIG_FILE" exit 1 fi } # Main logic main() { log_info "Starting $SCRIPT_NAME" log_debug "Arguments: ${ARGS[*]}" for arg in "${ARGS[@]}"; do if [[ "$DRY_RUN" == true ]]; then log_info "[DRY RUN] Would process: $arg" else log_info "Processing: $arg" # Actual work here fi done log_info "Done" } # Entry point ARGS=() parse_args "$@" validate main Error Handling Check Command Existence 1 2 3 4 5 6 7 8 9 10 require_command() { if ! command -v "$1" &> /dev/null; then log_error "Required command not found: $1" exit 1 fi } require_command curl require_command jq require_command docker Retry Pattern 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 retry() { local max_attempts=$1 local delay=$2 shift 2 local cmd=("$@") local attempt=1 while (( attempt <= max_attempts )); do if "${cmd[@]}"; then return 0 fi log_warn "Attempt $attempt/$max_attempts failed, retrying in ${delay}s..." sleep "$delay" ((attempt++)) done log_error "Command failed after $max_attempts attempts: ${cmd[*]}" return 1 } # Usage retry 3 5 curl -sf https://api.example.com/health Timeout Pattern 1 2 3 4 5 6 7 8 9 10 11 12 13 14 with_timeout() { local timeout=$1 shift timeout "$timeout" "$@" || { local exit_code=$? if [[ $exit_code -eq 124 ]]; then log_error "Command timed out after ${timeout}s" fi return $exit_code } } # Usage with_timeout 30 curl -s https://api.example.com Safe File Operations Temporary Files 1 2 3 4 5 6 7 # Create temp file that's automatically cleaned up TEMP_FILE=$(mktemp) trap 'rm -f "$TEMP_FILE"' EXIT # Or temp directory TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT Safe Write (Atomic) 1 2 3 4 5 6 7 8 9 10 11 12 safe_write() { local target=$1 local content=$2 local temp_file temp_file=$(mktemp) echo "$content" > "$temp_file" mv "$temp_file" "$target" } # Usage safe_write /etc/myapp/config.json '{"key": "value"}' Backup Before Modify 1 2 3 4 5 6 7 8 9 backup_file() { local file=$1 local backup="${file}.bak.$(date +%Y%m%d_%H%M%S)" if [[ -f "$file" ]]; then cp "$file" "$backup" log_info "Backup created: $backup" fi } Input Validation File Checks 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 validate_file() { local file=$1 [[ -e "$file" ]] || { log_error "File not found: $file"; return 1; } [[ -f "$file" ]] || { log_error "Not a regular file: $file"; return 1; } [[ -r "$file" ]] || { log_error "File not readable: $file"; return 1; } } validate_directory() { local dir=$1 [[ -e "$dir" ]] || { log_error "Directory not found: $dir"; return 1; } [[ -d "$dir" ]] || { log_error "Not a directory: $dir"; return 1; } [[ -w "$dir" ]] || { log_error "Directory not writable: $dir"; return 1; } } Input Sanitization 1 2 3 4 5 6 7 8 sanitize_filename() { local input=$1 # Remove path components and dangerous characters echo "${input##*/}" | tr -cd '[:alnum:]._-' } # Usage safe_name=$(sanitize_filename "$user_input") Configuration Config File Loading 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 load_config() { local config_file=$1 if [[ -f "$config_file" ]]; then log_debug "Loading config: $config_file" # shellcheck source=/dev/null source "$config_file" else log_warn "Config file not found, using defaults: $config_file" fi } # Config file format (config.sh): # DB_HOST="localhost" # DB_PORT=5432 # ENABLE_FEATURE=true Environment Variable Defaults 1 2 3 4 5 6 : "${DB_HOST:=localhost}" : "${DB_PORT:=5432}" : "${LOG_LEVEL:=info}" # Or with validation DB_HOST="${DB_HOST:?DB_HOST environment variable required}" Parallel Execution Using xargs 1 2 3 4 process_files() { find . -name "*.txt" -print0 | \ xargs -0 -P 4 -I {} bash -c 'process_single_file "$1"' _ {} } Using Background Jobs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 parallel_tasks() { local pids=() for item in "${items[@]}"; do process_item "$item" & pids+=($!) done # Wait for all and check results local failed=0 for pid in "${pids[@]}"; do if ! wait "$pid"; then ((failed++)) fi done [[ $failed -eq 0 ]] || log_error "$failed tasks failed" return $failed } Locking Prevent Concurrent Runs 1 2 3 4 5 6 7 8 9 10 11 LOCK_FILE="/var/run/${SCRIPT_NAME}.lock" acquire_lock() { if ! mkdir "$LOCK_FILE" 2>/dev/null; then log_error "Another instance is running (lock: $LOCK_FILE)" exit 1 fi trap 'rm -rf "$LOCK_FILE"' EXIT } acquire_lock With flock 1 2 3 4 5 6 7 8 9 10 11 12 LOCK_FD=200 LOCK_FILE="/var/run/${SCRIPT_NAME}.lock" acquire_lock() { eval "exec $LOCK_FD>$LOCK_FILE" if ! flock -n $LOCK_FD; then log_error "Another instance is running" exit 1 fi } acquire_lock Testing Assertions 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 assert_equals() { local expected=$1 actual=$2 message=${3:-"Assertion failed"} if [[ "$expected" != "$actual" ]]; then log_error "$message: expected '$expected', got '$actual'" return 1 fi } assert_file_exists() { local file=$1 if [[ ! -f "$file" ]]; then log_error "File should exist: $file" return 1 fi } # Usage assert_equals "200" "$status_code" "HTTP status check" assert_file_exists "/tmp/output.txt" Test Mode 1 2 3 4 5 6 7 if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then # Script is being run directly main "$@" else # Script is being sourced (for testing) log_debug "Script sourced, not executing main" fi Debugging Debug Mode 1 [[ "${DEBUG:-}" == true ]] && set -x Trace Function Calls 1 2 3 4 5 6 7 8 trace() { log_debug "TRACE: ${FUNCNAME[1]}(${*})" } my_function() { trace "$@" # function body } Common Pitfalls Quote Everything 1 2 3 4 5 # Bad if [ $var == "value" ]; then # Good if [[ "$var" == "value" ]]; then Use Arrays for Lists 1 2 3 4 5 6 7 # Bad files="file1.txt file2.txt file with spaces.txt" for f in $files; do echo "$f"; done # Good files=("file1.txt" "file2.txt" "file with spaces.txt") for f in "${files[@]}"; do echo "$f"; done Check Command Success 1 2 3 4 5 6 7 8 9 10 11 # Bad output=$(some_command) echo "$output" # Good if output=$(some_command 2>&1); then echo "$output" else log_error "Command failed: $output" exit 1 fi Good bash scripts are defensive, verbose when needed, and fail fast. Start with the template, add what you need, and resist the urge to be clever. ...

Cron has run scheduled tasks since 1975. It works, but systemd timers offer significant advantages: integrated logging, dependency management, randomized delays, and calendar-based scheduling that actually makes sense. Why Switch from Cron? Logging: Timer output goes to journald. No more digging through mail or custom log files. Dependencies: Wait for network, mounts, or other services before running. Accuracy: Monotonic timers don’t drift. Calendar timers handle DST correctly. Visibility: systemctl list-timers shows all scheduled jobs and when they’ll run next. ...

If you’re still typing ssh -i ~/.ssh/my-key.pem -p 2222 admin@192.168.1.50 every time you connect, you’re doing it wrong. The SSH config file is one of the most underutilized productivity tools in a developer’s arsenal. The Basics: ~/.ssh/config Create or edit ~/.ssh/config: 1 2 3 4 5 Host dev HostName dev.example.com User deploy IdentityFile ~/.ssh/deploy_key Port 22 Now you just type ssh dev. That’s it. Host Patterns Wildcards let you apply settings to multiple hosts: ...

systemd’s journal collects logs from all services, the kernel, and system messages in one place. journalctl is your tool for searching, filtering, and following those logs. Basic Usage 1 2 3 4 5 6 7 8 9 10 11 # Show all logs (oldest first) journalctl # Show all logs (newest first) journalctl -r # Follow new entries (like tail -f) journalctl -f # Show only errors and above journalctl -p err Filter by Time 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 # Since boot journalctl -b # Previous boot journalctl -b -1 # Since specific time journalctl --since "2024-02-25 10:00:00" # Until specific time journalctl --until "2024-02-25 12:00:00" # Time range journalctl --since "1 hour ago" journalctl --since "2024-02-25" --until "2024-02-26" # Relative times journalctl --since "yesterday" journalctl --since "10 minutes ago" Filter by Unit (Service) 1 2 3 4 5 6 7 8 9 10 11 # Specific service journalctl -u nginx # Multiple services journalctl -u nginx -u php-fpm # Follow specific service journalctl -u nginx -f # Service since boot journalctl -u nginx -b Filter by Priority Priority levels (0=emergency to 7=debug): ...

strace intercepts and records system calls made by a process. When a program hangs, crashes, or behaves mysteriously, strace reveals what it’s actually doing at the kernel level. Basic Usage 1 2 3 4 5 6 7 8 9 10 11 # Trace a command strace ls # Trace running process strace -p 1234 # Trace with timestamps strace -t ls # Trace with relative timestamps strace -r ls Filtering System Calls 1 2 3 4 5 6 7 8 9 10 11 12 13 14 # Only file operations strace -e trace=file ls # Only network operations strace -e trace=network curl example.com # Only process operations strace -e trace=process bash -c 'sleep 1' # Specific syscalls strace -e open,read,write cat file.txt # Exclude syscalls strace -e trace=!mmap ls Trace Categories 1 2 3 4 5 6 7 file # open, stat, chmod, etc. process # fork, exec, exit, etc. network # socket, connect, send, etc. signal # signal, kill, etc. ipc # shmget, semop, etc. desc # read, write, close, etc. memory # mmap, brk, etc. Output Options 1 2 3 4 5 6 7 8 9 10 11 12 13 14 # Write to file strace -o output.txt ls # Append to file strace -o output.txt -A ls # With timestamps (wall clock) strace -t ls # With microseconds strace -tt ls # Relative timestamps strace -r ls Following Children 1 2 3 4 5 6 # Follow forked processes strace -f bash -c 'ls; echo done' # Follow forks and write separate files strace -ff -o trace ls # Creates trace.1234, trace.1235, etc. String Output 1 2 3 4 5 # Show full strings (default truncates at 32 chars) strace -s 1000 cat file.txt # Show full strings for specific calls strace -e read -s 10000 cat file.txt Statistics 1 2 3 4 5 6 7 8 9 # Summary of syscalls strace -c ls # Sample output: # % time seconds usecs/call calls errors syscall # ------ ----------- ----------- --------- --------- ---------------- # 45.00 0.000045 45 1 execve # 25.00 0.000025 3 8 mmap # 15.00 0.000015 2 6 openat 1 2 3 # Summary with detailed trace strace -c -S time ls # Sort by time strace -c -S calls ls # Sort by call count Practical Examples Debug “File Not Found” 1 2 # See what files the program is trying to open strace -e openat ./myprogram 2>&1 | grep -i "no such file" Find Config File Locations 1 2 # See all files a program tries to read strace -e openat nginx -t 2>&1 | grep -E "openat.*O_RDONLY" Debug Connection Issues 1 2 3 4 5 # Watch network connections strace -e connect curl https://example.com # See DNS lookups strace -e socket,connect,sendto,recvfrom dig example.com Debug Hanging Process 1 2 3 4 5 6 7 # Attach to hung process strace -p $(pgrep hung-process) # Common findings: # - Waiting on read() = blocked on input # - Waiting on futex() = waiting for lock # - Waiting on poll/select = waiting for I/O Find Why Program is Slow 1 2 3 4 5 6 7 8 # Time each syscall strace -T ls # Shows time spent in each call: # openat(AT_FDCWD, ".", ...) = 3 <0.000015> # Summary to find slow operations strace -c -S time slow-program Debug Permission Issues 1 2 3 4 5 # See access denials strace -e openat,access ./program 2>&1 | grep -i denied # Sample output: # openat(AT_FDCWD, "/etc/secret", O_RDONLY) = -1 EACCES (Permission denied) Watch File I/O 1 2 3 4 5 # See all reads and writes strace -e read,write -s 100 cat file.txt # Count I/O operations strace -c -e read,write dd if=/dev/zero of=/dev/null bs=1M count=100 Debug Signal Handling 1 2 3 4 5 # Trace signals strace -e signal,kill ./program # See what signal killed a process strace -e trace=signal -p 1234 Find Library Loading Issues 1 2 3 4 5 6 # See shared library loading strace -e openat ./program 2>&1 | grep "\.so" # Common issues: # - Library not found # - Wrong library version loaded Advanced Usage Inject Faults 1 2 3 4 5 # Make open() fail with ENOENT strace -e fault=openat:error=ENOENT ls # Fail every 3rd call strace -e fault=read:error=EIO:when=3 cat file.txt Decode Arguments 1 2 3 4 5 # Decode socket addresses strace -yy curl example.com # Decode file descriptors strace -y cat file.txt Trace Specific Syscall Return 1 2 3 4 5 # Only show failed syscalls strace -Z ls /nonexistent # Show syscalls that return specific value strace -e status=failed ls /nonexistent Reading strace Output o │ │ │ │ │ └ p ─ e ─ n a S t y ( s A t T │ │ │ │ └ e _ ─ m F ─ D c C D a W i l D r l , e c n " t a / o m e r e t y c │ │ │ └ / ─ f p ─ d a s P ( s a A w t T d h _ " n F , a D m C O e W _ D R │ │ └ a D ─ r = O ─ g N u c L F m u Y l e r ) a n r g t e = s n t 3 │ └ d ─ i ─ r ) R e t u r n v a l u e ( f d 3 ) Common Return Values = = = = = = = 0 N - - - - - 1 1 1 1 1 E E E E E N A A E I O C G X N E C A I T N E I S R T S N T S F N P R F I u i o e e i n c l r s l t c e s m o e e e u i u r s d c s r e r s e h s c x u s i e i p ( c f o s t f r i n t t e o i l e s d r p e d m t e p s m o o n o y a r r i r s n e a t y o d d r e r i i m c r l a c e y c l o c a l u t u l s n o n l ) t r a y v a i l a b l e Alternatives 1 2 3 4 5 6 7 8 # ltrace - trace library calls (not syscalls) ltrace ./program # perf trace - lower overhead perf trace ls # bpftrace - more powerful, requires setup bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%s\n", str(args->filename)); }' Performance Note strace adds significant overhead — programs run much slower when traced. For production debugging: ...

lsof (list open files) shows which processes have which files open. Since Unix treats everything as a file — including network connections, devices, and pipes — lsof is incredibly powerful for system debugging. Basic Usage 1 2 3 4 5 6 7 8 9 10 11 # List all open files (overwhelming) lsof # List open files for specific file lsof /var/log/syslog # List open files for user lsof -u username # List open files for process lsof -p 1234 Find What’s Using a File 1 2 3 4 5 6 7 8 # Who has this file open? lsof /path/to/file # Who has files open in this directory? lsof +D /var/log # Recursive directory search lsof +D /var/log/ “Device or resource busy” Debugging 1 2 3 4 5 # Can't unmount? Find what's using it lsof +D /mnt/usb # Find open files on filesystem lsof /dev/sda1 Find What’s Using a Port 1 2 3 4 5 6 7 8 9 10 11 12 13 14 # What's on port 8080? lsof -i :8080 # What's on port 80 or 443? lsof -i :80 -i :443 # All network connections lsof -i # TCP only lsof -i TCP # UDP only lsof -i UDP Specific Protocol and Port 1 2 3 4 5 6 7 8 # TCP port 22 lsof -i TCP:22 # UDP port 53 lsof -i UDP:53 # Port range lsof -i :1-1024 Filter by Process 1 2 3 4 5 6 7 8 9 10 11 # By PID lsof -p 1234 # By process name lsof -c nginx # By multiple process names lsof -c nginx -c apache # Exclude process lsof -c ^nginx Filter by User 1 2 3 4 5 6 7 8 9 10 11 # By username lsof -u www-data # By UID lsof -u 1000 # Multiple users lsof -u user1,user2 # Exclude user lsof -u ^root Network Connections List All Connections 1 2 3 4 5 6 7 8 # All internet connections lsof -i # Established connections only lsof -i | grep ESTABLISHED # Listening sockets lsof -i | grep LISTEN Connections to Specific Host 1 2 3 4 5 6 7 8 # Connections to specific IP lsof -i @192.168.1.100 # Connections to hostname lsof -i @example.com # Connections to specific host and port lsof -i @192.168.1.100:22 IPv4 vs IPv6 1 2 3 4 5 # IPv4 only lsof -i 4 # IPv6 only lsof -i 6 Output Formatting 1 2 3 4 5 6 7 8 9 10 # Terse output (parseable) lsof -t /path/to/file # Returns just PIDs # No header lsof +D /var/log | tail -n +2 # Specific fields lsof -F p /path/to/file # PID only lsof -F c /path/to/file # Command only Combining Filters By default, filters are OR’d. Use -a to AND them: ...

Unix permissions determine who can do what with files and directories. Understanding them is fundamental to system security and avoiding “Permission denied” errors. The Permission Model Every file has three permission types for three user classes: Permission Types: r (read) = 4 w (write) = 2 x (execute) = 1 User Classes: u (user/owner) g (group) o (others) Reading Permissions 1 2 ls -l file.txt # -rw-r--r-- 1 user group 1234 Feb 25 12:00 file.txt Breaking down -rw-r--r--: ...

tar (tape archive) bundles files and directories into a single file. Combined with compression, it’s the standard way to package and distribute files on Unix systems. The Basics 1 2 3 4 5 6 7 8 # Create archive tar -cvf archive.tar /path/to/files # Extract archive tar -xvf archive.tar # List contents tar -tvf archive.tar Understanding the Flags 1 2 3 4 5 c = Create archive x = Extract archive t = List contents v = Verbose (show files) f = File (next arg is filename) So tar -cvf = Create, Verbose, File. ...

rsync synchronizes files between locations — local to local, local to remote, remote to local. It’s smart: it only transfers what’s changed, making it fast for incremental backups and deployments. Basic Syntax 1 rsync [options] source destination Local Sync 1 2 3 4 5 6 7 8 9 # Copy directory rsync -av /source/dir/ /dest/dir/ # Copy directory (trailing slash matters!) rsync -av /source/dir /dest/ # Creates /dest/dir/ rsync -av /source/dir/ /dest/ # Contents into /dest/ # Dry run (show what would happen) rsync -avn /source/ /dest/ Remote Sync (SSH) 1 2 3 4 5 6 7 8 9 10 11 # Local to remote rsync -av /local/dir/ user@remote:/remote/dir/ # Remote to local rsync -av user@remote:/remote/dir/ /local/dir/ # Different SSH port rsync -av -e 'ssh -p 2222' /local/ user@remote:/remote/ # With SSH key rsync -av -e 'ssh -i ~/.ssh/mykey' /local/ user@remote:/remote/ Common Options 1 2 3 4 5 6 7 8 9 -a, --archive Archive mode (preserves permissions, timestamps, etc.) -v, --verbose Verbose output -n, --dry-run Show what would be transferred -z, --compress Compress during transfer -P Progress + partial (resume interrupted transfers) --progress Show progress --delete Delete files in dest not in source -r, --recursive Recurse into directories -h, --human-readable Human-readable sizes Archive Mode (-a) -a is equivalent to -rlptgoD: ...

netcat (nc) does one thing: move bytes over a network. That simplicity makes it incredibly versatile — port scanning, file transfers, chat servers, proxies, and network debugging all become one-liners. Basic Usage 1 2 3 4 5 6 7 8 # Connect to host:port nc hostname 80 # Listen on port nc -l 8080 # Listen (keep listening after disconnect) nc -lk 8080 Test if Port is Open 1 2 3 4 5 6 7 8 9 # Quick port check nc -zv hostname 22 # Connection to hostname 22 port [tcp/ssh] succeeded! # Scan port range nc -zv hostname 20-25 # With timeout nc -zv -w 3 hostname 443 Simple Client-Server Communication Server (listener) 1 nc -l 9000 Client 1 nc localhost 9000 Now type in either terminal — text flows both ways. Ctrl+C to exit. ...