Backup Strategies: Because Hope Is Not a Disaster Recovery Plan

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# Document what's backed up and where
cat > /etc/backup-manifest.yml << EOF
databases:
  postgres_main:
    schedule: hourly
    retention: 30 days
    location: s3://backups/postgres/
  
configs:
  nginx:
    schedule: daily
    retention: 90 days
    location: s3://backups/configs/
    
secrets:
  vault_data:
    schedule: daily
    retention: 365 days
    location: s3://backups/vault/ (encrypted)
EOF

1
2
3
4
5

# Logical backup (portable, slower)
pg_dump -h localhost -U postgres mydb | gzip > mydb_$(date +%Y%m%d_%H%M%S).sql.gz

# Physical backup (faster, same version required)
pg_basebackup -h localhost -D /backup/base -Ft -z -P

1
2
3

# postgresql.conf
archive_mode = on
archive_command = 'aws s3 cp %p s3://wal-archive/%f'

1
2
3
4
5

# Logical backup
mysqldump --all-databases --single-transaction | gzip > mysql_$(date +%Y%m%d).sql.gz

# Physical backup with Percona XtraBackup
xtrabackup --backup --target-dir=/backup/mysql/

1
2
3
4
5
6
7

# Initialize repository
restic -r s3:s3.amazonaws.com/my-backups init

# Backup
restic -r s3:s3.amazonaws.com/my-backups backup /var/www /etc/nginx

# Incremental by default, deduplication built-in

1
2

# Automate with cron
0 * * * * restic -r s3:s3.amazonaws.com/my-backups backup /var/www --quiet

1
2
3
4
5

# Mirror to backup server
rsync -avz --delete /var/www/ backup-server:/backups/www/

# With hard links for space-efficient versioning
rsync -avz --delete --link-dest=/backups/www/latest /var/www/ /backups/www/$(date +%Y%m%d)/

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# RDS automated backups
aws rds modify-db-instance \
  --db-instance-identifier mydb \
  --backup-retention-period 30

# EBS snapshots
aws ec2 create-snapshot \
  --volume-id vol-1234567890 \
  --description "Daily backup $(date +%Y%m%d)"

# S3 versioning (not a backup, but helps)
aws s3api put-bucket-versioning \
  --bucket my-bucket \
  --versioning-configuration Status=Enabled

1
2
3
4
5
6
7
8

# Velero for cluster backup
velero install --provider aws --bucket velero-backups

# Backup namespace
velero backup create app-backup --include-namespaces production

# Schedule daily backups
velero schedule create daily --schedule="0 2 * * *" --include-namespaces production

1
2
3
4
5
6

retention:
  hourly: 24      # Last 24 hours
  daily: 30       # Last 30 days
  weekly: 12      # Last 12 weeks
  monthly: 12     # Last 12 months
  yearly: 5       # Last 5 years

1
2
3
4
5
6
7

# Restic retention
restic forget \
  --keep-hourly 24 \
  --keep-daily 30 \
  --keep-weekly 12 \
  --keep-monthly 12 \
  --prune

1
2
3
4
5
6

# Restic encrypts by default (repository password)
restic -r s3:bucket/backups init
# Enter password for new repository:

# GPG for manual encryption
pg_dump mydb | gzip | gpg --encrypt -r backup@company.com > backup.sql.gz.gpg

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Monthly restore drill
#!/bin/bash
set -e

echo "Starting restore test..."

# Restore to test environment
restic -r s3:bucket/backups restore latest --target /tmp/restore-test

# Verify database
pg_restore -d restore_test /tmp/restore-test/postgres/backup.dump
psql restore_test -c "SELECT count(*) FROM critical_table;"

# Compare checksums
find /tmp/restore-test -type f -exec md5sum {} \; > /tmp/restore-checksums.txt

echo "Restore test completed successfully"

1
2
3
4
5
6
7

# Check last backup age
last_backup=$(restic -r s3:bucket/backups snapshots --json | jq -r '.[-1].time')
age_hours=$(( ($(date +%s) - $(date -d "$last_backup" +%s)) / 3600 ))

if [ $age_hours -gt 24 ]; then
  echo "ALERT: Last backup is $age_hours hours old"
fi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

# Disaster Recovery Runbook

## Scenario: Complete datacenter loss

### Prerequisites
- AWS credentials in 1Password
- Terraform state in S3 (us-west-2)
- Backups in S3 (eu-west-1)

### Steps
1. Provision new infrastructure
   - `cd terraform/dr && terraform apply`
   
2. Restore database
   - Download latest backup: `restic restore latest`
   - Import: `pg_restore -d production backup.dump`
   
3. Deploy application
   - `kubectl apply -f k8s/`
   
4. Update DNS
   - Point api.example.com to new load balancer
   
### Verification
- [ ] API responds at /health
- [ ] Database connectivity confirmed
- [ ] User can log in

### Expected RTO: 2 hours
### Expected RPO: 1 hour (hourly backups)