Systemd Service Management: Running Applications Reliably

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# /etc/systemd/system/myapp.service
[Unit]
Description=My Application
After=network.target

[Service]
Type=simple
User=appuser
WorkingDirectory=/opt/myapp
ExecStart=/opt/myapp/bin/server
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

1
2
3
4

# Enable and start
sudo systemctl daemon-reload
sudo systemctl enable myapp
sudo systemctl start myapp

1
2
3

[Service]
Type=simple
ExecStart=/usr/bin/myapp

1
2
3
4

[Service]
Type=forking
PIDFile=/var/run/myapp.pid
ExecStart=/usr/bin/myapp --daemon

1
2
3
4

[Service]
Type=oneshot
ExecStart=/opt/myapp/setup.sh
RemainAfterExit=yes

1
2
3
4

[Service]
Type=notify
ExecStart=/usr/bin/myapp
# App must call sd_notify(0, "READY=1")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

[Service]
# Restart options
Restart=always          # Always restart
Restart=on-failure      # Only on non-zero exit
Restart=on-abnormal     # On signal, timeout, watchdog
Restart=no              # Never restart

RestartSec=5            # Wait 5 seconds before restart
StartLimitBurst=5       # Max 5 restarts...
StartLimitIntervalSec=60  # ...per 60 seconds

1

sudo systemctl reset-failed myapp

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

[Service]
# Direct assignment
Environment="NODE_ENV=production"
Environment="PORT=3000"

# From file
EnvironmentFile=/etc/myapp/env

# Multiple files
EnvironmentFile=-/etc/myapp/defaults  # - means optional
EnvironmentFile=/etc/myapp/overrides

1
2
3

# /etc/myapp/env
DATABASE_URL=postgres://localhost/myapp
API_KEY=secret123

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

[Service]
# Memory limits
MemoryMax=512M
MemoryHigh=400M

# CPU limits
CPUQuota=50%

# File descriptors
LimitNOFILE=65535

# Process limits
LimitNPROC=4096

# Nice value
Nice=10

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

[Service]
# Run as unprivileged user
User=appuser
Group=appgroup

# Filesystem restrictions
ProtectSystem=strict      # /usr, /boot read-only
ProtectHome=yes           # No access to /home
ReadWritePaths=/var/lib/myapp

# Namespace isolation
PrivateTmp=yes           # Private /tmp
PrivateDevices=yes       # No access to devices
PrivateNetwork=no        # Usually need network

# Capabilities
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
AmbientCapabilities=CAP_NET_BIND_SERVICE

# System call filtering
SystemCallFilter=@system-service
SystemCallFilter=~@privileged @resources

# No new privileges
NoNewPrivileges=yes

1
2
3
4
5
6
7
8
9

[Service]
ExecStartPre=/opt/myapp/pre-start.sh
ExecStart=/opt/myapp/bin/server
ExecStartPost=/opt/myapp/post-start.sh
ExecStop=/opt/myapp/graceful-stop.sh
ExecStopPost=/opt/myapp/cleanup.sh

# Timeout for stop
TimeoutStopSec=30

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

[Unit]
Description=My Application
Documentation=https://docs.myapp.com

# Start after these
After=network.target postgresql.service redis.service

# Require these (fail if they fail)
Requires=postgresql.service

# Want these (don't fail if they fail)
Wants=redis.service

# If this starts, start me too
WantedBy=multi-user.target

# Part of a target
PartOf=myapp.target

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# /etc/systemd/system/myapp.socket
[Unit]
Description=MyApp Socket

[Socket]
ListenStream=8080
Accept=no

[Install]
WantedBy=sockets.target

1
2
3
4
5
6
7
8
9

# /etc/systemd/system/myapp.service
[Unit]
Description=MyApp Service
Requires=myapp.socket

[Service]
Type=simple
ExecStart=/opt/myapp/bin/server
# Receives socket from systemd

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# /etc/systemd/system/backup.timer
[Unit]
Description=Daily Backup Timer

[Timer]
OnCalendar=*-*-* 02:00:00
Persistent=true

[Install]
WantedBy=timers.target

1
2
3
4
5
6
7

# /etc/systemd/system/backup.service
[Unit]
Description=Backup Service

[Service]
Type=oneshot
ExecStart=/opt/backup/run.sh

1
2
3

sudo systemctl enable backup.timer
sudo systemctl start backup.timer
systemctl list-timers

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# All logs for a service
journalctl -u myapp

# Follow logs (like tail -f)
journalctl -u myapp -f

# Since last boot
journalctl -u myapp -b

# Last 100 lines
journalctl -u myapp -n 100

# With timestamps
journalctl -u myapp --output=short-precise

# Filter by time
journalctl -u myapp --since "1 hour ago"
journalctl -u myapp --since "2024-01-15" --until "2024-01-16"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# Status
systemctl status myapp

# Start/stop/restart
systemctl start myapp
systemctl stop myapp
systemctl restart myapp
systemctl reload myapp  # If supported

# Enable/disable at boot
systemctl enable myapp
systemctl disable myapp

# Check if enabled
systemctl is-enabled myapp

# List all services
systemctl list-units --type=service

# List failed services
systemctl --failed

# Reload systemd after editing units
systemctl daemon-reload

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Why won't it start?
systemctl status myapp
journalctl -u myapp -n 50

# Check unit file syntax
systemd-analyze verify /etc/systemd/system/myapp.service

# See resolved unit file
systemctl cat myapp

# See all settings (including defaults)
systemctl show myapp

# Boot analysis
systemd-analyze blame  # Slowest units
systemd-analyze critical-chain myapp  # Dependency chain

1
2
3
4
5
6

# /etc/systemd/system/myapp@.service
[Unit]
Description=MyApp instance %i

[Service]
ExecStart=/opt/myapp/bin/server --port=%i

1
2
3

systemctl start myapp@8080
systemctl start myapp@8081
systemctl start myapp@8082