Comprehensive security update automation: 1. Debian OS (unattended-upgrades) — tightened to security-only: - Removed general Debian updates (prevents feature/breaking changes) - Only Debian-Security origins auto-installed - Email admin on every upgrade via ops@performancewest.net - Auto-reboot at 4 AM if kernel update requires it - needrestart auto-restarts services after library updates 2. Docker CE — major version guard: - Patch updates within pinned major version auto-applied - Major version jumps held + admin alerted for manual review - docker-ce, docker-ce-cli, containerd.io all version-guarded 3. Container base images — daily at 3:30 AM: - Pulls latest base images for all docker-compose services - Compares image digests — only rebuilds if changed - Restarts only affected services (not full stack) - Alerts admin on rebuild failures requiring manual intervention - Covers both prod and dev compose projects 4. k3s — weekly Sunday at 3:45 AM: - Patch updates within current minor auto-applied - Minor/major upgrades alert admin for manual review - Verifies node Ready status after update - Alerts on failures with investigation instructions 5. Admin notifications via SMTP: - [INFO] for successful patches - [WARNING] for available major upgrades needing review - [CRITICAL] for failures requiring immediate intervention - Falls back to syslog if SMTP unavailable Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
112 lines
4.7 KiB
Django/Jinja
112 lines
4.7 KiB
Django/Jinja
#!/usr/bin/env bash
|
|
# ══════════════════════════════════════════════════════════════════════════════
|
|
# Container Security Update — Performance West
|
|
#
|
|
# Pulls latest base images for all docker-compose services. If any image
|
|
# has changed (security patches in upstream base), rebuilds and restarts
|
|
# only the affected services. Sends admin alert if a rebuild fails.
|
|
#
|
|
# Runs via systemd timer (pw-container-update.timer)
|
|
# ══════════════════════════════════════════════════════════════════════════════
|
|
set -euo pipefail
|
|
|
|
LOG_TAG="pw-container-update"
|
|
ALERT_SCRIPT="/usr/local/bin/pw-security-alert"
|
|
COMPOSE_PROJECTS=({{ compose_projects | map('quote') | join(' ') }})
|
|
|
|
log() { logger -t "$LOG_TAG" "$*"; echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
|
|
|
updated_services=()
|
|
failed_services=()
|
|
|
|
for PROJECT_DIR in "${COMPOSE_PROJECTS[@]}"; do
|
|
if [ ! -f "$PROJECT_DIR/docker-compose.yml" ]; then
|
|
log "SKIP: No docker-compose.yml in $PROJECT_DIR"
|
|
continue
|
|
fi
|
|
|
|
log "Checking $PROJECT_DIR for base image updates..."
|
|
cd "$PROJECT_DIR"
|
|
|
|
# Get list of services that use a build context (not pre-built images)
|
|
build_services=$(docker compose config --services 2>/dev/null || true)
|
|
|
|
for svc in $build_services; do
|
|
# Get the base image from the Dockerfile
|
|
dockerfile=$(docker compose config | python3 -c "
|
|
import sys, yaml
|
|
cfg = yaml.safe_load(sys.stdin)
|
|
svc_cfg = cfg.get('services', {}).get('$svc', {})
|
|
build = svc_cfg.get('build', {})
|
|
if isinstance(build, str):
|
|
print(build + '/Dockerfile')
|
|
elif isinstance(build, dict):
|
|
ctx = build.get('context', '.')
|
|
df = build.get('dockerfile', 'Dockerfile')
|
|
print(ctx + '/' + df)
|
|
" 2>/dev/null || echo "")
|
|
|
|
if [ -z "$dockerfile" ] || [ ! -f "$dockerfile" ]; then
|
|
continue
|
|
fi
|
|
|
|
# Extract FROM image(s)
|
|
base_images=$(grep -i '^FROM ' "$dockerfile" | awk '{print $2}' | grep -v '^\$' | sort -u)
|
|
|
|
needs_rebuild=false
|
|
for img in $base_images; do
|
|
if [ "$img" = "scratch" ]; then continue; fi
|
|
|
|
# Pull and check if digest changed
|
|
old_digest=$(docker image inspect "$img" --format '{{ '{{' }}.Id{{ '}}' }}' 2>/dev/null || echo "none")
|
|
docker pull "$img" --quiet >/dev/null 2>&1 || continue
|
|
new_digest=$(docker image inspect "$img" --format '{{ '{{' }}.Id{{ '}}' }}' 2>/dev/null || echo "none2")
|
|
|
|
if [ "$old_digest" != "$new_digest" ]; then
|
|
log "UPDATE: Base image $img changed for service $svc"
|
|
needs_rebuild=true
|
|
fi
|
|
done
|
|
|
|
if $needs_rebuild; then
|
|
log "Rebuilding $svc in $PROJECT_DIR..."
|
|
if docker compose build --no-cache "$svc" 2>&1 | tail -5; then
|
|
docker compose up -d "$svc" 2>&1
|
|
updated_services+=("$svc ($PROJECT_DIR)")
|
|
log "OK: $svc rebuilt and restarted"
|
|
else
|
|
failed_services+=("$svc ($PROJECT_DIR)")
|
|
log "FAIL: $svc rebuild failed"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Also pull pre-built images (postgres, redis, mariadb, etc.)
|
|
pull_output=$(docker compose pull 2>&1 || true)
|
|
if echo "$pull_output" | grep -q "Pull complete"; then
|
|
log "Pre-built images updated in $PROJECT_DIR, restarting affected..."
|
|
docker compose up -d 2>&1 | tail -5
|
|
fi
|
|
done
|
|
|
|
# Clean up dangling images
|
|
docker image prune -f >/dev/null 2>&1 || true
|
|
|
|
# Report
|
|
if [ ${#updated_services[@]} -gt 0 ]; then
|
|
log "Updated services: ${updated_services[*]}"
|
|
$ALERT_SCRIPT "Container Security Update Complete" \
|
|
"The following services were rebuilt with updated base images:\n\n$(printf ' - %s\n' "${updated_services[@]}")\n\nAll services restarted successfully." \
|
|
"info"
|
|
fi
|
|
|
|
if [ ${#failed_services[@]} -gt 0 ]; then
|
|
log "FAILED services: ${failed_services[*]}"
|
|
$ALERT_SCRIPT "Container Rebuild FAILED — Manual Intervention Required" \
|
|
"The following services failed to rebuild after base image security updates:\n\n$(printf ' - %s\n' "${failed_services[@]}")\n\nPlease SSH in and investigate:\n ssh -p {{ ssh_port }} {{ deploy_user }}@$(hostname -f)\n cd <project_dir> && docker compose build <service> && docker compose up -d <service>" \
|
|
"critical"
|
|
fi
|
|
|
|
if [ ${#updated_services[@]} -eq 0 ] && [ ${#failed_services[@]} -eq 0 ]; then
|
|
log "All container base images are current. No updates needed."
|
|
fi
|