new-site/infra/ansible/roles/security-updates/templates/container-security-update.sh.j2
justin 97e8664cbf Add security-updates Ansible role for automated patching
Comprehensive security update automation:

1. Debian OS (unattended-upgrades) — tightened to security-only:
   - Removed general Debian updates (prevents feature/breaking changes)
   - Only Debian-Security origins auto-installed
   - Email admin on every upgrade via ops@performancewest.net
   - Auto-reboot at 4 AM if kernel update requires it
   - needrestart auto-restarts services after library updates

2. Docker CE — major version guard:
   - Patch updates within pinned major version auto-applied
   - Major version jumps held + admin alerted for manual review
   - docker-ce, docker-ce-cli, containerd.io all version-guarded

3. Container base images — daily at 3:30 AM:
   - Pulls latest base images for all docker-compose services
   - Compares image digests — only rebuilds if changed
   - Restarts only affected services (not full stack)
   - Alerts admin on rebuild failures requiring manual intervention
   - Covers both prod and dev compose projects

4. k3s — weekly Sunday at 3:45 AM:
   - Patch updates within current minor auto-applied
   - Minor/major upgrades alert admin for manual review
   - Verifies node Ready status after update
   - Alerts on failures with investigation instructions

5. Admin notifications via SMTP:
   - [INFO] for successful patches
   - [WARNING] for available major upgrades needing review
   - [CRITICAL] for failures requiring immediate intervention
   - Falls back to syslog if SMTP unavailable

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-30 01:24:57 -05:00

112 lines
4.7 KiB
Django/Jinja

#!/usr/bin/env bash
# ══════════════════════════════════════════════════════════════════════════════
# Container Security Update — Performance West
#
# Pulls latest base images for all docker-compose services. If any image
# has changed (security patches in upstream base), rebuilds and restarts
# only the affected services. Sends admin alert if a rebuild fails.
#
# Runs via systemd timer (pw-container-update.timer)
# ══════════════════════════════════════════════════════════════════════════════
set -euo pipefail
LOG_TAG="pw-container-update"
ALERT_SCRIPT="/usr/local/bin/pw-security-alert"
COMPOSE_PROJECTS=({{ compose_projects | map('quote') | join(' ') }})
log() { logger -t "$LOG_TAG" "$*"; echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
updated_services=()
failed_services=()
for PROJECT_DIR in "${COMPOSE_PROJECTS[@]}"; do
if [ ! -f "$PROJECT_DIR/docker-compose.yml" ]; then
log "SKIP: No docker-compose.yml in $PROJECT_DIR"
continue
fi
log "Checking $PROJECT_DIR for base image updates..."
cd "$PROJECT_DIR"
# Get list of services that use a build context (not pre-built images)
build_services=$(docker compose config --services 2>/dev/null || true)
for svc in $build_services; do
# Get the base image from the Dockerfile
dockerfile=$(docker compose config | python3 -c "
import sys, yaml
cfg = yaml.safe_load(sys.stdin)
svc_cfg = cfg.get('services', {}).get('$svc', {})
build = svc_cfg.get('build', {})
if isinstance(build, str):
print(build + '/Dockerfile')
elif isinstance(build, dict):
ctx = build.get('context', '.')
df = build.get('dockerfile', 'Dockerfile')
print(ctx + '/' + df)
" 2>/dev/null || echo "")
if [ -z "$dockerfile" ] || [ ! -f "$dockerfile" ]; then
continue
fi
# Extract FROM image(s)
base_images=$(grep -i '^FROM ' "$dockerfile" | awk '{print $2}' | grep -v '^\$' | sort -u)
needs_rebuild=false
for img in $base_images; do
if [ "$img" = "scratch" ]; then continue; fi
# Pull and check if digest changed
old_digest=$(docker image inspect "$img" --format '{{ '{{' }}.Id{{ '}}' }}' 2>/dev/null || echo "none")
docker pull "$img" --quiet >/dev/null 2>&1 || continue
new_digest=$(docker image inspect "$img" --format '{{ '{{' }}.Id{{ '}}' }}' 2>/dev/null || echo "none2")
if [ "$old_digest" != "$new_digest" ]; then
log "UPDATE: Base image $img changed for service $svc"
needs_rebuild=true
fi
done
if $needs_rebuild; then
log "Rebuilding $svc in $PROJECT_DIR..."
if docker compose build --no-cache "$svc" 2>&1 | tail -5; then
docker compose up -d "$svc" 2>&1
updated_services+=("$svc ($PROJECT_DIR)")
log "OK: $svc rebuilt and restarted"
else
failed_services+=("$svc ($PROJECT_DIR)")
log "FAIL: $svc rebuild failed"
fi
fi
done
# Also pull pre-built images (postgres, redis, mariadb, etc.)
pull_output=$(docker compose pull 2>&1 || true)
if echo "$pull_output" | grep -q "Pull complete"; then
log "Pre-built images updated in $PROJECT_DIR, restarting affected..."
docker compose up -d 2>&1 | tail -5
fi
done
# Clean up dangling images
docker image prune -f >/dev/null 2>&1 || true
# Report
if [ ${#updated_services[@]} -gt 0 ]; then
log "Updated services: ${updated_services[*]}"
$ALERT_SCRIPT "Container Security Update Complete" \
"The following services were rebuilt with updated base images:\n\n$(printf ' - %s\n' "${updated_services[@]}")\n\nAll services restarted successfully." \
"info"
fi
if [ ${#failed_services[@]} -gt 0 ]; then
log "FAILED services: ${failed_services[*]}"
$ALERT_SCRIPT "Container Rebuild FAILED — Manual Intervention Required" \
"The following services failed to rebuild after base image security updates:\n\n$(printf ' - %s\n' "${failed_services[@]}")\n\nPlease SSH in and investigate:\n ssh -p {{ ssh_port }} {{ deploy_user }}@$(hostname -f)\n cd <project_dir> && docker compose build <service> && docker compose up -d <service>" \
"critical"
fi
if [ ${#updated_services[@]} -eq 0 ] && [ ${#failed_services[@]} -eq 0 ]; then
log "All container base images are current. No updates needed."
fi