konnosbriven-backup-alert.sh67 lines · main
| 1 | #!/usr/bin/env bash |
| 2 | # Posts a Discord alert when briven-backup.service exits non-zero. Invoked |
| 3 | # by systemd's OnFailure= hook on briven-backup.service; see |
| 4 | # road-to-ga.md §0.1 + §0.2 for the contract. |
| 5 | # |
| 6 | # Reads $BRIVEN_DISCORD_WEBHOOK_ALERTS from /etc/briven/backup.env (via |
| 7 | # EnvironmentFile= in the .service unit). Reads /run/briven-backup-status |
| 8 | # for the failure detail written by briven-backup.sh. |
| 9 | # |
| 10 | # Behaviour: |
| 11 | # - webhook unset → log "no webhook configured" and exit 0 (silent |
| 12 | # skip; OnFailure already counted the parent unit as failed) |
| 13 | # - webhook set → POST a JSON payload with the failing DBs, exit 0 |
| 14 | # even on curl failure so systemd doesn't cascade alerts on the alert |
| 15 | |
| 16 | set -uo pipefail |
| 17 | |
| 18 | HOSTNAME_SHORT="$(hostname -s 2>/dev/null || echo unknown)" |
| 19 | STATUS_FILE="/run/briven-backup-status" |
| 20 | |
| 21 | log() { |
| 22 | printf '[%s] %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*" |
| 23 | } |
| 24 | |
| 25 | if [ -z "${BRIVEN_DISCORD_WEBHOOK_ALERTS:-}" ]; then |
| 26 | log "no BRIVEN_DISCORD_WEBHOOK_ALERTS configured — skipping Discord alert" |
| 27 | exit 0 |
| 28 | fi |
| 29 | |
| 30 | upload_failures="unknown" |
| 31 | upload_failure_dbs="unknown" |
| 32 | if [ -r "$STATUS_FILE" ]; then |
| 33 | # shellcheck disable=SC1090 |
| 34 | source "$STATUS_FILE" |
| 35 | fi |
| 36 | |
| 37 | # Discord embed payload. Title makes it visually red, description names |
| 38 | # the failing DBs. We deliberately keep this short — no recipient PII |
| 39 | # (none of these are user-bound) but also no oversharing of bucket / |
| 40 | # credentials. The journalctl tail is the source of truth for ops; this |
| 41 | # message just routes attention. |
| 42 | PAYLOAD=$(cat <<JSON |
| 43 | { |
| 44 | "username": "briven-backup", |
| 45 | "embeds": [{ |
| 46 | "title": "briven backup: off-site upload failed", |
| 47 | "color": 15158332, |
| 48 | "description": "Host: \`${HOSTNAME_SHORT}\`\nFailed uploads: \`${upload_failures}\`\nDatabases: \`${upload_failure_dbs}\`\n\nLocal dumps are safe. Investigate with:\n\`\`\`\njournalctl -u briven-backup.service -n 200\n\`\`\`" |
| 49 | }] |
| 50 | } |
| 51 | JSON |
| 52 | ) |
| 53 | |
| 54 | # curl: --silent --show-error to keep journal clean on success but |
| 55 | # capture errors. --fail to treat 4xx/5xx as failure. Timeout caps tail |
| 56 | # latency at 10s so OnFailure's cascade can't stall a reboot. |
| 57 | if ! curl --silent --show-error --fail \ |
| 58 | --max-time 10 \ |
| 59 | -H 'content-type: application/json' \ |
| 60 | -d "$PAYLOAD" \ |
| 61 | "$BRIVEN_DISCORD_WEBHOOK_ALERTS" >/dev/null; then |
| 62 | log "Discord webhook POST failed — alert was not delivered" |
| 63 | fi |
| 64 | |
| 65 | # Always exit 0 — the parent unit's failure status is what matters. |
| 66 | # A failure here just means the operator finds out via journal instead. |
| 67 | exit 0 |