#!/usr/bin/env bash
# shellcheck disable=SC1090  # env file path is a variable
# hostrepo-mirror <PEER>
# Keep an encrypted copy of a peer host's restic repository on this host:
#   B2 (b2:<bucket>:<PEER>)  ->  /hostrepo/mirror/<PEER>/   (outside every backed-up tree)
#
# Copy only, never delete: the mirror is a superset of what B2 has ever held,
# so it survives a deleted or maliciously pruned source. It can't be read
# without the peer's restic password, which this host never has.
# Verification uses B2's stored SHA-1s against local files: no download.
# Bit rot on this disk is found and repaired by hostrepo-heal: every file is
# hashed against its name each run, repaired from par2 sets kept in
# /hostrepo/par2/<PEER>, or refetched from B2. Damaged copies go to
# /hostrepo/quarantine/<PEER>.
# Env (/etc/hostrepo/mirror.env): MIRROR_B2_ACCOUNT, MIRROR_B2_KEY (read-only,
# ideally limited to <PEER>/), HOSTREPO_B2_BUCKET, NTFY_URL. Exit 0/1/2.
set -euo pipefail
export LC_ALL=C
PEER="${1:?usage: hostrepo-mirror <PEER>}"
ENV_FILE="${HOSTREPO_MIRROR_ENV:-/etc/hostrepo/mirror.env}"
# shellcheck source=/dev/null
set -a; . "$ENV_FILE"; set +a
RCLONE="${RCLONE_BIN:-$(command -v rclone)}"
export RCLONE_CONFIG=/dev/null RCLONE_CONFIG_SRC_TYPE=b2 RCLONE_CONFIG_SRC_ACCOUNT="$MIRROR_B2_ACCOUNT" RCLONE_CONFIG_SRC_KEY="$MIRROR_B2_KEY"
DST="/hostrepo/mirror/$PEER"
SRC="src:$HOSTREPO_B2_BUCKET/$PEER"
log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >&2; }
alert() { # level message
  [[ -n "${NTFY_URL:-}" ]] && curl -s -m 20 -H "Title: hostrepo mirror of $PEER on $(hostname)" \
    -H "Priority: $([[ $1 -ge 2 ]] && echo high || echo default)" -d "$2" "$NTFY_URL" >/dev/null || true
}
HEAL="$(dirname "$0")/hostrepo-heal"
# One status record per run, stored under the peer whose copy this is, also
# when the run stops early. Counts and sizes only.
started=$(date -u +%s)
count() { local n; n=$(grep -c -- "$1" "$2" 2>/dev/null); echo "${n:-0}"; }
record() {
  local rc=$? holder; holder=$(. "${HOSTREPO_CONF:-/etc/hostrepo/host.conf}" 2>/dev/null && echo "${HOST_ID:-}")
  jq -n --arg holder "${holder:-unknown}" --argjson rc "$rc" --argjson started "$started" --argjson ended "$(date -u +%s)" \
    --argjson bytes "$(du -sb "$DST" 2>/dev/null | cut -f1 || echo 0)" --argjson before "${before:-0}" \
    --argjson files "$(find "$DST" -type f 2>/dev/null | wc -l)" \
    --argjson sets "$(find "$PAR" -name '*.sums' 2>/dev/null | wc -l)" \
    --argjson repaired "$(count 'REPAIRED:' "$DST.heal-log")" \
    --argjson unrepaired "$(count 'NOT REPAIRED:' "$DST.heal-log")" \
    '{kind: "mirror", holder: $holder, result: $rc, started: ($started | todate), ended: ($ended | todate),
      duration_s: ($ended - $started), bytes: $bytes, new_bytes: ($bytes - $before), files: $files,
      par2_sets: $sets, heal: {repaired: ($repaired - $unrepaired), unrepaired: $unrepaired}}' 2>/dev/null |
    "$(dirname "$0")/hostrepo-status" put "$PEER" mirror || true
}
trap record EXIT
PAR="/hostrepo/par2/$PEER"; QUAR="/hostrepo/quarantine/$PEER"
STATUS=0
install -d -m 0700 /hostrepo /hostrepo/mirror /hostrepo/par2 /hostrepo/quarantine "$DST"
before=$(du -sb "$DST" | cut -f1)
heal() { # extra args; records the worst exit
  local rc=0
  RCLONE_BIN="$RCLONE" "$HEAL" local "$DST" "$PAR" --remote "$SRC" --quarantine "$QUAR" "$@" 2>>"$DST.heal-log" || rc=$?
  [[ $rc -gt $STATUS ]] && STATUS=$rc
  return 0
}
: > "$DST.heal-log"
# 1. Repair local damage first: a damaged file would make the copy refuse.
heal
# 2. --immutable: an existing file that changed on B2 is an error, never an
# overwrite (restic never rewrites a file in place, so that would mean
# tampering). --checksum compares content, not timestamps: a server-side copy
# on B2 changes an object's timestamp without changing a byte.
if ! "$RCLONE" copy --checksum --immutable --transfers 8 --checkers 16 -q "$SRC" "$DST" 2>"$DST.last-error"; then
  msg="copy failed: $(head -c 300 "$DST.last-error")"; log "$msg"; alert 2 "$msg"; exit 2
fi
rm -f "$DST.last-error"
# 3. Parity for the new files, and a rotating seventh of the sets checked
# against their own sums, so every set is checked once a week.
heal --create --par2-verify "$(( $(date -u +%j | sed 's/^0*//') % 7 + 1 ))/7"
if [[ $STATUS -ge 1 ]]; then
  msg="bit rot: $(grep -cE 'REPAIRED|NOT REPAIRED' "$DST.heal-log" || true) findings; $(grep -m1 -E 'NOT REPAIRED|REPAIRED' "$DST.heal-log" | cut -c22-220)"
  log "$msg"; alert "$STATUS" "$msg"
fi
# Every file on B2 must be present here with the same SHA-1 (no download).
if ! "$RCLONE" check --one-way --checksum -q "$SRC" "$DST" 2>"$DST.check-error"; then
  msg="check failed: $(grep -c ERROR "$DST.check-error" || true) differences, e.g. $(grep -m1 ERROR "$DST.check-error" | cut -c1-200)"
  log "$msg"; alert 2 "$msg"; exit 2
fi
rm -f "$DST.check-error"
# 4. The nightly lite drill of the peer's backups: every copy (the bucket by
# listing, this mirror by hashing, the EU replica if its key is in mirror.env)
# against the peer's recorded SHA-1s. No data is downloaded.
ld=$(mktemp); drc=0
RCLONE_BIN="$RCLONE" "$HEAL" lite-drill "$PEER" --mirror "$DST" --json "$ld" 2>>"$DST.heal-log" || drc=$?
[[ -s "$ld" ]] && "$(dirname "$0")/hostrepo-status" put "$PEER" litedrill "$ld"
rm -f "$ld"
if [[ $drc -ge 2 ]]; then
  msg="lite drill of $PEER failed: $(grep -m1 -E 'MISMATCH|lite-drill' "$DST.heal-log" | cut -c1-200)"; log "$msg"; alert 2 "$msg"
fi
[[ $drc -gt $STATUS ]] && STATUS=$drc
after=$(du -sb "$DST" | cut -f1)
log "mirror of $PEER: $(numfmt --to=iec "$after") ($(numfmt --to=iec $((after - before))) new), all B2 files present and matching, heal status $STATUS"
exit "$STATUS"
