#!/bin/bash
# obf-lab.sh: eight broken-server exercises from opsbreakfix.com
# https://opsbreakfix.com/troubleshooting/linux-troubleshooting-practice-broken-servers/
#
# RUN THIS ONLY ON A DISPOSABLE TEST MACHINE (a VM or cloud server you can delete).
# It installs packages, creates users, units and a small loop filesystem, and
# deliberately breaks things. Every change is undone by "reset", but do not run
# it on a server you care about.
#
# Usage, as root:
#   obf-lab.sh prepare        check the machine and install everything the exercises use
#   obf-lab.sh list           show the exercises
#   obf-lab.sh break N        break the machine for exercise N (1-8)
#   obf-lab.sh check N        tell you whether exercise N is fixed
#   obf-lab.sh reset N        undo everything exercise N created
#   obf-lab.sh reset all      undo every exercise
#   obf-lab.sh solve N        stuck? apply the reference fix, printing each command
#   obf-lab.sh restore        put the machine back as it was before the first "break":
#                             reset all, disable the services this script enabled and
#                             remove the packages this script installed
#
# Works on Ubuntu/Debian (apt) and RHEL, AlmaLinux, Rocky (dnf), with systemd.
set -u
VERSION=2026-11-11.3
STATE=/var/lib/obf-lab

die() { echo "obf-lab: $*" >&2; exit 2; }
[ "$(id -u)" -eq 0 ] || die "run as root: sudo bash $0 $*"
command -v systemctl >/dev/null || die "needs systemd"
if command -v apt-get >/dev/null; then FAMILY=debian
elif command -v dnf >/dev/null; then FAMILY=rhel
else die "needs apt-get or dnf"; fi

pkglist() { if [ "$FAMILY" = debian ]; then dpkg-query -W -f '${db:Status-Abbrev} ${Package}\n' | awk '$1 ~ /^ii/ {print $2}' | sort; else rpm -qa --qf '%{NAME}\n' | sort -u; fi; }
snapshot() { # remember the installed packages once, before this script installs anything
  mkdir -p "$STATE"; [ -s "$STATE/packages.before" ] || pkglist > "$STATE/packages.before"
}
enable_now() { # enable and start a unit, remembering it if it was not enabled before
  mkdir -p "$STATE"
  systemctl is-enabled -q "$1" 2>/dev/null || echo "$1" >> "$STATE/units.enabled"
  systemctl enable --now "$1" >/dev/null 2>&1
}
pkg() { # install packages quietly if missing; names differ per family
  local deb="$1" rpm="$2"
  snapshot
  if [ "$FAMILY" = debian ]; then
    DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $deb >/dev/null 2>&1 || { apt-get update -qq >/dev/null 2>&1; DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $deb >/dev/null; }
  else
    dnf install -y -q $rpm >/dev/null
  fi
}
ok()   { echo "FIXED: $*"; exit 0; }
fail() { echo "NOT YET: $*"; exit 1; }
svc_ssh() { [ "$FAMILY" = debian ] && echo ssh || echo sshd; }
svc_cron() { [ "$FAMILY" = debian ] && echo cron || echo crond; }

list() { cat <<'EOF'
1  The website is down: nginx will not start
2  "Permission denied (publickey)" for the deploy user
3  The nightly report job runs, but no report appears
4  "No space left on device", but df shows free space
5  The server is slow, and nobody started anything
6  A worker service keeps restarting
7  The app service fails right after it starts
8  Backups stopped after the last reboot
EOF
}

# --- 1: port 80 already taken -------------------------------------------------
break1() {
  pkg "nginx curl" "nginx curl"
  systemctl stop nginx 2>/dev/null
  systemd-run --unit=obf-legacy-ui --quiet python3 -m http.server 80 --bind 0.0.0.0 --directory /var/empty 2>/dev/null \
    || systemd-run --unit=obf-legacy-ui --quiet python3 -m http.server 80 --bind 0.0.0.0
  sleep 1
  systemctl start nginx 2>/dev/null
  echo "Broken. Symptom: http://localhost/ does not show the nginx page, and nginx is not running."
}
check1() {
  systemctl is-active -q nginx || fail "nginx is not running"
  curl -sI --max-time 3 http://127.0.0.1/ | grep -qi '^server: nginx' || fail "port 80 does not answer with nginx"
  ok "nginx is running and answers on port 80"
}
reset1() { systemctl stop obf-legacy-ui 2>/dev/null; systemctl reset-failed obf-legacy-ui 2>/dev/null; systemctl stop nginx 2>/dev/null; true; }

# --- 2: home directory too open for sshd StrictModes ---------------------------
break2() {
  pkg "openssh-server openssh-client" "openssh-server openssh-clients"
  id deploy >/dev/null 2>&1 || useradd -m -s /bin/bash deploy
  install -d -m 700 -o deploy -g deploy /home/deploy/.ssh
  rm -f /root/obf-deploy-key /root/obf-deploy-key.pub
  ssh-keygen -q -t ed25519 -N '' -f /root/obf-deploy-key -C obf-lab
  install -m 600 -o deploy -g deploy /root/obf-deploy-key.pub /home/deploy/.ssh/authorized_keys
  enable_now "$(svc_ssh)"
  chmod 777 /home/deploy
  echo "Broken. Symptom: ssh -i /root/obf-deploy-key deploy@localhost says Permission denied (publickey)."
}
check2() {
  ssh -i /root/obf-deploy-key -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 deploy@127.0.0.1 true 2>/dev/null \
    || fail "key login as deploy still fails"
  ok "deploy can log in with the key"
}
reset2() { id deploy >/dev/null 2>&1 && userdel -r deploy 2>/dev/null; rm -f /root/obf-deploy-key /root/obf-deploy-key.pub; true; }

# --- 3: cron job script is not executable --------------------------------------
break3() {
  pkg cron cronie
  enable_now "$(svc_cron)"
  cat > /usr/local/bin/obf-report <<'S'
#!/bin/sh
date '+%F %T report generated' >> /var/tmp/obf-report.txt
S
  chmod 644 /usr/local/bin/obf-report
  echo '* * * * * root /usr/local/bin/obf-report' > /etc/cron.d/obf-report
  rm -f /var/tmp/obf-report.txt
  echo "Broken. Symptom: the job in /etc/cron.d/obf-report runs every minute, but /var/tmp/obf-report.txt never appears."
}
check3() {
  [ -f /var/tmp/obf-report.txt ] || fail "no report yet (cron runs once a minute, so wait a minute after the fix)"
  [ $(( $(date +%s) - $(stat -c %Y /var/tmp/obf-report.txt) )) -lt 150 ] || fail "the report exists but is not being updated"
  ok "the report is written every minute"
}
reset3() { rm -f /etc/cron.d/obf-report /usr/local/bin/obf-report /var/tmp/obf-report.txt; }

# --- 4: out of inodes on a small filesystem ------------------------------------
break4() {
  reset4
  pkg e2fsprogs e2fsprogs
  truncate -s 32M /var/tmp/obf-cache.img || die "could not create the image file"
  mkfs.ext4 -q -F -N 2048 /var/tmp/obf-cache.img || die "mkfs.ext4 failed; nothing was changed"
  mkdir -p /srv/obf-cache && mount -o loop /var/tmp/obf-cache.img /srv/obf-cache || die "could not mount the image; nothing was filled"
  # Safety: fill only the small loop filesystem, never the real disk, and never more than 3000 files.
  mountpoint -q /srv/obf-cache || die "/srv/obf-cache is not a mount point; refusing to fill it"
  mkdir -p /srv/obf-cache/sessions /srv/obf-cache/app
  i=0; while [ $i -lt 3000 ] && { : > /srv/obf-cache/sessions/sess_$i; } 2>/dev/null; do i=$((i+1)); done
  echo "Broken. Symptom: touch /srv/obf-cache/app/ready fails with No space left on device, while df -h shows the filesystem almost empty."
}
check4() {
  mountpoint -q /srv/obf-cache || fail "/srv/obf-cache is not mounted (run break 4 again)"
  touch /srv/obf-cache/app/ready 2>/dev/null || fail "still cannot create a file in /srv/obf-cache/app"
  ok "files can be created again"
}
reset4() { umount /srv/obf-cache 2>/dev/null; rmdir /srv/obf-cache 2>/dev/null; rm -f /var/tmp/obf-cache.img; true; }

# --- 5: CPU hog started by a transient unit ------------------------------------
break5() {
  local n; for n in 1 2; do systemd-run --unit=obf-indexer-$n --quiet sh -c 'while :; do :; done'; done
  echo "Broken. Symptom: the machine feels slow and the load average climbs. Find what is burning CPU and stop it properly."
}
check5() {
  local n; for n in 1 2; do systemctl is-active -q obf-indexer-$n && fail "something is still burning CPU"; done
  ok "the CPU hog is stopped"
}
reset5() { local n; for n in 1 2; do systemctl stop obf-indexer-$n 2>/dev/null; systemctl reset-failed obf-indexer-$n 2>/dev/null; done; true; }

# --- 6: memory limit kills the worker -------------------------------------------
break6() {
  cat > /etc/systemd/system/obf-worker.service <<'U'
[Unit]
Description=obf-lab worker (loads 64 MB of data, then waits)

[Service]
ExecStart=/usr/bin/python3 -c "import time; data = bytearray(64 * 1024 * 1024); print('worker: data loaded', flush=True); time.sleep(100000)"
MemoryMax=32M
MemorySwapMax=0
Restart=on-failure
RestartSec=3
U
  systemctl daemon-reload && systemctl start obf-worker
  echo "Broken. Symptom: systemctl status obf-worker shows it restarting again and again. Find out why, and fix the cause."
}
check6() {
  systemctl is-active -q obf-worker || fail "obf-worker is not running"
  journalctl -u obf-worker --since "-20s" --no-pager -q 2>/dev/null | grep -q "worker: data loaded" || true
  sleep 5
  [ "$(systemctl show -p NRestarts --value obf-worker)" = "$(sleep 4; systemctl show -p NRestarts --value obf-worker)" ] || fail "obf-worker is still restarting"
  ok "obf-worker stays up"
}
reset6() { systemctl stop obf-worker 2>/dev/null; rm -rf /etc/systemd/system/obf-worker.service /etc/systemd/system/obf-worker.service.d; systemctl daemon-reload; systemctl reset-failed obf-worker 2>/dev/null; true; }

# --- 7: service user cannot write its log file ----------------------------------
break7() {
  id obfapp >/dev/null 2>&1 || useradd -r -s /usr/sbin/nologin obfapp
  install -d -m 755 -o root -g root /var/log/obf-app
  install -m 644 -o root -g root /dev/null /var/log/obf-app/app.log
  cat > /etc/systemd/system/obf-app.service <<'U'
[Unit]
Description=obf-lab app (writes a line to its log every 5 seconds)

[Service]
User=obfapp
ExecStart=/bin/sh -c 'while :; do date "+%%F %%T app alive" >> /var/log/obf-app/app.log || exit 1; sleep 5; done'
U
  systemctl daemon-reload && systemctl start obf-app
  echo "Broken. Symptom: systemctl start obf-app returns, but the service is failed a moment later."
}
check7() {
  systemctl is-active -q obf-app || fail "obf-app is not running"
  sleep 6; tail -n 1 /var/log/obf-app/app.log 2>/dev/null | grep -q "app alive" || fail "obf-app runs but does not write its log"
  ok "obf-app runs and writes its log"
}
reset7() { systemctl stop obf-app 2>/dev/null; rm -f /etc/systemd/system/obf-app.service; systemctl daemon-reload; systemctl reset-failed obf-app 2>/dev/null; rm -rf /var/log/obf-app; id obfapp >/dev/null 2>&1 && userdel obfapp; true; }

# --- 8: backup timer started by hand, never enabled ------------------------------
break8() {
  cat > /etc/systemd/system/obf-backup.service <<'U'
[Unit]
Description=obf-lab backup (writes a timestamp)

[Service]
Type=oneshot
ExecStart=/bin/sh -c 'date "+%%F %%T backup done" >> /var/tmp/obf-backup.log'
U
  cat > /etc/systemd/system/obf-backup.timer <<'U'
[Unit]
Description=obf-lab backup every 2 minutes

[Timer]
OnCalendar=*:0/2
Persistent=true

[Install]
WantedBy=timers.target
U
  systemctl daemon-reload
  systemctl disable obf-backup.timer >/dev/null 2>&1
  systemctl stop obf-backup.timer 2>/dev/null
  echo "Broken. Symptom: the last line in /var/tmp/obf-backup.log is from before the last reboot (here: there is none). The timer exists; why doesn't it run?"
}
check8() {
  systemctl is-enabled -q obf-backup.timer || fail "obf-backup.timer will not start after the next reboot"
  systemctl is-active -q obf-backup.timer || fail "obf-backup.timer is not running now"
  ok "the backup timer is enabled and running"
}
reset8() { systemctl disable --now obf-backup.timer >/dev/null 2>&1; rm -f /etc/systemd/system/obf-backup.service /etc/systemd/system/obf-backup.timer /var/tmp/obf-backup.log; systemctl daemon-reload; true; }

# --- solve: the reference fixes, printed and applied ------------------------------
say() { echo "+ $*"; "$@"; }
solve1() { say systemctl stop obf-legacy-ui; say systemctl start nginx; }
solve2() { say chmod 755 /home/deploy; }
solve3() { say chmod 755 /usr/local/bin/obf-report; echo "cron runs the job once a minute: run 'check 3' again in a minute."; }
solve4() { say find /srv/obf-cache/sessions -type f -delete; }
solve5() { say systemctl stop obf-indexer-1 obf-indexer-2; }
solve6() { say mkdir -p /etc/systemd/system/obf-worker.service.d
  echo "+ printf '[Service]\\nMemoryMax=128M\\n' > /etc/systemd/system/obf-worker.service.d/memory.conf"
  printf '[Service]\nMemoryMax=128M\n' > /etc/systemd/system/obf-worker.service.d/memory.conf
  say systemctl daemon-reload; say systemctl restart obf-worker; }
solve7() { say chown obfapp: /var/log/obf-app/app.log; say systemctl restart obf-app; }
solve8() { say systemctl enable --now obf-backup.timer; }

restore() {
  local k; for k in 1 2 3 4 5 6 7 8; do reset$k; done
  echo "All exercises reset."
  if [ -s "$STATE/units.enabled" ]; then
    for k in $(sort -u "$STATE/units.enabled"); do echo "Disabling $k (it was not enabled before obf-lab)"; systemctl disable --now "$k" >/dev/null 2>&1; done
  fi
  if [ -s "$STATE/packages.before" ]; then
    local extra; extra=$(pkglist | comm -13 "$STATE/packages.before" -)
    if [ -n "$extra" ]; then
      echo "Removing the packages installed since the first break:" $extra
      if [ "$FAMILY" = debian ]; then DEBIAN_FRONTEND=noninteractive apt-get purge -y -qq $extra >/dev/null; else dnf remove -y -q $extra >/dev/null; fi
    fi
  fi
  rm -rf "$STATE"
  echo "Restored: the machine is back to how it was before obf-lab."
}

# --- prepare: pre-flight checks and every package the exercises and their diagnosis use ---
prepare() {
  local bad=0 freemb memmb
  echo "obf-lab.sh $VERSION: preparing a $FAMILY machine"
  if [ -d /run/systemd/system ]; then echo "ok    systemd is running (PID 1: $(ps -p 1 -o comm= 2>/dev/null || cat /proc/1/comm))"
  else echo "FAIL  systemd is not running as PID 1. A normal Docker container will not work; use a VM, WSL2 with systemd, or a cloud server."; bad=1; fi
  freemb=$(df -Pm / | awk 'NR==2 {print $4}')
  if [ "${freemb:-0}" -ge 1024 ]; then echo "ok    ${freemb} MB free on /"; else echo "FAIL  only ${freemb} MB free on /, the exercises need about 1 GB"; bad=1; fi
  memmb=$(awk '/^MemTotal/ {print int($2/1024)}' /proc/meminfo)
  if [ "${memmb:-0}" -ge 700 ]; then echo "ok    ${memmb} MB of RAM"; else echo "FAIL  ${memmb} MB of RAM, use at least 1 GB"; bad=1; fi
  [ "$bad" = 0 ] || die "fix the FAIL lines above, then run prepare again"
  echo "...   installing packages (a minute or two the first time)"
  pkg "nginx curl python3 openssh-server openssh-client cron e2fsprogs iproute2 procps less"       "nginx curl python3 openssh-server openssh-clients cronie e2fsprogs iproute procps-ng less" || die "package install failed: check the internet connection and the package mirrors, then run prepare again"
  # Installing nginx starts it on Debian/Ubuntu; leave the machine as quiet as before.
  systemctl is-enabled -q nginx 2>/dev/null && ! grep -qx nginx "$STATE/packages.before" && { systemctl disable --now nginx >/dev/null 2>&1; }
  local c miss=""
  for c in curl python3 ss ps journalctl ssh sshd crontab mkfs.ext4 less; do command -v $c >/dev/null 2>&1 || [ -x /usr/sbin/$c ] || miss="$miss $c"; done
  [ -z "$miss" ] || die "still missing:$miss"
  echo "ok    tools: curl python3 ss ps journalctl ssh sshd crontab mkfs.ext4 less"
  echo "READY: start with 'sudo bash $0 break 1'. When you are done, 'sudo bash $0 restore' removes all of it."
}

case "${1:-}" in
  list) list ;;
  prepare) prepare ;;
  restore) restore ;;
  break|check|reset|solve)
    n="${2:-}"
    if [ "$1" = reset ] && [ "$n" = all ]; then for k in 1 2 3 4 5 6 7 8; do reset$k; done; echo "All exercises reset."; exit 0; fi
    case "$n" in [1-8]) ;; *) die "give an exercise number, 1 to 8 (see: $0 list)";; esac
    if [ "$1" = reset ]; then reset$n; echo "Exercise $n reset."
    elif [ "$1" = solve ]; then solve$n; [ "$n" = 3 ] || { sleep 2; check$n; }
    else $1$n; fi ;;
  *) echo "obf-lab.sh $VERSION ($FAMILY)"; echo "Usage: $0 prepare | list | break N | check N | solve N | reset N | reset all | restore"; exit 2 ;;
esac
