panel/agent/cmd/region-medic/autoheal.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/dbledeez/panel/pkg/regionmedic"
)

// region-medic autoheal — the auto-on-reboot flow, designed to run (in a sidecar)
// while the 7DTD server is STOPPED, just before the agent starts it:
//
//  1. Validate EVERY .7rg in the active world's Region dir (catches whole-file
//     corruption like a bad magic header, which has no error_backups and so is
//     invisible to the error_backup-cluster scan).
//  2. Heal each corrupt region from the newest source that has a validated-clean
//     copy: the rolling folder snapshots first (fast, local), then the panel's
//     tar backups as a fallback. Corrupt originals are quarantined first.
//  3. Snapshot the now-clean Region dir to a fresh rolling slot (real folder
//     copy — instant restore, no unzip) and rotate to keep the newest -keep.
//
// Emits a JSON report the agent relays to the controller for the Discord ping.
//
//	region-medic autoheal -region-dir D -rolling-dir R [-backups-dir B] [-keep 2] [-apply]
//
// Without -apply it only validates + reports (dry run: no heal, no snapshot).

type ahHeal struct {
	Region     string `json:"region"`      // r.X.Z
	Source     string `json:"source"`      // "rolling:<slot>" | "backup:<id>"
	SourceTime string `json:"source_time"` // RFC3339 of the snapshot/backup
}

type ahReport struct {
	RegionDir    string   `json:"region_dir"`
	TotalRegions int      `json:"total_regions"`
	CorruptFound int      `json:"corrupt_found"`
	Healed       []ahHeal `json:"healed"`
	Unhealable   []string `json:"unhealable"`
	SnapshotSlot string   `json:"snapshot_slot"`
	RollingKept  int      `json:"rolling_kept"`
	Applied      bool     `json:"applied"`
	DurationMs   int64    `json:"duration_ms"`
}

func cmdAutoheal(args []string) error {
	var regionDir, rollingDir, backupsDir, savesRoot, discordChannel, discordTokenFile, label string
	keep := 2
	apply := false
	for i := 0; i < len(args); i++ {
		switch args[i] {
		case "-region-dir":
			i++
			if i < len(args) {
				regionDir = args[i]
			}
		case "-saves-root":
			i++
			if i < len(args) {
				savesRoot = args[i]
			}
		case "-rolling-dir":
			i++
			if i < len(args) {
				rollingDir = args[i]
			}
		case "-backups-dir":
			i++
			if i < len(args) {
				backupsDir = args[i]
			}
		case "-keep":
			i++
			if i < len(args) {
				keep, _ = strconv.Atoi(args[i])
			}
		case "-discord-channel":
			i++
			if i < len(args) {
				discordChannel = args[i]
			}
		case "-discord-token-file":
			i++
			if i < len(args) {
				discordTokenFile = args[i]
			}
		case "-label":
			i++
			if i < len(args) {
				label = args[i]
			}
		case "-apply", "--apply":
			apply = true
		default:
			return fmt.Errorf("unknown flag %q", args[i])
		}
	}
	// -saves-root auto-resolves the active world's Region dir (so the agent
	// doesn't have to know GameWorld/GameName, and RWG seed-named worlds work).
	// Per-server config lives in the saves volume (panel writes it via FsWrite,
	// no recreate): world pin + keep + Discord channel; enabled:false skips.
	var cfgWorld string
	if savesRoot != "" {
		if c, ok := readSavesMedicConfig(savesRoot); ok {
			if c.Enabled != nil && !*c.Enabled {
				fmt.Fprintln(os.Stderr, "autoheal: disabled via region-medic.json, skipping")
				return nil
			}
			if c.Keep > 0 {
				keep = c.Keep
			}
			if c.DiscordChannel != "" {
				discordChannel = c.DiscordChannel
			}
			cfgWorld = strings.TrimSpace(c.World)
		}
	}
	// Resolve the Region dir. An explicit world pin wins over auto-detect — some
	// servers carry multiple saves, so the operator picks which one to heal. If
	// the pinned world has no Region dir we SKIP rather than silently auto-pick a
	// different world the operator didn't choose.
	if regionDir == "" && savesRoot != "" {
		if cfgWorld != "" {
			wd := filepath.Join(savesRoot, ".local/share/7DaysToDie/Saves", cfgWorld, "Region")
			if st, err := os.Stat(wd); err != nil || !st.IsDir() {
				fmt.Fprintf(os.Stderr, "autoheal: pinned world %q has no Region dir at %s — skipping\n", cfgWorld, wd)
				return nil
			}
			regionDir = wd
		} else {
			rd, rerr := resolveRegionDir(savesRoot)
			if rerr != nil {
				return rerr
			}
			regionDir = rd
		}
	}
	if regionDir == "" || rollingDir == "" {
		return fmt.Errorf("required: -region-dir (or -saves-root) and -rolling-dir (and -backups-dir for tar fallback)")
	}
	if keep < 1 {
		keep = 1
	}
	start := time.Now()
	rep := ahReport{RegionDir: regionDir, RollingKept: keep, Applied: apply, Healed: []ahHeal{}, Unhealable: []string{}}

	// 1. Validate every .7rg.
	ents, err := os.ReadDir(regionDir)
	if err != nil {
		return fmt.Errorf("read region dir: %w", err)
	}
	var corrupt []regionmedic.RegionCoord
	for _, e := range ents {
		if e.IsDir() {
			continue
		}
		rc, ok := regionmedic.ParseRegionFileName(e.Name())
		if !ok {
			continue
		}
		rep.TotalRegions++
		if !regionmedic.ValidateRegionFile(filepath.Join(regionDir, e.Name()), false).Healthy() {
			corrupt = append(corrupt, rc)
		}
	}
	rep.CorruptFound = len(corrupt)

	// 2. Heal (only with -apply).
	quarantineDir := filepath.Join(rollingDir, "_quarantine")
	if apply && len(corrupt) > 0 {
		slots := listRollingSlots(rollingDir) // newest first
		var panelBackups []regionmedic.BackupRef
		if backupsDir != "" {
			panelBackups, _ = regionmedic.ListPanelBackups(backupsDir)
		}
		stamp := start.UTC().Format("20060102-150405")
		for _, rc := range corrupt {
			fn := rc.FileName()
			done := false
			// 2a. newest rolling snapshot with a clean copy.
			for _, s := range slots {
				cand := filepath.Join(s.path, fn)
				if fi, e := os.Stat(cand); e == nil && !fi.IsDir() &&
					regionmedic.ValidateRegionFile(cand, false).Healthy() {
					if e := healFromFile(regionDir, fn, cand, quarantineDir, stamp); e == nil {
						rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "rolling:" + filepath.Base(s.path), SourceTime: s.t.UTC().Format(time.RFC3339)})
						done = true
						break
					}
				}
			}
			// 2b. panel tar backups fallback.
			if !done && len(panelBackups) > 0 {
				entry := regionmedic.SaveRelEntry(regionDir, fn)
				best, _, _ := regionmedic.FindLastGoodRegion(panelBackups, entry, false, time.Time{})
				if best.Found && best.Clean() {
					if e := healFromBytes(regionDir, fn, best.Bytes, quarantineDir, stamp); e == nil {
						rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "backup:" + best.Backup.ID, SourceTime: best.Backup.CreatedAt.UTC().Format(time.RFC3339)})
						done = true
					}
				}
			}
			if !done {
				rep.Unhealable = append(rep.Unhealable, rc.String())
			}
		}
	}

	// 3. Snapshot the now-clean Region dir to a fresh rolling slot + rotate.
	if apply {
		slotName := fmt.Sprintf("slot-%d", start.Unix())
		if err := snapshotRegions(regionDir, filepath.Join(rollingDir, slotName)); err != nil {
			rep.SnapshotSlot = "ERROR: " + err.Error()
		} else {
			rep.SnapshotSlot = slotName
			rotateRollingSlots(rollingDir, keep)
		}
	}

	rep.DurationMs = time.Since(start).Milliseconds()
	b, _ := json.MarshalIndent(rep, "", "  ")
	fmt.Println(string(b))
	fmt.Fprintf(os.Stderr, "autoheal: %d regions, %d corrupt, %d healed, %d unhealable, snapshot=%s (%dms)\n",
		rep.TotalRegions, rep.CorruptFound, len(rep.Healed), len(rep.Unhealable), rep.SnapshotSlot, rep.DurationMs)
	// Discord alert — only when we actually healed something on this reboot.
	if apply {
		if derr := postDiscordHealReport(discordTokenFile, discordChannel, label, rep); derr != nil {
			fmt.Fprintf(os.Stderr, "discord post failed: %v\n", derr)
		} else if discordChannel != "" && len(rep.Healed) > 0 {
			fmt.Fprintf(os.Stderr, "discord: posted heal embed to channel %s\n", discordChannel)
		}
	}
	return nil
}

type rollingSlot struct {
	path string
	t    time.Time
}

// listRollingSlots returns slot dirs named "slot-<unix>" newest first.
func listRollingSlots(dir string) []rollingSlot {
	ents, err := os.ReadDir(dir)
	if err != nil {
		return nil
	}
	var out []rollingSlot
	for _, e := range ents {
		if !e.IsDir() || !strings.HasPrefix(e.Name(), "slot-") {
			continue
		}
		secs, err := strconv.ParseInt(strings.TrimPrefix(e.Name(), "slot-"), 10, 64)
		if err != nil {
			continue
		}
		out = append(out, rollingSlot{path: filepath.Join(dir, e.Name()), t: time.Unix(secs, 0)})
	}
	sort.Slice(out, func(i, j int) bool { return out[i].t.After(out[j].t) })
	return out
}

// rotateRollingSlots keeps the newest `keep` slot dirs, removing older ones.
func rotateRollingSlots(dir string, keep int) {
	for i, s := range listRollingSlots(dir) {
		if i >= keep {
			_ = os.RemoveAll(s.path)
		}
	}
}

// snapshotRegions copies every *.7rg from regionDir into slotPath.
func snapshotRegions(regionDir, slotPath string) error {
	if err := os.MkdirAll(slotPath, 0o755); err != nil {
		return err
	}
	ents, err := os.ReadDir(regionDir)
	if err != nil {
		return err
	}
	for _, e := range ents {
		if e.IsDir() || !strings.HasSuffix(e.Name(), ".7rg") {
			continue
		}
		if err := copyFile(filepath.Join(regionDir, e.Name()), filepath.Join(slotPath, e.Name())); err != nil {
			return fmt.Errorf("copy %s: %w", e.Name(), err)
		}
	}
	return nil
}

// healFromFile quarantines the corrupt original then overwrites the live region
// file in place with a validated-clean source file.
func healFromFile(regionDir, fn, srcPath, quarantineDir, stamp string) error {
	_ = os.MkdirAll(quarantineDir, 0o755)
	live := filepath.Join(regionDir, fn)
	_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
	return copyFileInPlace(srcPath, live)
}

// healFromBytes is the same but the clean source is in-memory (a tar backup).
func healFromBytes(regionDir, fn string, data []byte, quarantineDir, stamp string) error {
	_ = os.MkdirAll(quarantineDir, 0o755)
	live := filepath.Join(regionDir, fn)
	_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
	tmp := live + ".medic-new"
	if err := os.WriteFile(tmp, data, 0o644); err != nil {
		return err
	}
	return os.Rename(tmp, live)
}

// copyFile copies src→dst atomically (temp+rename), same filesystem assumed for
// the rename target dir. Used for snapshots + quarantine (rolling dir).
func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	tmp := dst + ".tmp-copy"
	out, err := os.Create(tmp)
	if err != nil {
		return err
	}
	if _, err := io.Copy(out, in); err != nil {
		out.Close()
		os.Remove(tmp)
		return err
	}
	if err := out.Close(); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, dst)
}

// copyFileInPlace overwrites dst with src via a temp file in dst's own dir, so
// the rename is same-filesystem even when src lives elsewhere (a rolling slot).
func copyFileInPlace(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	tmp := dst + ".medic-new"
	out, err := os.Create(tmp)
	if err != nil {
		return err
	}
	if _, err := io.Copy(out, in); err != nil {
		out.Close()
		os.Remove(tmp)
		return err
	}
	if err := out.Close(); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, dst)
}