package main import ( "encoding/json" "fmt" "io" "os" "path/filepath" "sort" "strconv" "strings" "time" "github.com/dbledeez/panel/pkg/regionmedic" ) // region-medic autoheal — the auto-on-reboot flow, designed to run (in a sidecar) // while the 7DTD server is STOPPED, just before the agent starts it: // // 1. Validate EVERY .7rg in the active world's Region dir (catches whole-file // corruption like a bad magic header, which has no error_backups and so is // invisible to the error_backup-cluster scan). // 2. Heal each corrupt region from the newest source that has a validated-clean // copy: the rolling folder snapshots first (fast, local), then the panel's // tar backups as a fallback. Corrupt originals are quarantined first. // 3. Snapshot the now-clean Region dir to a fresh rolling slot (real folder // copy — instant restore, no unzip) and rotate to keep the newest -keep. // // Emits a JSON report the agent relays to the controller for the Discord ping. // // region-medic autoheal -region-dir D -rolling-dir R [-backups-dir B] [-keep 2] [-apply] // // Without -apply it only validates + reports (dry run: no heal, no snapshot). type ahHeal struct { Region string `json:"region"` // r.X.Z Source string `json:"source"` // "rolling:" | "backup:" SourceTime string `json:"source_time"` // RFC3339 of the snapshot/backup } type ahReport struct { RegionDir string `json:"region_dir"` TotalRegions int `json:"total_regions"` CorruptFound int `json:"corrupt_found"` Healed []ahHeal `json:"healed"` Unhealable []string `json:"unhealable"` SnapshotSlot string `json:"snapshot_slot"` RollingKept int `json:"rolling_kept"` Applied bool `json:"applied"` DurationMs int64 `json:"duration_ms"` } func cmdAutoheal(args []string) error { var regionDir, rollingDir, backupsDir, savesRoot, discordChannel, discordTokenFile, label string keep := 2 apply := false for i := 0; i < len(args); i++ { switch args[i] { case "-region-dir": i++ if i < len(args) { regionDir = args[i] } case "-saves-root": i++ if i < len(args) { savesRoot = args[i] } case "-rolling-dir": i++ if i < len(args) { rollingDir = args[i] } case "-backups-dir": i++ if i < len(args) { backupsDir = args[i] } case "-keep": i++ if i < len(args) { keep, _ = strconv.Atoi(args[i]) } case "-discord-channel": i++ if i < len(args) { discordChannel = args[i] } case "-discord-token-file": i++ if i < len(args) { discordTokenFile = args[i] } case "-label": i++ if i < len(args) { label = args[i] } case "-apply", "--apply": apply = true default: return fmt.Errorf("unknown flag %q", args[i]) } } // -saves-root auto-resolves the active world's Region dir (so the agent // doesn't have to know GameWorld/GameName, and RWG seed-named worlds work). // Per-server config lives in the saves volume (panel writes it via FsWrite, // no recreate): world pin + keep + Discord channel; enabled:false skips. var cfgWorld string if savesRoot != "" { if c, ok := readSavesMedicConfig(savesRoot); ok { if c.Enabled != nil && !*c.Enabled { fmt.Fprintln(os.Stderr, "autoheal: disabled via region-medic.json, skipping") return nil } if c.Keep > 0 { keep = c.Keep } if c.DiscordChannel != "" { discordChannel = c.DiscordChannel } cfgWorld = strings.TrimSpace(c.World) } } // Resolve the Region dir. An explicit world pin wins over auto-detect — some // servers carry multiple saves, so the operator picks which one to heal. If // the pinned world has no Region dir we SKIP rather than silently auto-pick a // different world the operator didn't choose. if regionDir == "" && savesRoot != "" { if cfgWorld != "" { wd := filepath.Join(savesRoot, ".local/share/7DaysToDie/Saves", cfgWorld, "Region") if st, err := os.Stat(wd); err != nil || !st.IsDir() { fmt.Fprintf(os.Stderr, "autoheal: pinned world %q has no Region dir at %s — skipping\n", cfgWorld, wd) return nil } regionDir = wd } else { rd, rerr := resolveRegionDir(savesRoot) if rerr != nil { return rerr } regionDir = rd } } if regionDir == "" || rollingDir == "" { return fmt.Errorf("required: -region-dir (or -saves-root) and -rolling-dir (and -backups-dir for tar fallback)") } if keep < 1 { keep = 1 } start := time.Now() rep := ahReport{RegionDir: regionDir, RollingKept: keep, Applied: apply, Healed: []ahHeal{}, Unhealable: []string{}} // 1. Validate every .7rg. ents, err := os.ReadDir(regionDir) if err != nil { return fmt.Errorf("read region dir: %w", err) } var corrupt []regionmedic.RegionCoord for _, e := range ents { if e.IsDir() { continue } rc, ok := regionmedic.ParseRegionFileName(e.Name()) if !ok { continue } rep.TotalRegions++ if !regionmedic.ValidateRegionFile(filepath.Join(regionDir, e.Name()), false).Healthy() { corrupt = append(corrupt, rc) } } rep.CorruptFound = len(corrupt) // 2. Heal (only with -apply). quarantineDir := filepath.Join(rollingDir, "_quarantine") if apply && len(corrupt) > 0 { slots := listRollingSlots(rollingDir) // newest first var panelBackups []regionmedic.BackupRef if backupsDir != "" { panelBackups, _ = regionmedic.ListPanelBackups(backupsDir) } stamp := start.UTC().Format("20060102-150405") for _, rc := range corrupt { fn := rc.FileName() done := false // 2a. newest rolling snapshot with a clean copy. for _, s := range slots { cand := filepath.Join(s.path, fn) if fi, e := os.Stat(cand); e == nil && !fi.IsDir() && regionmedic.ValidateRegionFile(cand, false).Healthy() { if e := healFromFile(regionDir, fn, cand, quarantineDir, stamp); e == nil { rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "rolling:" + filepath.Base(s.path), SourceTime: s.t.UTC().Format(time.RFC3339)}) done = true break } } } // 2b. panel tar backups fallback. if !done && len(panelBackups) > 0 { entry := regionmedic.SaveRelEntry(regionDir, fn) best, _, _ := regionmedic.FindLastGoodRegion(panelBackups, entry, false, time.Time{}) if best.Found && best.Clean() { if e := healFromBytes(regionDir, fn, best.Bytes, quarantineDir, stamp); e == nil { rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "backup:" + best.Backup.ID, SourceTime: best.Backup.CreatedAt.UTC().Format(time.RFC3339)}) done = true } } } if !done { rep.Unhealable = append(rep.Unhealable, rc.String()) } } } // 3. Snapshot the now-clean Region dir to a fresh rolling slot + rotate. if apply { slotName := fmt.Sprintf("slot-%d", start.Unix()) if err := snapshotRegions(regionDir, filepath.Join(rollingDir, slotName)); err != nil { rep.SnapshotSlot = "ERROR: " + err.Error() } else { rep.SnapshotSlot = slotName rotateRollingSlots(rollingDir, keep) } } rep.DurationMs = time.Since(start).Milliseconds() b, _ := json.MarshalIndent(rep, "", " ") fmt.Println(string(b)) fmt.Fprintf(os.Stderr, "autoheal: %d regions, %d corrupt, %d healed, %d unhealable, snapshot=%s (%dms)\n", rep.TotalRegions, rep.CorruptFound, len(rep.Healed), len(rep.Unhealable), rep.SnapshotSlot, rep.DurationMs) // Discord alert — only when we actually healed something on this reboot. if apply { if derr := postDiscordHealReport(discordTokenFile, discordChannel, label, rep); derr != nil { fmt.Fprintf(os.Stderr, "discord post failed: %v\n", derr) } else if discordChannel != "" && len(rep.Healed) > 0 { fmt.Fprintf(os.Stderr, "discord: posted heal embed to channel %s\n", discordChannel) } } return nil } type rollingSlot struct { path string t time.Time } // listRollingSlots returns slot dirs named "slot-" newest first. func listRollingSlots(dir string) []rollingSlot { ents, err := os.ReadDir(dir) if err != nil { return nil } var out []rollingSlot for _, e := range ents { if !e.IsDir() || !strings.HasPrefix(e.Name(), "slot-") { continue } secs, err := strconv.ParseInt(strings.TrimPrefix(e.Name(), "slot-"), 10, 64) if err != nil { continue } out = append(out, rollingSlot{path: filepath.Join(dir, e.Name()), t: time.Unix(secs, 0)}) } sort.Slice(out, func(i, j int) bool { return out[i].t.After(out[j].t) }) return out } // rotateRollingSlots keeps the newest `keep` slot dirs, removing older ones. func rotateRollingSlots(dir string, keep int) { for i, s := range listRollingSlots(dir) { if i >= keep { _ = os.RemoveAll(s.path) } } } // snapshotRegions copies every *.7rg from regionDir into slotPath. func snapshotRegions(regionDir, slotPath string) error { if err := os.MkdirAll(slotPath, 0o755); err != nil { return err } ents, err := os.ReadDir(regionDir) if err != nil { return err } for _, e := range ents { if e.IsDir() || !strings.HasSuffix(e.Name(), ".7rg") { continue } if err := copyFile(filepath.Join(regionDir, e.Name()), filepath.Join(slotPath, e.Name())); err != nil { return fmt.Errorf("copy %s: %w", e.Name(), err) } } return nil } // healFromFile quarantines the corrupt original then overwrites the live region // file in place with a validated-clean source file. func healFromFile(regionDir, fn, srcPath, quarantineDir, stamp string) error { _ = os.MkdirAll(quarantineDir, 0o755) live := filepath.Join(regionDir, fn) _ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp)) return copyFileInPlace(srcPath, live) } // healFromBytes is the same but the clean source is in-memory (a tar backup). func healFromBytes(regionDir, fn string, data []byte, quarantineDir, stamp string) error { _ = os.MkdirAll(quarantineDir, 0o755) live := filepath.Join(regionDir, fn) _ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp)) tmp := live + ".medic-new" if err := os.WriteFile(tmp, data, 0o644); err != nil { return err } return os.Rename(tmp, live) } // copyFile copies src→dst atomically (temp+rename), same filesystem assumed for // the rename target dir. Used for snapshots + quarantine (rolling dir). func copyFile(src, dst string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() tmp := dst + ".tmp-copy" out, err := os.Create(tmp) if err != nil { return err } if _, err := io.Copy(out, in); err != nil { out.Close() os.Remove(tmp) return err } if err := out.Close(); err != nil { os.Remove(tmp) return err } return os.Rename(tmp, dst) } // copyFileInPlace overwrites dst with src via a temp file in dst's own dir, so // the rename is same-filesystem even when src lives elsewhere (a rolling slot). func copyFileInPlace(src, dst string) error { in, err := os.Open(src) if err != nil { return err } defer in.Close() tmp := dst + ".medic-new" out, err := os.Create(tmp) if err != nil { return err } if _, err := io.Copy(out, in); err != nil { out.Close() os.Remove(tmp) return err } if err := out.Close(); err != nil { os.Remove(tmp) return err } return os.Rename(tmp, dst) }