panel v0.9.2 — open-source game server manager
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,367 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
)
|
||||
|
||||
// region-medic autoheal — the auto-on-reboot flow, designed to run (in a sidecar)
|
||||
// while the 7DTD server is STOPPED, just before the agent starts it:
|
||||
//
|
||||
// 1. Validate EVERY .7rg in the active world's Region dir (catches whole-file
|
||||
// corruption like a bad magic header, which has no error_backups and so is
|
||||
// invisible to the error_backup-cluster scan).
|
||||
// 2. Heal each corrupt region from the newest source that has a validated-clean
|
||||
// copy: the rolling folder snapshots first (fast, local), then the panel's
|
||||
// tar backups as a fallback. Corrupt originals are quarantined first.
|
||||
// 3. Snapshot the now-clean Region dir to a fresh rolling slot (real folder
|
||||
// copy — instant restore, no unzip) and rotate to keep the newest -keep.
|
||||
//
|
||||
// Emits a JSON report the agent relays to the controller for the Discord ping.
|
||||
//
|
||||
// region-medic autoheal -region-dir D -rolling-dir R [-backups-dir B] [-keep 2] [-apply]
|
||||
//
|
||||
// Without -apply it only validates + reports (dry run: no heal, no snapshot).
|
||||
|
||||
type ahHeal struct {
|
||||
Region string `json:"region"` // r.X.Z
|
||||
Source string `json:"source"` // "rolling:<slot>" | "backup:<id>"
|
||||
SourceTime string `json:"source_time"` // RFC3339 of the snapshot/backup
|
||||
}
|
||||
|
||||
type ahReport struct {
|
||||
RegionDir string `json:"region_dir"`
|
||||
TotalRegions int `json:"total_regions"`
|
||||
CorruptFound int `json:"corrupt_found"`
|
||||
Healed []ahHeal `json:"healed"`
|
||||
Unhealable []string `json:"unhealable"`
|
||||
SnapshotSlot string `json:"snapshot_slot"`
|
||||
RollingKept int `json:"rolling_kept"`
|
||||
Applied bool `json:"applied"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
}
|
||||
|
||||
func cmdAutoheal(args []string) error {
|
||||
var regionDir, rollingDir, backupsDir, savesRoot, discordChannel, discordTokenFile, label string
|
||||
keep := 2
|
||||
apply := false
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "-region-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
regionDir = args[i]
|
||||
}
|
||||
case "-saves-root":
|
||||
i++
|
||||
if i < len(args) {
|
||||
savesRoot = args[i]
|
||||
}
|
||||
case "-rolling-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
rollingDir = args[i]
|
||||
}
|
||||
case "-backups-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
backupsDir = args[i]
|
||||
}
|
||||
case "-keep":
|
||||
i++
|
||||
if i < len(args) {
|
||||
keep, _ = strconv.Atoi(args[i])
|
||||
}
|
||||
case "-discord-channel":
|
||||
i++
|
||||
if i < len(args) {
|
||||
discordChannel = args[i]
|
||||
}
|
||||
case "-discord-token-file":
|
||||
i++
|
||||
if i < len(args) {
|
||||
discordTokenFile = args[i]
|
||||
}
|
||||
case "-label":
|
||||
i++
|
||||
if i < len(args) {
|
||||
label = args[i]
|
||||
}
|
||||
case "-apply", "--apply":
|
||||
apply = true
|
||||
default:
|
||||
return fmt.Errorf("unknown flag %q", args[i])
|
||||
}
|
||||
}
|
||||
// -saves-root auto-resolves the active world's Region dir (so the agent
|
||||
// doesn't have to know GameWorld/GameName, and RWG seed-named worlds work).
|
||||
// Per-server config lives in the saves volume (panel writes it via FsWrite,
|
||||
// no recreate): world pin + keep + Discord channel; enabled:false skips.
|
||||
var cfgWorld string
|
||||
if savesRoot != "" {
|
||||
if c, ok := readSavesMedicConfig(savesRoot); ok {
|
||||
if c.Enabled != nil && !*c.Enabled {
|
||||
fmt.Fprintln(os.Stderr, "autoheal: disabled via region-medic.json, skipping")
|
||||
return nil
|
||||
}
|
||||
if c.Keep > 0 {
|
||||
keep = c.Keep
|
||||
}
|
||||
if c.DiscordChannel != "" {
|
||||
discordChannel = c.DiscordChannel
|
||||
}
|
||||
cfgWorld = strings.TrimSpace(c.World)
|
||||
}
|
||||
}
|
||||
// Resolve the Region dir. An explicit world pin wins over auto-detect — some
|
||||
// servers carry multiple saves, so the operator picks which one to heal. If
|
||||
// the pinned world has no Region dir we SKIP rather than silently auto-pick a
|
||||
// different world the operator didn't choose.
|
||||
if regionDir == "" && savesRoot != "" {
|
||||
if cfgWorld != "" {
|
||||
wd := filepath.Join(savesRoot, ".local/share/7DaysToDie/Saves", cfgWorld, "Region")
|
||||
if st, err := os.Stat(wd); err != nil || !st.IsDir() {
|
||||
fmt.Fprintf(os.Stderr, "autoheal: pinned world %q has no Region dir at %s — skipping\n", cfgWorld, wd)
|
||||
return nil
|
||||
}
|
||||
regionDir = wd
|
||||
} else {
|
||||
rd, rerr := resolveRegionDir(savesRoot)
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
regionDir = rd
|
||||
}
|
||||
}
|
||||
if regionDir == "" || rollingDir == "" {
|
||||
return fmt.Errorf("required: -region-dir (or -saves-root) and -rolling-dir (and -backups-dir for tar fallback)")
|
||||
}
|
||||
if keep < 1 {
|
||||
keep = 1
|
||||
}
|
||||
start := time.Now()
|
||||
rep := ahReport{RegionDir: regionDir, RollingKept: keep, Applied: apply, Healed: []ahHeal{}, Unhealable: []string{}}
|
||||
|
||||
// 1. Validate every .7rg.
|
||||
ents, err := os.ReadDir(regionDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read region dir: %w", err)
|
||||
}
|
||||
var corrupt []regionmedic.RegionCoord
|
||||
for _, e := range ents {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
rc, ok := regionmedic.ParseRegionFileName(e.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rep.TotalRegions++
|
||||
if !regionmedic.ValidateRegionFile(filepath.Join(regionDir, e.Name()), false).Healthy() {
|
||||
corrupt = append(corrupt, rc)
|
||||
}
|
||||
}
|
||||
rep.CorruptFound = len(corrupt)
|
||||
|
||||
// 2. Heal (only with -apply).
|
||||
quarantineDir := filepath.Join(rollingDir, "_quarantine")
|
||||
if apply && len(corrupt) > 0 {
|
||||
slots := listRollingSlots(rollingDir) // newest first
|
||||
var panelBackups []regionmedic.BackupRef
|
||||
if backupsDir != "" {
|
||||
panelBackups, _ = regionmedic.ListPanelBackups(backupsDir)
|
||||
}
|
||||
stamp := start.UTC().Format("20060102-150405")
|
||||
for _, rc := range corrupt {
|
||||
fn := rc.FileName()
|
||||
done := false
|
||||
// 2a. newest rolling snapshot with a clean copy.
|
||||
for _, s := range slots {
|
||||
cand := filepath.Join(s.path, fn)
|
||||
if fi, e := os.Stat(cand); e == nil && !fi.IsDir() &&
|
||||
regionmedic.ValidateRegionFile(cand, false).Healthy() {
|
||||
if e := healFromFile(regionDir, fn, cand, quarantineDir, stamp); e == nil {
|
||||
rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "rolling:" + filepath.Base(s.path), SourceTime: s.t.UTC().Format(time.RFC3339)})
|
||||
done = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2b. panel tar backups fallback.
|
||||
if !done && len(panelBackups) > 0 {
|
||||
entry := regionmedic.SaveRelEntry(regionDir, fn)
|
||||
best, _, _ := regionmedic.FindLastGoodRegion(panelBackups, entry, false, time.Time{})
|
||||
if best.Found && best.Clean() {
|
||||
if e := healFromBytes(regionDir, fn, best.Bytes, quarantineDir, stamp); e == nil {
|
||||
rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "backup:" + best.Backup.ID, SourceTime: best.Backup.CreatedAt.UTC().Format(time.RFC3339)})
|
||||
done = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !done {
|
||||
rep.Unhealable = append(rep.Unhealable, rc.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Snapshot the now-clean Region dir to a fresh rolling slot + rotate.
|
||||
if apply {
|
||||
slotName := fmt.Sprintf("slot-%d", start.Unix())
|
||||
if err := snapshotRegions(regionDir, filepath.Join(rollingDir, slotName)); err != nil {
|
||||
rep.SnapshotSlot = "ERROR: " + err.Error()
|
||||
} else {
|
||||
rep.SnapshotSlot = slotName
|
||||
rotateRollingSlots(rollingDir, keep)
|
||||
}
|
||||
}
|
||||
|
||||
rep.DurationMs = time.Since(start).Milliseconds()
|
||||
b, _ := json.MarshalIndent(rep, "", " ")
|
||||
fmt.Println(string(b))
|
||||
fmt.Fprintf(os.Stderr, "autoheal: %d regions, %d corrupt, %d healed, %d unhealable, snapshot=%s (%dms)\n",
|
||||
rep.TotalRegions, rep.CorruptFound, len(rep.Healed), len(rep.Unhealable), rep.SnapshotSlot, rep.DurationMs)
|
||||
// Discord alert — only when we actually healed something on this reboot.
|
||||
if apply {
|
||||
if derr := postDiscordHealReport(discordTokenFile, discordChannel, label, rep); derr != nil {
|
||||
fmt.Fprintf(os.Stderr, "discord post failed: %v\n", derr)
|
||||
} else if discordChannel != "" && len(rep.Healed) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "discord: posted heal embed to channel %s\n", discordChannel)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type rollingSlot struct {
|
||||
path string
|
||||
t time.Time
|
||||
}
|
||||
|
||||
// listRollingSlots returns slot dirs named "slot-<unix>" newest first.
|
||||
func listRollingSlots(dir string) []rollingSlot {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []rollingSlot
|
||||
for _, e := range ents {
|
||||
if !e.IsDir() || !strings.HasPrefix(e.Name(), "slot-") {
|
||||
continue
|
||||
}
|
||||
secs, err := strconv.ParseInt(strings.TrimPrefix(e.Name(), "slot-"), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, rollingSlot{path: filepath.Join(dir, e.Name()), t: time.Unix(secs, 0)})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].t.After(out[j].t) })
|
||||
return out
|
||||
}
|
||||
|
||||
// rotateRollingSlots keeps the newest `keep` slot dirs, removing older ones.
|
||||
func rotateRollingSlots(dir string, keep int) {
|
||||
for i, s := range listRollingSlots(dir) {
|
||||
if i >= keep {
|
||||
_ = os.RemoveAll(s.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// snapshotRegions copies every *.7rg from regionDir into slotPath.
|
||||
func snapshotRegions(regionDir, slotPath string) error {
|
||||
if err := os.MkdirAll(slotPath, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
ents, err := os.ReadDir(regionDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range ents {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".7rg") {
|
||||
continue
|
||||
}
|
||||
if err := copyFile(filepath.Join(regionDir, e.Name()), filepath.Join(slotPath, e.Name())); err != nil {
|
||||
return fmt.Errorf("copy %s: %w", e.Name(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// healFromFile quarantines the corrupt original then overwrites the live region
|
||||
// file in place with a validated-clean source file.
|
||||
func healFromFile(regionDir, fn, srcPath, quarantineDir, stamp string) error {
|
||||
_ = os.MkdirAll(quarantineDir, 0o755)
|
||||
live := filepath.Join(regionDir, fn)
|
||||
_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
|
||||
return copyFileInPlace(srcPath, live)
|
||||
}
|
||||
|
||||
// healFromBytes is the same but the clean source is in-memory (a tar backup).
|
||||
func healFromBytes(regionDir, fn string, data []byte, quarantineDir, stamp string) error {
|
||||
_ = os.MkdirAll(quarantineDir, 0o755)
|
||||
live := filepath.Join(regionDir, fn)
|
||||
_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
|
||||
tmp := live + ".medic-new"
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, live)
|
||||
}
|
||||
|
||||
// copyFile copies src→dst atomically (temp+rename), same filesystem assumed for
|
||||
// the rename target dir. Used for snapshots + quarantine (rolling dir).
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
tmp := dst + ".tmp-copy"
|
||||
out, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
// copyFileInPlace overwrites dst with src via a temp file in dst's own dir, so
|
||||
// the rename is same-filesystem even when src lives elsewhere (a rolling slot).
|
||||
func copyFileInPlace(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
tmp := dst + ".medic-new"
|
||||
out, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
_ "time/tzdata" // embed the IANA tz database so America/Los_Angeles resolves
|
||||
// even in debian:12-slim, which ships no /usr/share/zoneinfo.
|
||||
)
|
||||
|
||||
// resolveRegionDir finds the active world's Region dir under a 7DTD saves root
|
||||
// (the dir mounted as the saves volume — holds serverconfig.xml + .local/...).
|
||||
// Prefers serverconfig GameWorld/GameName when that world has real data
|
||||
// (main.ttw); else falls back to the most-recently-written Saves/*/<GameName>
|
||||
// with data — handles RWG seed-named worlds.
|
||||
func resolveRegionDir(savesRoot string) (string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(savesRoot, "serverconfig.xml"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read serverconfig.xml: %w", err)
|
||||
}
|
||||
gw := xmlProp(string(raw), "GameWorld")
|
||||
gn := xmlProp(string(raw), "GameName")
|
||||
base := filepath.Join(savesRoot, ".local", "share", "7DaysToDie", "Saves")
|
||||
hasWorld := func(d string) bool {
|
||||
if _, e := os.Stat(filepath.Join(d, "main.ttw")); e == nil {
|
||||
return true
|
||||
}
|
||||
fi, e := os.Stat(filepath.Join(d, "Region"))
|
||||
return e == nil && fi.IsDir()
|
||||
}
|
||||
if gw != "" && gn != "" {
|
||||
lit := filepath.Join(base, gw, gn)
|
||||
if hasWorld(lit) {
|
||||
return filepath.Join(lit, "Region"), nil
|
||||
}
|
||||
}
|
||||
if gn != "" { // RWG fallback: newest world dir with this GameName.
|
||||
worlds, _ := os.ReadDir(base)
|
||||
var newest string
|
||||
var newestT time.Time
|
||||
for _, w := range worlds {
|
||||
if !w.IsDir() {
|
||||
continue
|
||||
}
|
||||
cand := filepath.Join(base, w.Name(), gn)
|
||||
if !hasWorld(cand) {
|
||||
continue
|
||||
}
|
||||
if fi, e := os.Stat(cand); e == nil && fi.ModTime().After(newestT) {
|
||||
newestT, newest = fi.ModTime(), cand
|
||||
}
|
||||
}
|
||||
if newest != "" {
|
||||
return filepath.Join(newest, "Region"), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("could not resolve active Region dir (GameWorld=%q GameName=%q)", gw, gn)
|
||||
}
|
||||
|
||||
// xmlProp pulls the value from <property name="X" value="Y" /> (tolerant of
|
||||
// attribute order + quote style; skips comments).
|
||||
func xmlProp(content, name string) string {
|
||||
for _, raw := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "<!--") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(line, `name="`+name+`"`) {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(line, "value=")
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
rest := line[i+len("value="):]
|
||||
if len(rest) < 1 {
|
||||
continue
|
||||
}
|
||||
q := rest[0]
|
||||
rest = rest[1:]
|
||||
if end := strings.IndexByte(rest, q); end >= 0 {
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// postDiscordHealReport posts a rich embed heal summary to a Discord channel via
|
||||
// a bot token read from tokenFile. No-op when token/channel missing or nothing
|
||||
// healed. All times are rendered in Pacific (America/Los_Angeles → PST/PDT).
|
||||
func postDiscordHealReport(tokenFile, channel, label string, rep ahReport) error {
|
||||
if tokenFile == "" || channel == "" || len(rep.Healed) == 0 {
|
||||
return nil
|
||||
}
|
||||
tb, err := os.ReadFile(tokenFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read discord token: %w", err)
|
||||
}
|
||||
token := strings.TrimSpace(string(tb))
|
||||
if token == "" {
|
||||
return nil
|
||||
}
|
||||
if label == "" {
|
||||
label = "7DTD server"
|
||||
}
|
||||
loc, lerr := time.LoadLocation("America/Los_Angeles")
|
||||
if lerr != nil || loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
pst := func(rfc string) string {
|
||||
if t, e := time.Parse(time.RFC3339, rfc); e == nil {
|
||||
return t.In(loc).Format("Jan 2, 3:04 PM MST")
|
||||
}
|
||||
return rfc
|
||||
}
|
||||
srcLabel := func(s string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(s, "rolling:"):
|
||||
return "rolling snapshot"
|
||||
case strings.HasPrefix(s, "backup:"):
|
||||
return "panel backup"
|
||||
default:
|
||||
return s
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
for _, h := range rep.Healed {
|
||||
lines = append(lines, fmt.Sprintf("• `%s` ← %s _(saved %s)_", h.Region, srcLabel(h.Source), pst(h.SourceTime)))
|
||||
}
|
||||
color := 0x57F287 // green — everything healed
|
||||
fields := []map[string]any{{
|
||||
"name": fmt.Sprintf("✅ Healed %d region(s)", len(rep.Healed)),
|
||||
"value": strings.Join(lines, "\n"),
|
||||
"inline": false,
|
||||
}}
|
||||
if len(rep.Unhealable) > 0 {
|
||||
color = 0xED4245 // red — some couldn't be healed
|
||||
fields = append(fields, map[string]any{
|
||||
"name": fmt.Sprintf("⚠️ Could NOT heal %d (no clean copy anywhere)", len(rep.Unhealable)),
|
||||
"value": "`" + strings.Join(rep.Unhealable, "`, `") + "`",
|
||||
"inline": false,
|
||||
})
|
||||
}
|
||||
nowPST := time.Now().In(loc).Format("Mon Jan 2, 3:04 PM MST")
|
||||
embed := map[string]any{
|
||||
"title": "🩺 Region Medic — " + label,
|
||||
"description": fmt.Sprintf("Detected and repaired **%d** corrupt region file(s) on reboot, before the world loaded.", rep.CorruptFound),
|
||||
"color": color,
|
||||
"fields": fields,
|
||||
"footer": map[string]any{"text": "Region Medic · " + nowPST},
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"embeds": []any{embed}})
|
||||
req, err := http.NewRequest(http.MethodPost, "https://discord.com/api/v10/channels/"+channel+"/messages", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bot "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "RefugeBot-RegionMedic (https://refugegaming.org, 1.0)")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("discord HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// savesMedicConfig is the per-server medic config the panel writes (via FsWrite,
|
||||
// no container recreate) into the saves volume at region-medic.json. Enabled is a
|
||||
// pointer so an absent field defaults to ON.
|
||||
type savesMedicConfig struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
Keep int `json:"keep"`
|
||||
World string `json:"world"` // "<World>/<GameName>"; empty = auto-detect active world
|
||||
DiscordChannel string `json:"discord_channel"`
|
||||
}
|
||||
|
||||
// readSavesMedicConfig reads <savesRoot>/region-medic.json; ok=false if absent or
|
||||
// unparseable (the caller then keeps its flag defaults).
|
||||
func readSavesMedicConfig(savesRoot string) (savesMedicConfig, bool) {
|
||||
b, err := os.ReadFile(filepath.Join(savesRoot, "region-medic.json"))
|
||||
if err != nil {
|
||||
return savesMedicConfig{}, false
|
||||
}
|
||||
var c savesMedicConfig
|
||||
if json.Unmarshal(b, &c) != nil {
|
||||
return savesMedicConfig{}, false
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
// Command region-medic is a standalone operator tool that detects and heals
|
||||
// corrupted 7DTD ".7rg" region files from the panel's own backups, using the
|
||||
// shared github.com/dbledeez/panel/pkg/regionmedic engine.
|
||||
//
|
||||
// It is the same logic that ships inside the agent's RegionScan/RegionHeal RPCs,
|
||||
// exposed as a CLI so it can be run by hand against a stopped instance without
|
||||
// redeploying the controller. Subcommands:
|
||||
//
|
||||
// region-medic validate <file.7rg> [-deep]
|
||||
// region-medic scan <region-dir>
|
||||
// region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
|
||||
// region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply
|
||||
// [-deep] [-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
|
||||
// [-controller URL -instance ID] # stop before / start after (panel-safe)
|
||||
// [-container NAME] # post-start: watch for ready + new corruption
|
||||
// region-medic verify <region-dir> # count error_backups now
|
||||
//
|
||||
// stop/start go through the controller HTTP API (never raw docker), so panel
|
||||
// state stays correct; readiness/state polling reads docker locally.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
var err error
|
||||
switch os.Args[1] {
|
||||
case "validate":
|
||||
err = cmdValidate(os.Args[2:])
|
||||
case "scan":
|
||||
err = cmdScan(os.Args[2:])
|
||||
case "plan":
|
||||
err = cmdHeal(os.Args[2:], false)
|
||||
case "heal":
|
||||
err = cmdHeal(os.Args[2:], true)
|
||||
case "verify":
|
||||
err = cmdVerify(os.Args[2:])
|
||||
case "autoheal":
|
||||
err = cmdAutoheal(os.Args[2:])
|
||||
case "-h", "--help", "help":
|
||||
usage()
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1])
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `region-medic — detect & heal corrupt 7DTD .7rg regions from panel backups
|
||||
|
||||
region-medic validate <file.7rg> [-deep]
|
||||
region-medic scan <region-dir>
|
||||
region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
|
||||
region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply [-deep]
|
||||
[-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
|
||||
[-controller URL -instance ID] [-container NAME]
|
||||
region-medic verify <region-dir>
|
||||
`)
|
||||
}
|
||||
|
||||
func printJSON(v any) {
|
||||
b, _ := json.MarshalIndent(v, "", " ")
|
||||
fmt.Println(string(b))
|
||||
}
|
||||
|
||||
// --- validate ---------------------------------------------------------------
|
||||
|
||||
func cmdValidate(args []string) error {
|
||||
deep := false
|
||||
var file string
|
||||
for _, a := range args {
|
||||
if a == "-deep" || a == "--deep" {
|
||||
deep = true
|
||||
} else if !strings.HasPrefix(a, "-") {
|
||||
file = a
|
||||
}
|
||||
}
|
||||
if file == "" {
|
||||
return fmt.Errorf("usage: region-medic validate <file.7rg> [-deep]")
|
||||
}
|
||||
rep := regionmedic.ValidateRegionFile(file, deep)
|
||||
printJSON(rep)
|
||||
if rep.Healthy() {
|
||||
fmt.Fprintf(os.Stderr, "HEALTHY: %d present chunks, %d sectors\n", rep.PresentChunks, rep.TotalSectors)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "UNHEALTHY: err=%q badChunks=%d\n", rep.Err, len(rep.BadChunks))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- scan -------------------------------------------------------------------
|
||||
|
||||
func cmdScan(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("usage: region-medic scan <region-dir>")
|
||||
}
|
||||
scan, err := regionmedic.ScanRegionDir(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printJSON(scan)
|
||||
fmt.Fprintf(os.Stderr, "%d error_backups across %d affected region(s)\n", scan.ErrorBackups, len(scan.Affected))
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- verify -----------------------------------------------------------------
|
||||
|
||||
func cmdVerify(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("usage: region-medic verify <region-dir>")
|
||||
}
|
||||
scan, err := regionmedic.ScanRegionDir(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "error_backups now: %d (affected regions: %d)\n", scan.ErrorBackups, len(scan.Affected))
|
||||
printJSON(scan.Affected)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- plan / heal ------------------------------------------------------------
|
||||
|
||||
type healFlags struct {
|
||||
regionDir, region, backupsDir string
|
||||
snapshotDir, quarantineDir string
|
||||
deep, apply bool
|
||||
notAfter string
|
||||
controller, instance, container string
|
||||
}
|
||||
|
||||
func parseHealFlags(args []string) (healFlags, error) {
|
||||
f := healFlags{}
|
||||
for i := 0; i < len(args); i++ {
|
||||
a := args[i]
|
||||
next := func() string {
|
||||
if i+1 < len(args) {
|
||||
i++
|
||||
return args[i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
switch a {
|
||||
case "-region-dir":
|
||||
f.regionDir = next()
|
||||
case "-region":
|
||||
f.region = next()
|
||||
case "-backups-dir":
|
||||
f.backupsDir = next()
|
||||
case "-snapshot-dir":
|
||||
f.snapshotDir = next()
|
||||
case "-quarantine-dir":
|
||||
f.quarantineDir = next()
|
||||
case "-not-after":
|
||||
f.notAfter = next()
|
||||
case "-controller":
|
||||
f.controller = next()
|
||||
case "-instance":
|
||||
f.instance = next()
|
||||
case "-container":
|
||||
f.container = next()
|
||||
case "-deep", "--deep":
|
||||
f.deep = true
|
||||
case "-apply", "--apply":
|
||||
f.apply = true
|
||||
default:
|
||||
return f, fmt.Errorf("unknown flag %q", a)
|
||||
}
|
||||
}
|
||||
if f.regionDir == "" || f.region == "" || f.backupsDir == "" {
|
||||
return f, fmt.Errorf("required: -region-dir, -region, -backups-dir")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func cmdHeal(args []string, allowApply bool) error {
|
||||
f, err := parseHealFlags(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
region, ok := regionmedic.ParseRegionFileName(regionArg(f.region))
|
||||
if !ok {
|
||||
return fmt.Errorf("bad -region %q (want r.X.Z or r.X.Z.7rg)", f.region)
|
||||
}
|
||||
var notAfter time.Time
|
||||
if f.notAfter != "" {
|
||||
notAfter, err = time.Parse(time.RFC3339, f.notAfter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bad -not-after: %w", err)
|
||||
}
|
||||
}
|
||||
backups, err := regionmedic.ListPanelBackups(f.backupsDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list backups: %w", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "found %d backups in %s\n", len(backups), f.backupsDir)
|
||||
|
||||
plan, err := regionmedic.PlanHeal(f.regionDir, region, backups, f.deep, notAfter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summarizePlan(plan)
|
||||
|
||||
if !allowApply || !f.apply {
|
||||
fmt.Fprintln(os.Stderr, "(dry run — re-run `heal ... -apply` to execute)")
|
||||
return nil
|
||||
}
|
||||
if !plan.Ready() {
|
||||
return fmt.Errorf("refusing to heal: no clean backup copy found for %s", region)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// Orchestrate stop (panel-safe) if controller configured.
|
||||
if f.controller != "" && f.instance != "" {
|
||||
fmt.Fprintf(os.Stderr, "stopping %s via panel...\n", f.instance)
|
||||
if err := panelAction(ctx, f.controller, f.instance, "stop"); err != nil {
|
||||
return fmt.Errorf("panel stop: %w", err)
|
||||
}
|
||||
if f.container != "" {
|
||||
if err := waitContainer(ctx, f.container, false, 90*time.Second); err != nil {
|
||||
return fmt.Errorf("wait stopped: %w", err)
|
||||
}
|
||||
} else {
|
||||
time.Sleep(8 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
snapDir := orDefault(f.snapshotDir, f.regionDir+"/_medic_snapshots")
|
||||
qDir := orDefault(f.quarantineDir, f.regionDir+"/_medic_quarantine")
|
||||
res, err := regionmedic.ApplyHeal(plan, snapDir, qDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("apply heal: %w", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "HEALED %s: wrote %d bytes from %s; snapshot=%s; quarantined=%d\n",
|
||||
res.Region, res.BytesWritten, plan.Source.Backup.ID, res.SnapshotPath, len(res.QuarantinedFiles))
|
||||
printJSON(res)
|
||||
|
||||
// Start back up (panel-safe) + verify.
|
||||
if f.controller != "" && f.instance != "" {
|
||||
fmt.Fprintf(os.Stderr, "starting %s via panel...\n", f.instance)
|
||||
if err := panelAction(ctx, f.controller, f.instance, "start"); err != nil {
|
||||
return fmt.Errorf("panel start: %w", err)
|
||||
}
|
||||
if f.container != "" {
|
||||
fmt.Fprintln(os.Stderr, "watching boot for readiness + new corruption...")
|
||||
if err := watchBoot(ctx, f.container, f.regionDir, region, 240*time.Second); err != nil {
|
||||
return fmt.Errorf("post-heal verify: %w", err)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "VERIFY OK: instance ready, no new Wrong chunk header for the healed region")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func summarizePlan(plan regionmedic.HealPlan) {
|
||||
fmt.Fprintf(os.Stderr, "region %s: live healthy=%v (badChunks=%d, err=%q)\n",
|
||||
plan.Region, plan.LiveReport.Healthy(), len(plan.LiveReport.BadChunks), plan.LiveReport.Err)
|
||||
fmt.Fprintf(os.Stderr, "error_backups for region: %d\n", len(plan.ErrorBackups))
|
||||
for _, c := range plan.Evaluated {
|
||||
status := "no region in backup"
|
||||
if c.Found {
|
||||
if c.Report.Healthy() {
|
||||
status = fmt.Sprintf("CLEAN (%d chunks)", c.Report.PresentChunks)
|
||||
} else {
|
||||
status = fmt.Sprintf("corrupt (%d bad chunks, err=%q)", len(c.Report.BadChunks), c.Report.Err)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " %s %s %s\n", c.Backup.CreatedAt.Format(time.RFC3339), c.Backup.ID, status)
|
||||
}
|
||||
if plan.Ready() {
|
||||
fmt.Fprintf(os.Stderr, "=> heal source: %s (%s)\n", plan.Source.Backup.ID, plan.Source.Backup.CreatedAt.Format(time.RFC3339))
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "=> NO clean source found")
|
||||
}
|
||||
}
|
||||
|
||||
// --- orchestration helpers --------------------------------------------------
|
||||
|
||||
func panelAction(ctx context.Context, controller, instance, action string) error {
|
||||
url := strings.TrimRight(controller, "/") + "/api/instances/" + instance + "/" + action
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, bytes.TrimSpace(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitContainer polls `docker inspect` until the container's Running state
|
||||
// matches want, or timeout.
|
||||
func waitContainer(ctx context.Context, name string, want bool, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
out, _ := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.State.Running}}", name).Output()
|
||||
running := strings.TrimSpace(string(out)) == "true"
|
||||
if running == want {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for %s running=%v", name, want)
|
||||
}
|
||||
|
||||
// watchBoot tails the container log until the 7DTD ready marker appears, while
|
||||
// asserting no "Wrong chunk header!" reappears AND no new error_backup files
|
||||
// land for the healed region.
|
||||
func watchBoot(ctx context.Context, container, regionDir string, region regionmedic.RegionCoord, timeout time.Duration) error {
|
||||
if err := waitContainer(ctx, container, true, 60*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
readyRe := []string{"StartGame done", "GameStartDone", "by Telnet from"}
|
||||
for time.Now().Before(deadline) {
|
||||
out, _ := exec.CommandContext(ctx, "docker", "logs", "--since", "5m", container).CombinedOutput()
|
||||
s := string(out)
|
||||
if strings.Contains(s, "Wrong chunk header") {
|
||||
// Only fail if it's for our region — re-scan the dir to be sure.
|
||||
scan, _ := regionmedic.ScanRegionDir(regionDir)
|
||||
for _, a := range scan.Affected {
|
||||
if a.Region == region {
|
||||
return fmt.Errorf("healed region %s threw new error_backups (%d) after restart", region, a.ErrorBackups)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range readyRe {
|
||||
if strings.Contains(s, m) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(4 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for ready marker")
|
||||
}
|
||||
|
||||
func regionArg(s string) string {
|
||||
if strings.HasSuffix(s, ".7rg") {
|
||||
return s
|
||||
}
|
||||
return s + ".7rg"
|
||||
}
|
||||
|
||||
func orDefault(s, def string) string {
|
||||
if s == "" {
|
||||
return def
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user