// Command region-medic is a standalone operator tool that detects and heals // corrupted 7DTD ".7rg" region files from the panel's own backups, using the // shared github.com/dbledeez/panel/pkg/regionmedic engine. // // It is the same logic that ships inside the agent's RegionScan/RegionHeal RPCs, // exposed as a CLI so it can be run by hand against a stopped instance without // redeploying the controller. Subcommands: // // region-medic validate [-deep] // region-medic scan // region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339] // region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply // [-deep] [-not-after RFC3339] -snapshot-dir S -quarantine-dir Q // [-controller URL -instance ID] # stop before / start after (panel-safe) // [-container NAME] # post-start: watch for ready + new corruption // region-medic verify # count error_backups now // // stop/start go through the controller HTTP API (never raw docker), so panel // state stays correct; readiness/state polling reads docker locally. package main import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "os" "os/exec" "strings" "time" "github.com/dbledeez/panel/pkg/regionmedic" ) func main() { if len(os.Args) < 2 { usage() os.Exit(2) } var err error switch os.Args[1] { case "validate": err = cmdValidate(os.Args[2:]) case "scan": err = cmdScan(os.Args[2:]) case "plan": err = cmdHeal(os.Args[2:], false) case "heal": err = cmdHeal(os.Args[2:], true) case "verify": err = cmdVerify(os.Args[2:]) case "autoheal": err = cmdAutoheal(os.Args[2:]) case "-h", "--help", "help": usage() return default: fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1]) usage() os.Exit(2) } if err != nil { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) } } func usage() { fmt.Fprint(os.Stderr, `region-medic — detect & heal corrupt 7DTD .7rg regions from panel backups region-medic validate [-deep] region-medic scan region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339] region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply [-deep] [-not-after RFC3339] -snapshot-dir S -quarantine-dir Q [-controller URL -instance ID] [-container NAME] region-medic verify `) } func printJSON(v any) { b, _ := json.MarshalIndent(v, "", " ") fmt.Println(string(b)) } // --- validate --------------------------------------------------------------- func cmdValidate(args []string) error { deep := false var file string for _, a := range args { if a == "-deep" || a == "--deep" { deep = true } else if !strings.HasPrefix(a, "-") { file = a } } if file == "" { return fmt.Errorf("usage: region-medic validate [-deep]") } rep := regionmedic.ValidateRegionFile(file, deep) printJSON(rep) if rep.Healthy() { fmt.Fprintf(os.Stderr, "HEALTHY: %d present chunks, %d sectors\n", rep.PresentChunks, rep.TotalSectors) } else { fmt.Fprintf(os.Stderr, "UNHEALTHY: err=%q badChunks=%d\n", rep.Err, len(rep.BadChunks)) } return nil } // --- scan ------------------------------------------------------------------- func cmdScan(args []string) error { if len(args) < 1 { return fmt.Errorf("usage: region-medic scan ") } scan, err := regionmedic.ScanRegionDir(args[0]) if err != nil { return err } printJSON(scan) fmt.Fprintf(os.Stderr, "%d error_backups across %d affected region(s)\n", scan.ErrorBackups, len(scan.Affected)) return nil } // --- verify ----------------------------------------------------------------- func cmdVerify(args []string) error { if len(args) < 1 { return fmt.Errorf("usage: region-medic verify ") } scan, err := regionmedic.ScanRegionDir(args[0]) if err != nil { return err } fmt.Fprintf(os.Stderr, "error_backups now: %d (affected regions: %d)\n", scan.ErrorBackups, len(scan.Affected)) printJSON(scan.Affected) return nil } // --- plan / heal ------------------------------------------------------------ type healFlags struct { regionDir, region, backupsDir string snapshotDir, quarantineDir string deep, apply bool notAfter string controller, instance, container string } func parseHealFlags(args []string) (healFlags, error) { f := healFlags{} for i := 0; i < len(args); i++ { a := args[i] next := func() string { if i+1 < len(args) { i++ return args[i] } return "" } switch a { case "-region-dir": f.regionDir = next() case "-region": f.region = next() case "-backups-dir": f.backupsDir = next() case "-snapshot-dir": f.snapshotDir = next() case "-quarantine-dir": f.quarantineDir = next() case "-not-after": f.notAfter = next() case "-controller": f.controller = next() case "-instance": f.instance = next() case "-container": f.container = next() case "-deep", "--deep": f.deep = true case "-apply", "--apply": f.apply = true default: return f, fmt.Errorf("unknown flag %q", a) } } if f.regionDir == "" || f.region == "" || f.backupsDir == "" { return f, fmt.Errorf("required: -region-dir, -region, -backups-dir") } return f, nil } func cmdHeal(args []string, allowApply bool) error { f, err := parseHealFlags(args) if err != nil { return err } region, ok := regionmedic.ParseRegionFileName(regionArg(f.region)) if !ok { return fmt.Errorf("bad -region %q (want r.X.Z or r.X.Z.7rg)", f.region) } var notAfter time.Time if f.notAfter != "" { notAfter, err = time.Parse(time.RFC3339, f.notAfter) if err != nil { return fmt.Errorf("bad -not-after: %w", err) } } backups, err := regionmedic.ListPanelBackups(f.backupsDir) if err != nil { return fmt.Errorf("list backups: %w", err) } fmt.Fprintf(os.Stderr, "found %d backups in %s\n", len(backups), f.backupsDir) plan, err := regionmedic.PlanHeal(f.regionDir, region, backups, f.deep, notAfter) if err != nil { return err } summarizePlan(plan) if !allowApply || !f.apply { fmt.Fprintln(os.Stderr, "(dry run — re-run `heal ... -apply` to execute)") return nil } if !plan.Ready() { return fmt.Errorf("refusing to heal: no clean backup copy found for %s", region) } ctx := context.Background() // Orchestrate stop (panel-safe) if controller configured. if f.controller != "" && f.instance != "" { fmt.Fprintf(os.Stderr, "stopping %s via panel...\n", f.instance) if err := panelAction(ctx, f.controller, f.instance, "stop"); err != nil { return fmt.Errorf("panel stop: %w", err) } if f.container != "" { if err := waitContainer(ctx, f.container, false, 90*time.Second); err != nil { return fmt.Errorf("wait stopped: %w", err) } } else { time.Sleep(8 * time.Second) } } snapDir := orDefault(f.snapshotDir, f.regionDir+"/_medic_snapshots") qDir := orDefault(f.quarantineDir, f.regionDir+"/_medic_quarantine") res, err := regionmedic.ApplyHeal(plan, snapDir, qDir) if err != nil { return fmt.Errorf("apply heal: %w", err) } fmt.Fprintf(os.Stderr, "HEALED %s: wrote %d bytes from %s; snapshot=%s; quarantined=%d\n", res.Region, res.BytesWritten, plan.Source.Backup.ID, res.SnapshotPath, len(res.QuarantinedFiles)) printJSON(res) // Start back up (panel-safe) + verify. if f.controller != "" && f.instance != "" { fmt.Fprintf(os.Stderr, "starting %s via panel...\n", f.instance) if err := panelAction(ctx, f.controller, f.instance, "start"); err != nil { return fmt.Errorf("panel start: %w", err) } if f.container != "" { fmt.Fprintln(os.Stderr, "watching boot for readiness + new corruption...") if err := watchBoot(ctx, f.container, f.regionDir, region, 240*time.Second); err != nil { return fmt.Errorf("post-heal verify: %w", err) } fmt.Fprintln(os.Stderr, "VERIFY OK: instance ready, no new Wrong chunk header for the healed region") } } return nil } func summarizePlan(plan regionmedic.HealPlan) { fmt.Fprintf(os.Stderr, "region %s: live healthy=%v (badChunks=%d, err=%q)\n", plan.Region, plan.LiveReport.Healthy(), len(plan.LiveReport.BadChunks), plan.LiveReport.Err) fmt.Fprintf(os.Stderr, "error_backups for region: %d\n", len(plan.ErrorBackups)) for _, c := range plan.Evaluated { status := "no region in backup" if c.Found { if c.Report.Healthy() { status = fmt.Sprintf("CLEAN (%d chunks)", c.Report.PresentChunks) } else { status = fmt.Sprintf("corrupt (%d bad chunks, err=%q)", len(c.Report.BadChunks), c.Report.Err) } } fmt.Fprintf(os.Stderr, " %s %s %s\n", c.Backup.CreatedAt.Format(time.RFC3339), c.Backup.ID, status) } if plan.Ready() { fmt.Fprintf(os.Stderr, "=> heal source: %s (%s)\n", plan.Source.Backup.ID, plan.Source.Backup.CreatedAt.Format(time.RFC3339)) } else { fmt.Fprintln(os.Stderr, "=> NO clean source found") } } // --- orchestration helpers -------------------------------------------------- func panelAction(ctx context.Context, controller, instance, action string) error { url := strings.TrimRight(controller, "/") + "/api/instances/" + instance + "/" + action req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer resp.Body.Close() body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { return fmt.Errorf("HTTP %d: %s", resp.StatusCode, bytes.TrimSpace(body)) } return nil } // waitContainer polls `docker inspect` until the container's Running state // matches want, or timeout. func waitContainer(ctx context.Context, name string, want bool, timeout time.Duration) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { out, _ := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.State.Running}}", name).Output() running := strings.TrimSpace(string(out)) == "true" if running == want { return nil } time.Sleep(2 * time.Second) } return fmt.Errorf("timed out waiting for %s running=%v", name, want) } // watchBoot tails the container log until the 7DTD ready marker appears, while // asserting no "Wrong chunk header!" reappears AND no new error_backup files // land for the healed region. func watchBoot(ctx context.Context, container, regionDir string, region regionmedic.RegionCoord, timeout time.Duration) error { if err := waitContainer(ctx, container, true, 60*time.Second); err != nil { return err } deadline := time.Now().Add(timeout) readyRe := []string{"StartGame done", "GameStartDone", "by Telnet from"} for time.Now().Before(deadline) { out, _ := exec.CommandContext(ctx, "docker", "logs", "--since", "5m", container).CombinedOutput() s := string(out) if strings.Contains(s, "Wrong chunk header") { // Only fail if it's for our region — re-scan the dir to be sure. scan, _ := regionmedic.ScanRegionDir(regionDir) for _, a := range scan.Affected { if a.Region == region { return fmt.Errorf("healed region %s threw new error_backups (%d) after restart", region, a.ErrorBackups) } } } for _, m := range readyRe { if strings.Contains(s, m) { return nil } } time.Sleep(4 * time.Second) } return fmt.Errorf("timed out waiting for ready marker") } func regionArg(s string) string { if strings.HasSuffix(s, ".7rg") { return s } return s + ".7rg" } func orDefault(s, def string) string { if s == "" { return def } return s }