panel/agent/cmd/region-medic/main.go

// Command region-medic is a standalone operator tool that detects and heals
// corrupted 7DTD ".7rg" region files from the panel's own backups, using the
// shared github.com/dbledeez/panel/pkg/regionmedic engine.
//
// It is the same logic that ships inside the agent's RegionScan/RegionHeal RPCs,
// exposed as a CLI so it can be run by hand against a stopped instance without
// redeploying the controller. Subcommands:
//
//	region-medic validate <file.7rg> [-deep]
//	region-medic scan     <region-dir>
//	region-medic plan     -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
//	region-medic heal     -region-dir D -region r.X.Z -backups-dir B -apply
//	                      [-deep] [-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
//	                      [-controller URL -instance ID]   # stop before / start after (panel-safe)
//	                      [-container NAME]                # post-start: watch for ready + new corruption
//	region-medic verify   <region-dir>                    # count error_backups now
//
// stop/start go through the controller HTTP API (never raw docker), so panel
// state stays correct; readiness/state polling reads docker locally.
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/dbledeez/panel/pkg/regionmedic"
)

func main() {
	if len(os.Args) < 2 {
		usage()
		os.Exit(2)
	}
	var err error
	switch os.Args[1] {
	case "validate":
		err = cmdValidate(os.Args[2:])
	case "scan":
		err = cmdScan(os.Args[2:])
	case "plan":
		err = cmdHeal(os.Args[2:], false)
	case "heal":
		err = cmdHeal(os.Args[2:], true)
	case "verify":
		err = cmdVerify(os.Args[2:])
	case "autoheal":
		err = cmdAutoheal(os.Args[2:])
	case "-h", "--help", "help":
		usage()
		return
	default:
		fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1])
		usage()
		os.Exit(2)
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

func usage() {
	fmt.Fprint(os.Stderr, `region-medic — detect & heal corrupt 7DTD .7rg regions from panel backups

  region-medic validate <file.7rg> [-deep]
  region-medic scan     <region-dir>
  region-medic plan     -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
  region-medic heal     -region-dir D -region r.X.Z -backups-dir B -apply [-deep]
                        [-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
                        [-controller URL -instance ID] [-container NAME]
  region-medic verify   <region-dir>
`)
}

func printJSON(v any) {
	b, _ := json.MarshalIndent(v, "", "  ")
	fmt.Println(string(b))
}

// --- validate ---------------------------------------------------------------

func cmdValidate(args []string) error {
	deep := false
	var file string
	for _, a := range args {
		if a == "-deep" || a == "--deep" {
			deep = true
		} else if !strings.HasPrefix(a, "-") {
			file = a
		}
	}
	if file == "" {
		return fmt.Errorf("usage: region-medic validate <file.7rg> [-deep]")
	}
	rep := regionmedic.ValidateRegionFile(file, deep)
	printJSON(rep)
	if rep.Healthy() {
		fmt.Fprintf(os.Stderr, "HEALTHY: %d present chunks, %d sectors\n", rep.PresentChunks, rep.TotalSectors)
	} else {
		fmt.Fprintf(os.Stderr, "UNHEALTHY: err=%q badChunks=%d\n", rep.Err, len(rep.BadChunks))
	}
	return nil
}

// --- scan -------------------------------------------------------------------

func cmdScan(args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("usage: region-medic scan <region-dir>")
	}
	scan, err := regionmedic.ScanRegionDir(args[0])
	if err != nil {
		return err
	}
	printJSON(scan)
	fmt.Fprintf(os.Stderr, "%d error_backups across %d affected region(s)\n", scan.ErrorBackups, len(scan.Affected))
	return nil
}

// --- verify -----------------------------------------------------------------

func cmdVerify(args []string) error {
	if len(args) < 1 {
		return fmt.Errorf("usage: region-medic verify <region-dir>")
	}
	scan, err := regionmedic.ScanRegionDir(args[0])
	if err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "error_backups now: %d (affected regions: %d)\n", scan.ErrorBackups, len(scan.Affected))
	printJSON(scan.Affected)
	return nil
}

// --- plan / heal ------------------------------------------------------------

type healFlags struct {
	regionDir, region, backupsDir   string
	snapshotDir, quarantineDir      string
	deep, apply                     bool
	notAfter                        string
	controller, instance, container string
}

func parseHealFlags(args []string) (healFlags, error) {
	f := healFlags{}
	for i := 0; i < len(args); i++ {
		a := args[i]
		next := func() string {
			if i+1 < len(args) {
				i++
				return args[i]
			}
			return ""
		}
		switch a {
		case "-region-dir":
			f.regionDir = next()
		case "-region":
			f.region = next()
		case "-backups-dir":
			f.backupsDir = next()
		case "-snapshot-dir":
			f.snapshotDir = next()
		case "-quarantine-dir":
			f.quarantineDir = next()
		case "-not-after":
			f.notAfter = next()
		case "-controller":
			f.controller = next()
		case "-instance":
			f.instance = next()
		case "-container":
			f.container = next()
		case "-deep", "--deep":
			f.deep = true
		case "-apply", "--apply":
			f.apply = true
		default:
			return f, fmt.Errorf("unknown flag %q", a)
		}
	}
	if f.regionDir == "" || f.region == "" || f.backupsDir == "" {
		return f, fmt.Errorf("required: -region-dir, -region, -backups-dir")
	}
	return f, nil
}

func cmdHeal(args []string, allowApply bool) error {
	f, err := parseHealFlags(args)
	if err != nil {
		return err
	}
	region, ok := regionmedic.ParseRegionFileName(regionArg(f.region))
	if !ok {
		return fmt.Errorf("bad -region %q (want r.X.Z or r.X.Z.7rg)", f.region)
	}
	var notAfter time.Time
	if f.notAfter != "" {
		notAfter, err = time.Parse(time.RFC3339, f.notAfter)
		if err != nil {
			return fmt.Errorf("bad -not-after: %w", err)
		}
	}
	backups, err := regionmedic.ListPanelBackups(f.backupsDir)
	if err != nil {
		return fmt.Errorf("list backups: %w", err)
	}
	fmt.Fprintf(os.Stderr, "found %d backups in %s\n", len(backups), f.backupsDir)

	plan, err := regionmedic.PlanHeal(f.regionDir, region, backups, f.deep, notAfter)
	if err != nil {
		return err
	}
	summarizePlan(plan)

	if !allowApply || !f.apply {
		fmt.Fprintln(os.Stderr, "(dry run — re-run `heal ... -apply` to execute)")
		return nil
	}
	if !plan.Ready() {
		return fmt.Errorf("refusing to heal: no clean backup copy found for %s", region)
	}

	ctx := context.Background()
	// Orchestrate stop (panel-safe) if controller configured.
	if f.controller != "" && f.instance != "" {
		fmt.Fprintf(os.Stderr, "stopping %s via panel...\n", f.instance)
		if err := panelAction(ctx, f.controller, f.instance, "stop"); err != nil {
			return fmt.Errorf("panel stop: %w", err)
		}
		if f.container != "" {
			if err := waitContainer(ctx, f.container, false, 90*time.Second); err != nil {
				return fmt.Errorf("wait stopped: %w", err)
			}
		} else {
			time.Sleep(8 * time.Second)
		}
	}

	snapDir := orDefault(f.snapshotDir, f.regionDir+"/_medic_snapshots")
	qDir := orDefault(f.quarantineDir, f.regionDir+"/_medic_quarantine")
	res, err := regionmedic.ApplyHeal(plan, snapDir, qDir)
	if err != nil {
		return fmt.Errorf("apply heal: %w", err)
	}
	fmt.Fprintf(os.Stderr, "HEALED %s: wrote %d bytes from %s; snapshot=%s; quarantined=%d\n",
		res.Region, res.BytesWritten, plan.Source.Backup.ID, res.SnapshotPath, len(res.QuarantinedFiles))
	printJSON(res)

	// Start back up (panel-safe) + verify.
	if f.controller != "" && f.instance != "" {
		fmt.Fprintf(os.Stderr, "starting %s via panel...\n", f.instance)
		if err := panelAction(ctx, f.controller, f.instance, "start"); err != nil {
			return fmt.Errorf("panel start: %w", err)
		}
		if f.container != "" {
			fmt.Fprintln(os.Stderr, "watching boot for readiness + new corruption...")
			if err := watchBoot(ctx, f.container, f.regionDir, region, 240*time.Second); err != nil {
				return fmt.Errorf("post-heal verify: %w", err)
			}
			fmt.Fprintln(os.Stderr, "VERIFY OK: instance ready, no new Wrong chunk header for the healed region")
		}
	}
	return nil
}

func summarizePlan(plan regionmedic.HealPlan) {
	fmt.Fprintf(os.Stderr, "region %s: live healthy=%v (badChunks=%d, err=%q)\n",
		plan.Region, plan.LiveReport.Healthy(), len(plan.LiveReport.BadChunks), plan.LiveReport.Err)
	fmt.Fprintf(os.Stderr, "error_backups for region: %d\n", len(plan.ErrorBackups))
	for _, c := range plan.Evaluated {
		status := "no region in backup"
		if c.Found {
			if c.Report.Healthy() {
				status = fmt.Sprintf("CLEAN (%d chunks)", c.Report.PresentChunks)
			} else {
				status = fmt.Sprintf("corrupt (%d bad chunks, err=%q)", len(c.Report.BadChunks), c.Report.Err)
			}
		}
		fmt.Fprintf(os.Stderr, "  %s  %s  %s\n", c.Backup.CreatedAt.Format(time.RFC3339), c.Backup.ID, status)
	}
	if plan.Ready() {
		fmt.Fprintf(os.Stderr, "=> heal source: %s (%s)\n", plan.Source.Backup.ID, plan.Source.Backup.CreatedAt.Format(time.RFC3339))
	} else {
		fmt.Fprintln(os.Stderr, "=> NO clean source found")
	}
}

// --- orchestration helpers --------------------------------------------------

func panelAction(ctx context.Context, controller, instance, action string) error {
	url := strings.TrimRight(controller, "/") + "/api/instances/" + instance + "/" + action
	req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("HTTP %d: %s", resp.StatusCode, bytes.TrimSpace(body))
	}
	return nil
}

// waitContainer polls `docker inspect` until the container's Running state
// matches want, or timeout.
func waitContainer(ctx context.Context, name string, want bool, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		out, _ := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.State.Running}}", name).Output()
		running := strings.TrimSpace(string(out)) == "true"
		if running == want {
			return nil
		}
		time.Sleep(2 * time.Second)
	}
	return fmt.Errorf("timed out waiting for %s running=%v", name, want)
}

// watchBoot tails the container log until the 7DTD ready marker appears, while
// asserting no "Wrong chunk header!" reappears AND no new error_backup files
// land for the healed region.
func watchBoot(ctx context.Context, container, regionDir string, region regionmedic.RegionCoord, timeout time.Duration) error {
	if err := waitContainer(ctx, container, true, 60*time.Second); err != nil {
		return err
	}
	deadline := time.Now().Add(timeout)
	readyRe := []string{"StartGame done", "GameStartDone", "by Telnet from"}
	for time.Now().Before(deadline) {
		out, _ := exec.CommandContext(ctx, "docker", "logs", "--since", "5m", container).CombinedOutput()
		s := string(out)
		if strings.Contains(s, "Wrong chunk header") {
			// Only fail if it's for our region — re-scan the dir to be sure.
			scan, _ := regionmedic.ScanRegionDir(regionDir)
			for _, a := range scan.Affected {
				if a.Region == region {
					return fmt.Errorf("healed region %s threw new error_backups (%d) after restart", region, a.ErrorBackups)
				}
			}
		}
		for _, m := range readyRe {
			if strings.Contains(s, m) {
				return nil
			}
		}
		time.Sleep(4 * time.Second)
	}
	return fmt.Errorf("timed out waiting for ready marker")
}

func regionArg(s string) string {
	if strings.HasSuffix(s, ".7rg") {
		return s
	}
	return s + ".7rg"
}

func orDefault(s, def string) string {
	if s == "" {
		return def
	}
	return s
}