panel — open-source game server manager (public release)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

agent/internal/dispatch/ark_save_restore.go

@@ -0,0 +1,264 @@
+package dispatch
+
+// ArkSaveRestore: swap a SavedArks/<subdir>/ rolling backup into the
+// active <subdir>.ark slot. The previously-active save is moved aside
+// (NEVER deleted) — even abandoned attempts are recoverable from the
+// timestamped *_replaced.ark filename.
+//
+// Sequence (all server-side, atomic from the operator's POV):
+//   1. Validate inputs.
+//   2. Confirm the main container is NOT running. We refuse the swap
+//      if it is, because mid-write file ops in ARK's save dir corrupt
+//      the rolling backup chain.
+//   3. mv  <subdir>/<subdir>.ark  ->  <subdir>/<subdir>_<dd.mm.yyyy>_<HH.MM.SS>_replaced.ark
+//   4. cp  <subdir>/<target_filename>  ->  <subdir>/<subdir>.ark
+//      For .arkrbf rolling backups the dest still ends in .ark — the
+//      engine doesn't care about the source file's extension, only the
+//      slot name.
+//
+// Caller (controller) is responsible for stopping the container before
+// this RPC. If the operator started the container in the meantime,
+// this handler short-circuits cleanly with a friendly error.
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// arkSavedArksContainerRoot is the relative path to ARK's rolling-save
+// directory FROM the ark-sa module's BrowseableRoot. The module roots
+// browsing at ".../ShooterGame/Saved" (see modules/ark-sa/module.yaml),
+// so this is the segment beneath that — NOT the full container path.
+const arkSavedArksContainerRoot = "SavedArks"
+
+func (d *Dispatcher) handleArkSaveRestore(corrID string, req *panelv1.ArkSaveRestoreRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
+		return
+	}
+	subdir := strings.TrimSpace(req.SavedArksSubdir)
+	target := strings.TrimSpace(req.TargetFilename)
+	if subdir == "" || target == "" {
+		d.sendArkSaveRestoreResult(corrID, "", "", "saved_arks_subdir and target_filename are required")
+		return
+	}
+	// Path-safety: subdir is one segment under SavedArks/, target is
+	// one segment under that. Reject any traversal attempt up front.
+	if strings.ContainsAny(subdir, "/\\") || subdir == "." || subdir == ".." {
+		d.sendArkSaveRestoreResult(corrID, "", "", "saved_arks_subdir must be a single directory name")
+		return
+	}
+	if strings.ContainsAny(target, "/\\") || target == "." || target == ".." {
+		d.sendArkSaveRestoreResult(corrID, "", "", "target_filename must be a single file name")
+		return
+	}
+	// Sanity-check extension. Active slot is .ark; sources may be .ark
+	// or .arkrbf (rolling backup). .bak is ARK's anti-corruption file
+	// and we don't restore from it directly — operators rename to .ark
+	// manually if they need to.
+	lowerTarget := strings.ToLower(target)
+	if !strings.HasSuffix(lowerTarget, ".ark") && !strings.HasSuffix(lowerTarget, ".arkrbf") {
+		d.sendArkSaveRestoreResult(corrID, "", "", "target_filename must end in .ark or .arkrbf")
+		return
+	}
+
+	activeName := subdir + ".ark"
+	if target == activeName {
+		d.sendArkSaveRestoreResult(corrID, "", "", "target is already the active save")
+		return
+	}
+
+	// Stamp the aside name with current wall time. We use the user's
+	// "dd.mm.yyyy_HH.MM.SS" format from ARK's own rolling backups so
+	// the aside file sorts naturally next to genuine engine backups
+	// in the file browser. Suffix "_replaced" disambiguates them.
+	// Uses UTC because that's what the engine writes — keeps the
+	// frontend's parser (which assumes UTC) honest for both sources.
+	now := time.Now().UTC()
+	asideName := fmt.Sprintf("%s_%s_replaced.ark",
+		subdir, now.Format("02.01.2006_15.04.05"))
+
+	if d.useContainerOps(rec) {
+		d.arkRestoreInContainer(corrID, rec, subdir, target, activeName, asideName)
+		return
+	}
+	d.arkRestoreOnHost(corrID, rec, subdir, target, activeName, asideName)
+}
+
+func (d *Dispatcher) arkRestoreInContainer(corrID string, rec *instanceRecord, subdir, target, activeName, asideName string) {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	// Refuse if the main game container is still running. We don't
+	// want file ops happening inside SavedArks while the engine is
+	// writing rolling backups.
+	mainName := "panel-" + rec.InstanceID
+	if st, err := d.runtime.InspectByName(ctx, mainName); err == nil && st.Status == "running" {
+		d.sendArkSaveRestoreResult(corrID, "", "",
+			"refusing to restore while server is running — stop the server first")
+		return
+	}
+
+	subdirRel := path.Join(arkSavedArksContainerRoot, subdir)
+	subdirAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), subdirRel)
+	if err != nil {
+		d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
+		return
+	}
+	activeAbs := path.Join(subdirAbs, activeName)
+	asideAbs := path.Join(subdirAbs, asideName)
+	targetAbs := path.Join(subdirAbs, target)
+
+	targetID, err := d.getFsTargetContainerID(ctx, rec)
+	if err != nil {
+		d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
+		return
+	}
+
+	// Verify the target file exists before we touch the active slot.
+	// `test -f` returns exit 1 (no file), 0 (exists), other (error).
+	_, _, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		"test -f " + shellQuote(targetAbs)})
+	if err != nil {
+		d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
+		return
+	}
+	if code != 0 {
+		d.sendArkSaveRestoreResult(corrID, "", "", fmt.Sprintf("target save %q not found", target))
+		return
+	}
+
+	// Move the previously-active save aside (only if it exists; some
+	// ops may run on a freshly-installed instance with no active save).
+	finalAside := ""
+	_, _, code, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		"test -f " + shellQuote(activeAbs)})
+	if code == 0 {
+		_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+			"mv -- " + shellQuote(activeAbs) + " " + shellQuote(asideAbs)})
+		if err != nil {
+			d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
+			return
+		}
+		if code != 0 {
+			msg := strings.TrimSpace(string(stderr))
+			if msg == "" {
+				msg = fmt.Sprintf("mv aside exited %d", code)
+			}
+			d.sendArkSaveRestoreResult(corrID, "", "", msg)
+			return
+		}
+		finalAside = asideName
+	}
+
+	// Copy the chosen backup into the active slot. cp preserves the
+	// source so the operator can pick it again later if they change
+	// their mind. -p preserves mtime so the engine sees the original
+	// save's modification time.
+	_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		"cp -p -- " + shellQuote(targetAbs) + " " + shellQuote(activeAbs)})
+	if err != nil {
+		d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
+		return
+	}
+	if code != 0 {
+		msg := strings.TrimSpace(string(stderr))
+		if msg == "" {
+			msg = fmt.Sprintf("cp exited %d", code)
+		}
+		// Roll the aside back if cp failed mid-flight.
+		if finalAside != "" {
+			_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+				"mv -- " + shellQuote(asideAbs) + " " + shellQuote(activeAbs)})
+			finalAside = ""
+		}
+		d.sendArkSaveRestoreResult(corrID, finalAside, "", msg)
+		return
+	}
+	d.sendArkSaveRestoreResult(corrID, finalAside, activeName, "")
+}
+
+func (d *Dispatcher) arkRestoreOnHost(corrID string, rec *instanceRecord, subdir, target, activeName, asideName string) {
+	if rec.DataPath == "" {
+		d.sendArkSaveRestoreResult(corrID, "", "", "no file storage available")
+		return
+	}
+	subdirRel := filepath.Join(arkSavedArksContainerRoot, subdir)
+	subdirAbs, err := safeJoinHost(rec.DataPath, subdirRel)
+	if err != nil {
+		d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
+		return
+	}
+	activeAbs := filepath.Join(subdirAbs, activeName)
+	asideAbs := filepath.Join(subdirAbs, asideName)
+	targetAbs := filepath.Join(subdirAbs, target)
+
+	if _, err := os.Stat(targetAbs); err != nil {
+		d.sendArkSaveRestoreResult(corrID, "", "", fmt.Sprintf("target save %q not found", target))
+		return
+	}
+	finalAside := ""
+	if _, err := os.Stat(activeAbs); err == nil {
+		if err := os.Rename(activeAbs, asideAbs); err != nil {
+			d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
+			return
+		}
+		finalAside = asideName
+	}
+	in, err := os.Open(targetAbs)
+	if err != nil {
+		d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
+		return
+	}
+	defer in.Close()
+	out, err := os.OpenFile(activeAbs, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		// Roll aside back so we don't leave the server with no save.
+		if finalAside != "" {
+			_ = os.Rename(asideAbs, activeAbs)
+			finalAside = ""
+		}
+		d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
+		return
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		_ = out.Close()
+		if finalAside != "" {
+			_ = os.Remove(activeAbs)
+			_ = os.Rename(asideAbs, activeAbs)
+			finalAside = ""
+		}
+		d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
+		return
+	}
+	if err := out.Close(); err != nil {
+		d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
+		return
+	}
+	d.sendArkSaveRestoreResult(corrID, finalAside, activeName, "")
+}
+
+func (d *Dispatcher) sendArkSaveRestoreResult(corrID, asideFilename, installedFilename, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_ArkSaveRestoreResult{
+			ArkSaveRestoreResult: &panelv1.ArkSaveRestoreResult{
+				AsideFilename:     asideFilename,
+				InstalledFilename: installedFilename,
+				Error:             errMsg,
+			},
+		},
+	})
+}

agent/internal/dispatch/backup.go

@@ -0,0 +1,505 @@
+package dispatch
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	agentmodule "github.com/dbledeez/panel/agent/internal/module"
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	modulepkg "github.com/dbledeez/panel/pkg/module"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// buildBackupTarCmd assembles the tar command run inside the sidecar.
+// Manifest BackupSpec wins when Include is set: the tar runs against
+// the listed paths (each relative to BrowseableRoot, missing ones
+// silently skipped). Otherwise it falls back to a whole-tree archive
+// of BrowseableRoot. Excludes are passed through tar's --exclude.
+//
+// Resulting tarball preserves the directory layout under BrowseableRoot
+// so restore can extract straight into the same root.
+func buildBackupTarCmd(manifest *modulepkgManifest, root, fileName string, keep int) string {
+	if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
+		root = manifest.Backup.Root
+	}
+	var base string
+	if manifest == nil || manifest.Backup == nil || len(manifest.Backup.Include) == 0 {
+		// Fallback: whole tree.
+		base = fmt.Sprintf("cd %q && tar czf /backup/%s . && stat -c %%s /backup/%s",
+			root, shellEscape(fileName), shellEscape(fileName))
+	} else {
+		// Per-game include list. Missing paths are non-fatal — players will
+		// create new instances on a fresh server before the save dir exists,
+		// and we don't want their first Back-up-now to fail the run.
+		excludes := ""
+		for _, e := range manifest.Backup.Exclude {
+			excludes += " --exclude=" + shellEscape(e)
+		}
+		// Build "test -e <path> && echo <path>" for each include — only
+		// existing paths get added to the tar argv, so the operator doesn't
+		// see a confusing "Cannot stat: No such file" on first run.
+		//
+		// Newline-delimited (NOT NUL-delimited) because the sidecar image is
+		// alpine, which ships busybox tar — and busybox tar doesn't grok
+		// `--null`. Manifest paths never contain whitespace or newlines (we
+		// hand-curate them per game), so a plain newline list is safe.
+		var existsChecks []string
+		for _, p := range manifest.Backup.Include {
+			clean := strings.TrimPrefix(p, "./")
+			clean = strings.TrimPrefix(clean, "/")
+			existsChecks = append(existsChecks, fmt.Sprintf("[ -e %s ] && printf '%%s\\n' %s", shellEscape(clean), shellEscape(clean)))
+		}
+		listExpr := "{ " + strings.Join(existsChecks, " ; ") + " ; }"
+		base = fmt.Sprintf("cd %q && %s | tar czf /backup/%s%s -T - && stat -c %%s /backup/%s",
+			root, listExpr, shellEscape(fileName), excludes, shellEscape(fileName))
+	}
+	return base + backupPruneSuffix(keep)
+}
+
+// backupPruneSuffix appends a best-effort retention prune that keeps only the
+// newest `keep` *.tar.gz in /backup (by mtime) and deletes the rest. It runs in
+// the SAME root sidecar that just wrote the new tarball — the backup files are
+// root-owned, so the unprivileged agent can't prune them itself. keep<=0 leaves
+// pruning off. busybox-safe (alpine): ls -1t + tail -n +N + while-read — no
+// `head -n -N` (GNU-only) and no `xargs -r`.
+func backupPruneSuffix(keep int) string {
+	if keep <= 0 {
+		return ""
+	}
+	return fmt.Sprintf(" ; echo 'prune: keeping newest %d backups' ; ls -1t /backup/*.tar.gz 2>/dev/null | tail -n +%d | while IFS= read -r f; do echo \"prune: rm $f\" ; rm -f \"$f\" ; done",
+		keep, keep+1)
+}
+
+// readBackupKeep returns the backup retention for this instance: the integer in
+// <dataPath>/backup-keep when present, else the PANEL_BACKUP_KEEP env default,
+// else 10. A rolling cap stops the unbounded pileup that filled 90 GB on a busy
+// 7DTD season. Set per-instance by writing the file (the Backups UI exposes it).
+func readBackupKeep(dataPath string) int {
+	def := 10
+	if v := os.Getenv("PANEL_BACKUP_KEEP"); v != "" {
+		if n, e := strconv.Atoi(strings.TrimSpace(v)); e == nil && n > 0 {
+			def = n
+		}
+	}
+	b, err := os.ReadFile(filepath.Join(dataPath, "backup-keep"))
+	if err != nil {
+		return def
+	}
+	if n, e := strconv.Atoi(strings.TrimSpace(string(b))); e == nil && n > 0 {
+		return n
+	}
+	return def
+}
+
+// modulepkgManifest is a local alias so the helper signature below
+// doesn't drag modulepkg into its return type at the file top.
+type modulepkgManifest = modulepkg.Manifest
+
+// Backup / restore via sidecar containers. Two operations:
+//
+//   - Backup:  tar -czf /backup/<file>.tar.gz -C /src .   (volume readonly at /src)
+//   - Restore: tar -xzf /backup/<file>.tar.gz -C /dst      (volume read-write at /dst)
+//
+// The sidecar uses alpine:3.21 + busybox tar. Backup files live on the
+// agent host under $BACKUP_DIR/<instance_id>/<ts>-<bkpID>.tar.gz; both
+// sidecars bind-mount $BACKUP_DIR/<instance_id> at /backup.
+//
+// For restore we refuse if the main container is running: extracting
+// files under a live server would corrupt world state. Operator stops
+// first. (Symmetrical to the SteamCMD updater's running-container check.)
+
+const (
+	backupImage      = "alpine:3.21"
+	backupSidecarTTL = 30 * time.Minute
+)
+
+// ---- Backup ----
+
+func (d *Dispatcher) handleBackup(ctx context.Context, corrID string, req *panelv1.BackupRequest) {
+	log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
+
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendBackupResult(corrID, "", "", 0, err.Error())
+		return
+	}
+	if rec.BrowseableRoot == "" {
+		d.sendBackupResult(corrID, "", "", 0, "instance has no BrowseableRoot to tar (module declares no volumes)")
+		return
+	}
+	if d.backupDir == "" {
+		d.sendBackupResult(corrID, "", "", 0, "agent has no --backup-dir configured")
+		return
+	}
+
+	// Prepare the agent-side target directory.
+	bkpID := "bkp_" + randHex(8)
+	hostDir := filepath.Join(d.backupDir, req.InstanceId)
+	if err := os.MkdirAll(hostDir, 0o755); err != nil {
+		d.sendBackupResult(corrID, "", "", 0, fmt.Sprintf("mkdir %s: %s", hostDir, err.Error()))
+		return
+	}
+	ts := time.Now().UTC().Format("20060102-150405")
+	fileName := fmt.Sprintf("%s-%s.tar.gz", ts, bkpID)
+	hostFile := filepath.Join(hostDir, fileName)
+
+	log.Info("backup starting", "backup_id", bkpID, "host_file", hostFile)
+
+	// Build sidecar spec that mounts the same volumes as the instance,
+	// plus the host backup directory at /backup.
+	volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
+	// Mark volumes read-only for backup safety.
+	for i := range volumes {
+		volumes[i].ReadOnly = true
+	}
+	// Add the host backup dir as a (read-write) bind.
+	volumes = append(volumes, runtime.VolumeSpec{
+		Type:          "bind",
+		HostPath:      hostDir,
+		ContainerPath: "/backup",
+	})
+
+	// Per-game backup paths. If the manifest declares Backup.Include,
+	// only those paths get archived (saves-only — keeps ARK SA tarballs
+	// at ~250 MB instead of the 18 GB whole-install). Without a spec,
+	// fall back to the original "tar everything under BrowseableRoot"
+	// behavior so older modules don't regress.
+	manifest := d.modulesMustGet(rec.ModuleID)
+	keep := readBackupKeep(rec.DataPath)
+	tarCmd := buildBackupTarCmd(manifest, rec.BrowseableRoot, fileName, keep)
+	if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
+		log.Info("backup using manifest-declared paths", "include_count", len(manifest.Backup.Include))
+	} else {
+		log.Warn("backup falling back to whole BrowseableRoot — module manifest has no `backup.include`",
+			"module_id", rec.ModuleID, "root", rec.BrowseableRoot)
+	}
+	spec := runtime.InstanceSpec{
+		InstanceID:    req.InstanceId + "-backup",
+		IsSidecar:     true,
+		Image:         backupImage,
+		Entrypoint:    []string{"sh"},
+		Command:       []string{"-c", tarCmd},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		d.sendBackupResult(corrID, bkpID, "", 0, "create sidecar: "+err.Error())
+		return
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		d.sendBackupResult(corrID, bkpID, "", 0, "start sidecar: "+err.Error())
+		return
+	}
+	// Stream log lines for visibility.
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: rec.InstanceID, Stream: "backup",
+					At: timestamppb.Now(), Line: line,
+				}},
+			})
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		d.sendBackupResult(corrID, bkpID, "", 0, "wait: "+err.Error())
+		return
+	}
+	if exitCode != 0 {
+		d.sendBackupResult(corrID, bkpID, "", 0, fmt.Sprintf("sidecar exited %d", exitCode))
+		return
+	}
+
+	info, err := os.Stat(hostFile)
+	if err != nil {
+		d.sendBackupResult(corrID, bkpID, hostFile, 0, "stat result: "+err.Error())
+		return
+	}
+	log.Info("backup complete", "backup_id", bkpID, "size_bytes", info.Size())
+	d.sendBackupResult(corrID, bkpID, hostFile, info.Size(), "")
+}
+
+func (d *Dispatcher) sendBackupResult(corrID, bkpID, path string, size int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_BackupResult{
+			BackupResult: &panelv1.BackupResult{
+				BackupId:  bkpID,
+				Path:      path,
+				SizeBytes: size,
+				Error:     errMsg,
+			},
+		},
+	})
+}
+
+// ---- Wipe paths (used by the env-config recreate flow) ----
+//
+// Spawns an alpine sidecar that mounts the same volumes the instance
+// uses, runs `rm -rf` on each path (relative to BrowseableRoot), then
+// exits. Designed for the ARK map-switch flow — operator picked a new
+// SERVER_MAP and checked "wipe existing world data" so the old map's
+// SavedArks/<MapName>/ goes away. Generic enough that any future
+// env-config field can declare its own wipe paths.
+func (d *Dispatcher) runWipeSidecar(ctx context.Context, manifest *modulepkg.Manifest, instanceID, dataPath string, paths []string) error {
+	if manifest == nil {
+		return errors.New("nil manifest")
+	}
+	root := resolveBrowseableRoot(manifest)
+	if root == "" {
+		return errors.New("module declares no browseable_root — wipe needs a base path")
+	}
+	volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
+	// Build the rm -rf command. Each path is rooted under
+	// BrowseableRoot. Missing paths are silently skipped (first run).
+	var rmParts []string
+	for _, p := range paths {
+		clean := p
+		// Defensive: reject absolute paths and parent traversal so a
+		// rogue config_values value can't `rm -rf /` the container.
+		if strings.HasPrefix(clean, "/") || strings.Contains(clean, "..") {
+			return fmt.Errorf("refusing wipe of suspicious path %q", p)
+		}
+		full := root + "/" + clean
+		rmParts = append(rmParts, fmt.Sprintf("rm -rf %s", shellEscape(full)))
+	}
+	cmd := strings.Join(rmParts, " && ") + " && echo wiped"
+
+	spec := runtime.InstanceSpec{
+		InstanceID:    instanceID + "-wipe",
+		IsSidecar:     true,
+		Image:         backupImage,
+		Entrypoint:    []string{"sh"},
+		Command:       []string{"-c", cmd},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		return fmt.Errorf("create wipe sidecar: %w", err)
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		return fmt.Errorf("start wipe sidecar: %w", err)
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: instanceID, Stream: "wipe",
+					At: timestamppb.Now(), Line: line,
+				}},
+			})
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		return fmt.Errorf("wait wipe sidecar: %w", err)
+	}
+	if exitCode != 0 {
+		return fmt.Errorf("wipe sidecar exited %d", exitCode)
+	}
+	return nil
+}
+
+// ---- Restore ----
+
+func (d *Dispatcher) handleRestore(ctx context.Context, corrID string, req *panelv1.RestoreRequest) {
+	log := d.log.With("instance_id", req.InstanceId, "backup_id", req.BackupId, "correlation_id", corrID)
+
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendRestoreResult(corrID, 0, err.Error())
+		return
+	}
+	if rec.BrowseableRoot == "" {
+		d.sendRestoreResult(corrID, 0, "instance has no BrowseableRoot to restore into")
+		return
+	}
+	if req.BackupPath == "" {
+		d.sendRestoreResult(corrID, 0, "backup_path required")
+		return
+	}
+	// Refuse if container is running.
+	if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil && state.Status == "running" {
+		d.sendRestoreResult(corrID, 0, "stop the instance first — main container is running")
+		return
+	}
+	info, err := os.Stat(req.BackupPath)
+	if err != nil || info.IsDir() {
+		d.sendRestoreResult(corrID, 0, "backup file missing or is a directory")
+		return
+	}
+
+	volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
+	// Add the backup file's parent as a read-only bind at /backup.
+	hostDir := filepath.Dir(req.BackupPath)
+	fileName := filepath.Base(req.BackupPath)
+	volumes = append(volumes, runtime.VolumeSpec{
+		Type:          "bind",
+		HostPath:      hostDir,
+		ContainerPath: "/backup",
+		ReadOnly:      true,
+	})
+
+	// Extract strategy depends on whether the backup is per-game-paths
+	// or a whole-tree archive. When the manifest declares Backup.Include,
+	// only THOSE paths are wiped before extraction — preserves the
+	// 18 GB game install while still giving a clean save dir. Without a
+	// spec, fall back to wiping all of BrowseableRoot (legacy whole-tree
+	// behavior).
+	manifest := d.modulesMustGet(rec.ModuleID)
+	// Resolve the SAME root the backup wrote against. buildBackupTarCmd
+	// archives relative to manifest.Backup.Root when set (e.g. 7DTD's
+	// /game-saves, where world data actually lives — distinct from the
+	// /game install volume that is BrowseableRoot). Restore MUST extract
+	// into that same root or the tarball lands in the wrong volume and the
+	// live server never sees it (silent data loss: restore "succeeds" but
+	// players' bases are gone). Mirror the backup's resolution exactly.
+	restoreRoot := rec.BrowseableRoot
+	if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
+		restoreRoot = manifest.Backup.Root
+	}
+	var wipeCmd string
+	if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
+		// Wipe only the manifest-declared include paths so we don't
+		// nuke unrelated install files. Each path is rm -rf'd; missing
+		// ones are skipped silently (first-run case).
+		var parts []string
+		for _, p := range manifest.Backup.Include {
+			clean := strings.TrimPrefix(strings.TrimPrefix(p, "./"), "/")
+			parts = append(parts, fmt.Sprintf("rm -rf %s/%s", shellEscape(restoreRoot), shellEscape(clean)))
+		}
+		wipeCmd = strings.Join(parts, " ; ") + " 2>/dev/null"
+		log.Info("restore: wiping manifest-declared paths only", "root", restoreRoot, "include_count", len(manifest.Backup.Include))
+	} else {
+		wipeCmd = fmt.Sprintf("rm -rf %q/* %q/.[!.]* 2>/dev/null", restoreRoot, restoreRoot)
+		log.Warn("restore: wiping entire root (whole-tree backup)", "root", restoreRoot)
+	}
+	extractCmd := fmt.Sprintf(
+		"%s ; tar xzf /backup/%s -C %q && du -sb %q | cut -f1",
+		wipeCmd,
+		shellEscape(fileName),
+		restoreRoot,
+		restoreRoot,
+	)
+	spec := runtime.InstanceSpec{
+		InstanceID:    req.InstanceId + "-restore",
+		IsSidecar:     true,
+		Image:         backupImage,
+		Entrypoint:    []string{"sh"},
+		Command:       []string{"-c", extractCmd},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		d.sendRestoreResult(corrID, 0, "create sidecar: "+err.Error())
+		return
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		d.sendRestoreResult(corrID, 0, "start sidecar: "+err.Error())
+		return
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: rec.InstanceID, Stream: "restore",
+					At: timestamppb.Now(), Line: line,
+				}},
+			})
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		d.sendRestoreResult(corrID, 0, "wait: "+err.Error())
+		return
+	}
+	if exitCode != 0 {
+		d.sendRestoreResult(corrID, 0, fmt.Sprintf("sidecar exited %d", exitCode))
+		return
+	}
+	log.Info("restore complete")
+	d.sendRestoreResult(corrID, info.Size(), "")
+}
+
+func (d *Dispatcher) sendRestoreResult(corrID string, bytesRestored int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_RestoreResult{
+			RestoreResult: &panelv1.RestoreResult{BytesRestored: bytesRestored, Error: errMsg},
+		},
+	})
+}
+
+// ---- helpers ----
+
+func randHex(n int) string {
+	b := make([]byte, n)
+	_, _ = rand.Read(b)
+	return hex.EncodeToString(b)
+}
+
+// shellEscape wraps a filename in single quotes for safe inclusion in an
+// sh -c command. Assumes filename never contains a single-quote (ours
+// never do — timestamps + bkp_<hex>).
+func shellEscape(s string) string { return "'" + s + "'" }
+
+// modulesMustGet returns the dispatcher's manifest for the given id, or
+// nil if it's gone missing (shouldn't happen for a live instance).
+func (d *Dispatcher) modulesMustGet(id string) *modulepkgManifest {
+	m, _ := d.modules.Get(id)
+	return m
+}
+
+// Keep errors imported for future error-wrapping.
+var _ = errors.New

agent/internal/dispatch/backup_browse.go

@@ -0,0 +1,198 @@
+package dispatch
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// Backup browse — list / read entries inside a backup tarball without
+// extracting it. The agent has direct host access to the .tar.gz under
+// $BACKUP_DIR, so we don't need a sidecar; standard library archive/tar
+// + compress/gzip handles both the list and the single-file extract.
+//
+// Sized for safety: BackupReadFile caps the in-memory buffer per-call
+// (default 8 MiB) so a malicious or corrupt tar entry can't OOM the
+// agent. Truncation is reported back to the caller so the UI can show
+// "(truncated, download to see full file)".
+
+const (
+	defaultBackupReadCap = 8 << 20  // 8 MiB
+	maxBackupReadCap     = 64 << 20 // 64 MiB hard ceiling regardless of caller request
+	binarySniffLen       = 8 << 10  // 8 KiB
+)
+
+func (d *Dispatcher) handleBackupListEntries(corrID string, req *panelv1.BackupListEntriesRequest) {
+	d.log.Info("backup list entries: start", "correlation_id", corrID, "backup_path", req.BackupPath, "instance_id", req.InstanceId)
+	if req.BackupPath == "" {
+		d.sendBackupListEntriesResult(corrID, nil, 0, "backup_path required")
+		return
+	}
+	f, err := os.Open(req.BackupPath)
+	if err != nil {
+		d.sendBackupListEntriesResult(corrID, nil, 0, "open backup: "+err.Error())
+		return
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		d.sendBackupListEntriesResult(corrID, nil, 0, "gzip: "+err.Error())
+		return
+	}
+	defer gz.Close()
+	tr := tar.NewReader(gz)
+
+	var entries []*panelv1.BackupTarEntry
+	var totalBytes int64
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			d.sendBackupListEntriesResult(corrID, nil, 0, "tar header: "+err.Error())
+			return
+		}
+		isDir := hdr.Typeflag == tar.TypeDir || strings.HasSuffix(hdr.Name, "/")
+		clean := strings.TrimPrefix(hdr.Name, "./")
+		clean = strings.TrimPrefix(clean, "/")
+		entry := &panelv1.BackupTarEntry{
+			Path:    clean,
+			Size:    hdr.Size,
+			IsDir:   isDir,
+			Mode:    hdr.FileInfo().Mode().String(),
+			ModTime: timestamppb.New(hdr.ModTime),
+		}
+		entries = append(entries, entry)
+		if !isDir {
+			totalBytes += hdr.Size
+		}
+	}
+	d.log.Info("backup list entries: ok", "correlation_id", corrID, "count", len(entries), "total_bytes", totalBytes)
+	d.sendBackupListEntriesResult(corrID, entries, totalBytes, "")
+}
+
+func (d *Dispatcher) handleBackupReadFile(corrID string, req *panelv1.BackupReadFileRequest) {
+	if req.BackupPath == "" {
+		d.sendBackupReadFileResult(corrID, nil, 0, false, false, "backup_path required")
+		return
+	}
+	if req.EntryPath == "" {
+		d.sendBackupReadFileResult(corrID, nil, 0, false, false, "entry_path required")
+		return
+	}
+	maxRead := req.MaxBytes
+	if maxRead <= 0 {
+		maxRead = defaultBackupReadCap
+	}
+	if maxRead > maxBackupReadCap {
+		maxRead = maxBackupReadCap
+	}
+
+	f, err := os.Open(req.BackupPath)
+	if err != nil {
+		d.sendBackupReadFileResult(corrID, nil, 0, false, false, "open backup: "+err.Error())
+		return
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		d.sendBackupReadFileResult(corrID, nil, 0, false, false, "gzip: "+err.Error())
+		return
+	}
+	defer gz.Close()
+	tr := tar.NewReader(gz)
+
+	// Match by exact path; tarballs sometimes prefix entries with "./" so
+	// also accept that variant.
+	want := strings.TrimPrefix(req.EntryPath, "/")
+	wantAlt := "./" + want
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			d.sendBackupReadFileResult(corrID, nil, 0, false, false, fmt.Sprintf("entry %q not found in backup", req.EntryPath))
+			return
+		}
+		if err != nil {
+			d.sendBackupReadFileResult(corrID, nil, 0, false, false, "tar header: "+err.Error())
+			return
+		}
+		if hdr.Name != want && hdr.Name != wantAlt {
+			continue
+		}
+		if hdr.Typeflag == tar.TypeDir || strings.HasSuffix(hdr.Name, "/") {
+			d.sendBackupReadFileResult(corrID, nil, hdr.Size, false, false, "entry is a directory")
+			return
+		}
+		// Read up to maxRead+1 to detect truncation cleanly.
+		buf := &bytes.Buffer{}
+		_, err = io.Copy(buf, io.LimitReader(tr, maxRead+1))
+		if err != nil {
+			d.sendBackupReadFileResult(corrID, nil, hdr.Size, false, false, "read entry: "+err.Error())
+			return
+		}
+		truncated := false
+		content := buf.Bytes()
+		if int64(len(content)) > maxRead {
+			content = content[:maxRead]
+			truncated = true
+		}
+		isBinary := looksBinary(content)
+		d.sendBackupReadFileResult(corrID, content, hdr.Size, truncated, isBinary, "")
+		return
+	}
+}
+
+// looksBinary returns true if the first binarySniffLen bytes contain a
+// NUL byte. Standard heuristic — text files don't contain NULs, binaries
+// almost always do (executables, images, compressed save formats).
+func looksBinary(b []byte) bool {
+	limit := binarySniffLen
+	if len(b) < limit {
+		limit = len(b)
+	}
+	for i := 0; i < limit; i++ {
+		if b[i] == 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func (d *Dispatcher) sendBackupListEntriesResult(corrID string, entries []*panelv1.BackupTarEntry, totalBytes int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_BackupListEntriesResult{
+			BackupListEntriesResult: &panelv1.BackupListEntriesResult{
+				Entries:    entries,
+				TotalBytes: totalBytes,
+				Error:      errMsg,
+			},
+		},
+	})
+}
+
+func (d *Dispatcher) sendBackupReadFileResult(corrID string, content []byte, size int64, truncated, isBinary bool, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_BackupReadFileResult{
+			BackupReadFileResult: &panelv1.BackupReadFileResult{
+				Content:   content,
+				Size:      size,
+				Truncated: truncated,
+				IsBinary:  isBinary,
+				Error:     errMsg,
+			},
+		},
+	})
+}

agent/internal/dispatch/basemods.go

@@ -0,0 +1,206 @@
+package dispatch
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	agentmodule "github.com/dbledeez/panel/agent/internal/module"
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	"github.com/dbledeez/panel/agent/internal/updater"
+	modulepkg "github.com/dbledeez/panel/pkg/module"
+)
+
+// installBaseMods runs a SteamCMD validate (if the module has a steamcmd
+// update provider) followed by an alpine sidecar that downloads + installs
+// the third-party 7DTD base-mod set:
+//
+//   - Allocs Server Fixes — tarball from illy.bz (canonical upstream).
+//     Contains Allocs_CommandExtensions + Allocs_WebAndMapRendering +
+//     Alloc_XmlPatch, all under a Mods/ prefix when extracted to /game.
+//   - PrismaCore — latest GitHub release .zip from Prisma501/PrismaCore.
+//
+// SteamCMD validate step (new):
+//   - If the module declares at least one update provider with kind="steamcmd",
+//     the first such provider is used to run `app_update <app_id> validate`
+//     with anonymous login BEFORE the alpine sidecar runs. This re-downloads
+//     or restores stock TFP mods (TFP_*, 0_TFP_*, xMarkers) that ship with
+//     the dedicated server install.
+//   - If the SteamCMD validate fails, the function returns an error and does
+//     NOT proceed to the alpine step.
+//   - If there is no steamcmd provider (e.g., Conan Exiles), the validate is
+//     skipped and the function goes straight to the alpine path.
+//
+// What the alpine step does NOT touch (and does not need to):
+//   - TFP_* / 0_TFP_* — shipped by SteamCMD as part of the dedicated
+//     server install. TFP_Harmony is the runtime patcher; wiping it
+//     would brick the engine. Always present in /game/Mods/ after a
+//     fresh `app_update 294420`.
+//   - xMarkers — also ships with the SteamCMD 7DTD install per the
+//     operator (2026-05-04). Already in /game/Mods/ after install.
+//   - Custom mods (RefugeBot, AGF-VP-*, AsylumRoboticInbox, anything not
+//     in the recognized base-mod name set). Those are operator territory.
+//
+// Behavior on existing installs: any directory with a name that matches
+// the incoming mod gets renamed to `<name>.bak-<UTCstamp>` first so a
+// re-run is reversible. Direct port of the safety pattern in
+// refugebotserver's DependencyInstaller.cs.
+//
+// Caller responsibility: the instance container MUST be stopped before
+// calling this — the game holds DLLs in /game/Mods open while running,
+// and rename-on-extract will fail or corrupt under that.
+func (d *Dispatcher) installBaseMods(ctx context.Context, instanceID string, manifest *modulepkg.Manifest, dataPath string) error {
+	if manifest == nil || len(manifest.UpdateProviders) == 0 {
+		return errors.New("install-base-mods: module has no update provider")
+	}
+	installPath := manifest.UpdateProviders[0].InstallPath
+	if installPath == "" {
+		installPath = manifest.Runtime.Docker.BrowseableRoot
+	}
+	if installPath == "" {
+		return errors.New("install-base-mods: install_path missing on module")
+	}
+	volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
+	gameVol := volumeNameForPath(volumes, installPath)
+	if gameVol == "" {
+		return fmt.Errorf("install-base-mods: install_path %q not in module volumes", installPath)
+	}
+
+	// ── SteamCMD validate step ──────────────────────────────────────────
+	// If the module has a steamcmd update provider, run app_update validate
+	// with anonymous login FIRST to re-download/restore stock TFP mods.
+	// This ensures TFP_*, 0_TFP_*, and xMarkers are present before the
+	// alpine sidecar installs third-party base mods.
+	var steamcmdSpec *modulepkg.UpdateProvider
+	for i := range manifest.UpdateProviders {
+		if manifest.UpdateProviders[i].Kind == "steamcmd" {
+			steamcmdSpec = &manifest.UpdateProviders[i]
+			break
+		}
+	}
+	if steamcmdSpec != nil {
+		d.log.Info("install-base-mods: steamcmd validate starting", "instance_id", instanceID, "app_id", steamcmdSpec.AppID)
+		prov, err := updater.Get("steamcmd")
+		if err != nil {
+			return fmt.Errorf("install-base-mods: get steamcmd provider: %w", err)
+		}
+		// Build a log sink that streams through the same "install-base-mods log" logger.
+		sink := func(line string) {
+			d.log.Info("install-base-mods log", "instance_id", instanceID, "line", line)
+		}
+		uc := &updater.Context{
+			InstanceID:     instanceID,
+			Manifest:       manifest,
+			BrowseableRoot: installPath,
+			DataPath:       dataPath,
+			Runtime:        d.runtime,
+			Log:            sink,
+			SteamUsername:  "", // anonymous login
+			SteamPassword:  "", // anonymous login
+		}
+		if err := prov.Update(ctx, uc, steamcmdSpec); err != nil {
+			return fmt.Errorf("basemods: steamcmd validate failed: %w", err)
+		}
+		d.log.Info("install-base-mods: steamcmd validate succeeded", "instance_id", instanceID)
+	}
+
+	// alpine:3.19 is a 7 MB image with apk available. apk add fetches
+	// curl + unzip + tar + jq in ~5s on a warm cache. Cheaper than
+	// shipping our own base image and good enough for occasional runs.
+	script := `set -e
+apk add --no-cache --quiet curl tar unzip jq ca-certificates >/dev/null
+STAMP=$(date -u +%Y%m%d%H%M%S)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+backup_if_exists() {
+  for d in "$@"; do
+    if [ -d "$d" ] && [ ! -L "$d" ]; then
+      mv "$d" "$d.bak-$STAMP" 2>/dev/null || true
+    fi
+  done
+}
+
+# ── Allocs Server Fixes ───────────────────────────────────────────────
+echo "[basemods] downloading Allocs from illy.bz"
+curl -fsSL -o "$TMP/allocs.tgz" "https://illy.bz/fi/7dtd/server_fixes.tar.gz"
+# Magic-byte check: gzip starts with 0x1f 0x8b — protects against
+# CDN-served HTML 404 pages landing as a "valid" archive.
+head -c 2 "$TMP/allocs.tgz" | od -An -tx1 | tr -d ' \n' | grep -q '^1f8b' \
+  || { echo "[basemods] Allocs download wasn't gzip — aborting"; exit 1; }
+backup_if_exists /game/Mods/Allocs_CommandExtensions /game/Mods/Allocs_WebAndMapRendering /game/Mods/Alloc_XmlPatch
+tar -xzf "$TMP/allocs.tgz" -C /game
+echo "[basemods] Allocs installed"
+
+# ── PrismaCore (latest GitHub release zip) ─────────────────────────────
+echo "[basemods] resolving latest PrismaCore release"
+PRISMA_JSON="$TMP/prisma.json"
+curl -fsSL -A "panel-basemods/1.0" -H "Accept: application/vnd.github+json" \
+  -o "$PRISMA_JSON" "https://api.github.com/repos/Prisma501/PrismaCore/releases/latest"
+PRISMA_TAG=$(jq -r '.tag_name // empty' "$PRISMA_JSON")
+PRISMA_URL=$(jq -r '.assets[]? | select(.name | test("\\.zip$"; "i")) | .browser_download_url' "$PRISMA_JSON" | head -1)
+if [ -z "$PRISMA_URL" ]; then
+  echo "[basemods] no PrismaCore .zip asset — skipping"
+else
+  echo "[basemods] downloading PrismaCore $PRISMA_TAG from $PRISMA_URL"
+  curl -fsSL -A "panel-basemods/1.0" -o "$TMP/prisma.zip" "$PRISMA_URL"
+  head -c 2 "$TMP/prisma.zip" | od -An -tx1 | tr -d ' \n' | grep -q '^504b' \
+    || { echo "[basemods] PrismaCore download wasn't a zip — aborting"; exit 1; }
+  for d in /game/Mods/Prisma*; do backup_if_exists "$d"; done
+  unzip -o -q "$TMP/prisma.zip" -d /game/Mods/
+  echo "[basemods] PrismaCore installed"
+fi
+
+# Note: TFP_* and xMarkers are shipped as part of the SteamCMD dedicated
+# server install (app 294420). They're already in /game/Mods/ after a
+# fresh install and we don't touch them here — wiping TFP_Harmony would
+# brick the engine.
+echo BASEMODS_OK
+`
+
+	sidecarID := fmt.Sprintf("%s-basemods", instanceID)
+	spec := runtime.InstanceSpec{
+		InstanceID: sidecarID,
+		IsSidecar:  true,
+		Image:      "alpine:3.19",
+		Command:    []string{"sh", "-c", script},
+		Volumes: []runtime.VolumeSpec{
+			{Type: "volume", VolumeName: gameVol, ContainerPath: installPath},
+		},
+		RestartPolicy: "no",
+	}
+	d.log.Info("install-base-mods: starting sidecar", "instance_id", instanceID, "vol", gameVol)
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		return fmt.Errorf("install-base-mods: create: %w", err)
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		return fmt.Errorf("install-base-mods: start: %w", err)
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	defer cancelLogs()
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.log.Info("install-base-mods log", "instance_id", instanceID, "line", line)
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		return fmt.Errorf("install-base-mods: wait: %w", err)
+	}
+	if exitCode != 0 {
+		return fmt.Errorf("install-base-mods: sidecar exited %d", exitCode)
+	}
+	d.log.Info("install-base-mods: success", "instance_id", instanceID)
+	return nil
+}

agent/internal/dispatch/clusterplayer.go

@@ -0,0 +1,143 @@
+package dispatch
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Cluster player-save snapshots.
+//
+// A 7DTD cluster shares ONE Player/ folder across every member server (each
+// member's active-world Player dir is a symlink to the shared /cluster/Player
+// bind). That shared dir holds every character's .ttp (inventory, skills,
+// position, quests) — the single most irreplaceable per-player state. But it
+// is NOT captured by the panel tar backup: the backup tars /game-saves, where
+// the active Player dir is a symlink pointing OUT to /cluster, so tar stores
+// the dangling symlink, not the contents. Region Medic only heals world
+// regions, not player files. Net result before this: one corrupt write to a
+// .ttp and that character was gone with no restore path.
+//
+// Fix (mirrors the region-medic rolling-snapshot pattern): on every 7DTD
+// start, if the instance is clustered, copy the shared Player dir to a fresh
+// rolling slot keyed by cluster id and prune to keep N. All members write to
+// the same per-cluster rolling history, so a member's 4×/day restart cadence
+// gives a dense set of restore points. Best-effort: never blocks the start.
+
+func clusterPlayerRollingRoot() string {
+	return medicEnvOr("PANEL_CLUSTER_PLAYER_ROLLING_DIR", "/home/refuge/cluster-player-rolling")
+}
+
+func clusterPlayerKeep() int {
+	if v := os.Getenv("PANEL_CLUSTER_PLAYER_KEEP"); v != "" {
+		if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && n > 0 {
+			return n
+		}
+	}
+	return 12
+}
+
+// snapshotClusterPlayerOnStart resolves the instance's /cluster bind to its
+// host path, and if it's a real shared bind holding player data, snapshots
+// <bind>/Player into a rolling slot keyed by the cluster id (= the bind dir's
+// basename, e.g. cl_37e3a7f72cdc). No-op for standalone servers (whose
+// /cluster is an empty per-instance named volume, not a bind).
+func (d *Dispatcher) snapshotClusterPlayerOnStart(ctx context.Context, rec *instanceRecord) {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer cancel()
+	log := d.log.With("instance_id", rec.InstanceID, "cluster_player_snapshot", true)
+
+	mounts, err := d.runtime.ContainerMounts(ctx, "panel-"+rec.InstanceID)
+	if err != nil {
+		log.Warn("cluster-player snapshot: inspect mounts failed (skipping)", "err", err)
+		return
+	}
+	var bind string
+	for _, m := range mounts {
+		if m.Destination == "/cluster" && m.Type == "bind" && m.Source != "" {
+			bind = m.Source
+			break
+		}
+	}
+	if bind == "" {
+		return // standalone server (named-volume /cluster) — nothing shared to snapshot
+	}
+	playerDir := filepath.Join(bind, "Player")
+	if st, err := os.Stat(playerDir); err != nil || !st.IsDir() {
+		return // cluster joined but no Player dir yet (first boot)
+	}
+	if empty, _ := dirIsEmpty(playerDir); empty {
+		return
+	}
+
+	clusterID := filepath.Base(bind)
+	destRoot := filepath.Join(clusterPlayerRollingRoot(), clusterID)
+	if err := os.MkdirAll(destRoot, 0o755); err != nil {
+		log.Warn("cluster-player snapshot: mkdir rolling root failed", "err", err)
+		return
+	}
+	slot := filepath.Join(destRoot, "slot-"+strconv.FormatInt(time.Now().Unix(), 10))
+	dst := filepath.Join(slot, "Player")
+	if err := os.MkdirAll(slot, 0o755); err != nil {
+		log.Warn("cluster-player snapshot: mkdir slot failed", "err", err)
+		return
+	}
+	// cp -a preserves perms/acls/timestamps and the dir tree (player subdirs
+	// of .7rm map files). figaro is Linux; cp is always present. Best-effort.
+	cmd := exec.CommandContext(ctx, "cp", "-a", playerDir+"/.", dst)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		log.Warn("cluster-player snapshot: copy failed", "err", err, "out", strings.TrimSpace(string(out)))
+		_ = os.RemoveAll(slot) // don't leave a half-copied slot that prune would keep
+		return
+	}
+	keep := clusterPlayerKeep()
+	pruned := pruneRollingSlots(destRoot, keep)
+	log.Info("cluster-player snapshot taken", "cluster", clusterID, "slot", filepath.Base(slot), "keep", keep, "pruned", pruned)
+}
+
+// dirIsEmpty reports whether dir has no entries.
+func dirIsEmpty(dir string) (bool, error) {
+	f, err := os.Open(dir)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+	names, err := f.Readdirnames(1)
+	if err != nil {
+		// io.EOF → empty.
+		return len(names) == 0, nil
+	}
+	return len(names) == 0, nil
+}
+
+// pruneRollingSlots keeps the newest `keep` slot-* dirs under root (by name,
+// which embeds a unix timestamp so lexical == chronological) and removes the
+// rest. Returns how many it deleted.
+func pruneRollingSlots(root string, keep int) int {
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		return 0
+	}
+	var slots []string
+	for _, e := range entries {
+		if e.IsDir() && strings.HasPrefix(e.Name(), "slot-") {
+			slots = append(slots, e.Name())
+		}
+	}
+	if len(slots) <= keep {
+		return 0
+	}
+	sort.Strings(slots) // oldest first
+	removed := 0
+	for _, name := range slots[:len(slots)-keep] {
+		if err := os.RemoveAll(filepath.Join(root, name)); err == nil {
+			removed++
+		}
+	}
+	return removed
+}

agent/internal/dispatch/dayzmods.go

@@ -0,0 +1,275 @@
+package dispatch
+
+// DayZ mod install/uninstall + FsSymlink handlers. The controller drives
+// the high-level workflow (SteamCMD workshop_download runs controller-side,
+// writing to the shared panel-dayz-workshop volume); the agent's job is to
+// wire the downloaded content into the instance's /game tree:
+//
+//   1. Create /game/<folder_name> as a symlink to
+//      /game/steamapps/workshop/content/221100/<workshop_id>
+//   2. Copy every .bikey from that mod's keys/ (or Keys/) subfolder into
+//      /game/keys/ so BattlEye signs mod content on boot.
+//   3. On uninstall, remove the symlink + every bikey whose content hash
+//      matches a bikey in the workshop mod's keys dir (so we don't remove
+//      keys that belong to OTHER installed mods of the same vendor).
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// ---- FsSymlink ----
+//
+// The symlink is created inside the instance's container namespace so that
+// any `/game/@Mod` symlink resolves correctly from the DayZServer binary's
+// perspective.
+
+func (d *Dispatcher) handleFsSymlink(corrID string, req *panelv1.FsSymlinkRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsSymlinkResult(corrID, err.Error())
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	targetID, err := d.getFsTargetContainerID(ctx, rec)
+	if err != nil {
+		d.sendFsSymlinkResult(corrID, err.Error())
+		return
+	}
+	// `ln -sfn` replaces an existing symlink atomically. We require
+	// abs paths for both target and link_path and refuse anything outside
+	// the configured browseable roots for link_path (the symlink itself;
+	// target can point outside for workshop shares).
+	if !strings.HasPrefix(req.LinkPath, "/") || !strings.HasPrefix(req.Target, "/") {
+		d.sendFsSymlinkResult(corrID, "target and link_path must be absolute")
+		return
+	}
+	if _, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.LinkPath); err != nil {
+		d.sendFsSymlinkResult(corrID, "link_path outside browseable roots: "+err.Error())
+		return
+	}
+	// `ln` errors with "No such file or directory" if the parent doesn't
+	// exist yet — common on first mod install for modules whose entrypoint
+	// hasn't lazily created the Mods/ folder yet (Conan Exiles). `mkdir -p`
+	// the parent before linking so the symlink-then-game-restart flow works
+	// even on a fresh recreate. Idempotent: mkdir on existing dir is a
+	// no-op, and the safeJoinAny check above already proved link_path is
+	// inside a declared root so we're not creating dirs in arbitrary spots.
+	parentDir := path.Dir(req.LinkPath)
+	_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		fmt.Sprintf("mkdir -p %q && ln -sfn %q %q", parentDir, req.Target, req.LinkPath)})
+	if err != nil {
+		d.sendFsSymlinkResult(corrID, err.Error())
+		return
+	}
+	if code != 0 {
+		d.sendFsSymlinkResult(corrID, fmt.Sprintf("ln exited %d: %s", code, strings.TrimSpace(string(stderr))))
+		return
+	}
+	d.sendFsSymlinkResult(corrID, "")
+}
+
+func (d *Dispatcher) sendFsSymlinkResult(corrID, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsSymlinkResult{
+			FsSymlinkResult: &panelv1.FsSymlinkResult{Error: errMsg},
+		},
+	})
+}
+
+// ---- DayzModInstall ----
+
+func (d *Dispatcher) handleDayzModInstall(corrID string, req *panelv1.DayzModInstallRequest) {
+	d.log.Info("dayz mod install", "instance_id", req.InstanceId, "workshop_id", req.WorkshopId, "folder", req.FolderName)
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendDayzModInstallResult(corrID, err.Error(), nil, "")
+		return
+	}
+	if req.WorkshopId == "" || req.FolderName == "" {
+		d.sendDayzModInstallResult(corrID, "workshop_id and folder_name are required", nil, "")
+		return
+	}
+	// Normalize folder name: must start with '@', no slashes, no whitespace.
+	folder := strings.TrimSpace(req.FolderName)
+	if !strings.HasPrefix(folder, "@") {
+		folder = "@" + folder
+	}
+	if strings.ContainsAny(folder, "/\\ \t\n") {
+		d.sendDayzModInstallResult(corrID, "folder_name must not contain slashes or whitespace", nil, "")
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	targetID, err := d.getFsTargetContainerID(ctx, rec)
+	if err != nil {
+		d.sendDayzModInstallResult(corrID, err.Error(), nil, "")
+		return
+	}
+
+	modPath := "/game/steamapps/workshop/content/221100/" + req.WorkshopId
+	linkPath := "/game/" + folder
+
+	// 1. Verify workshop content exists.
+	_, _, code, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		fmt.Sprintf("test -d %q", modPath)})
+	if code != 0 {
+		d.sendDayzModInstallResult(corrID, "workshop content not found at "+modPath+" — did SteamCMD download succeed?", nil, modPath)
+		return
+	}
+
+	// 2. Create symlink.
+	_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		fmt.Sprintf("ln -sfn %q %q", modPath, linkPath)})
+	if err != nil {
+		d.sendDayzModInstallResult(corrID, "symlink: "+err.Error(), nil, modPath)
+		return
+	}
+	if code != 0 {
+		d.sendDayzModInstallResult(corrID, "symlink failed: "+strings.TrimSpace(string(stderr)), nil, modPath)
+		return
+	}
+
+	// 3. Copy bikeys. DayZ mods ship them under keys/ OR Keys/ inside the
+	// workshop tree. Copy every .bikey we find there into /game/keys/.
+	// Keep a manifest of copied filenames so uninstall can remove them.
+	// Use `find` + `cp` for portability; busybox and GNU both understand.
+	stdout, stderr2, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		fmt.Sprintf(`mkdir -p /game/keys && \
+		 for src in %q/keys %q/Keys; do \
+		     [ -d "$src" ] || continue; \
+		     for f in "$src"/*.bikey "$src"/*.BIKEY; do \
+		         [ -f "$f" ] || continue; \
+		         base=$(basename "$f"); \
+		         cp -f "$f" /game/keys/"$base"; \
+		         printf '%%s\n' "$base"; \
+		     done; \
+		 done`, modPath, modPath)})
+	if err != nil {
+		d.sendDayzModInstallResult(corrID, "copy bikeys: "+err.Error(), nil, modPath)
+		return
+	}
+	if code != 0 {
+		d.sendDayzModInstallResult(corrID, "copy bikeys failed: "+strings.TrimSpace(string(stderr2)), nil, modPath)
+		return
+	}
+	var bikeys []string
+	for _, ln := range strings.Split(string(stdout), "\n") {
+		ln = strings.TrimSpace(strings.TrimPrefix(ln, `\n`))
+		if ln == "" {
+			continue
+		}
+		bikeys = append(bikeys, ln)
+	}
+	d.sendDayzModInstallResult(corrID, "", bikeys, modPath)
+}
+
+func (d *Dispatcher) sendDayzModInstallResult(corrID, errMsg string, bikeys []string, resolvedPath string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_DayzModInstallResult{
+			DayzModInstallResult: &panelv1.DayzModInstallResult{
+				Error:            errMsg,
+				BikeysCopied:     bikeys,
+				ResolvedModPath:  resolvedPath,
+			},
+		},
+	})
+}
+
+// ---- DayzModUninstall ----
+
+func (d *Dispatcher) handleDayzModUninstall(corrID string, req *panelv1.DayzModUninstallRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendDayzModUninstallResult(corrID, err.Error(), nil)
+		return
+	}
+	folder := strings.TrimSpace(req.FolderName)
+	if !strings.HasPrefix(folder, "@") || strings.ContainsAny(folder, "/\\ \t\n") {
+		d.sendDayzModUninstallResult(corrID, "invalid folder_name", nil)
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	targetID, err := d.getFsTargetContainerID(ctx, rec)
+	if err != nil {
+		d.sendDayzModUninstallResult(corrID, err.Error(), nil)
+		return
+	}
+	linkPath := "/game/" + folder
+
+	// Resolve the symlink to find the workshop directory.
+	stdout, _, code, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		fmt.Sprintf("readlink -f %q 2>/dev/null || true", linkPath)})
+	modPath := strings.TrimSpace(string(stdout))
+	if code != 0 || modPath == "" {
+		// Symlink is already gone or broken — just drop stale /game/keys entries.
+		modPath = ""
+	}
+
+	// Remove bikeys whose *name* matches a bikey in the workshop mod's keys.
+	// Matching by name (not hash) is what DayZ tooling does — the server
+	// looks up bikeys by exact basename, and mods typically ship unique
+	// key names.
+	var removed []string
+	if modPath != "" {
+		out, _, _, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+			fmt.Sprintf(`for d in %q/keys %q/Keys; do \
+			    [ -d "$d" ] || continue; \
+			    for f in "$d"/*.bikey "$d"/*.BIKEY; do \
+			        [ -f "$f" ] || continue; \
+			        base=$(basename "$f"); \
+			        if [ -f "/game/keys/$base" ]; then \
+			            rm -f "/game/keys/$base"; \
+			            echo "$base"; \
+			        fi; \
+			    done; \
+			done`, modPath, modPath)})
+		for _, ln := range strings.Split(string(out), "\n") {
+			ln = strings.TrimSpace(ln)
+			if ln == "" {
+				continue
+			}
+			removed = append(removed, ln)
+		}
+	}
+
+	// Remove the symlink itself.
+	_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		fmt.Sprintf("rm -f %q", linkPath)})
+	if err != nil {
+		d.sendDayzModUninstallResult(corrID, "rm symlink: "+err.Error(), removed)
+		return
+	}
+	if code != 0 {
+		d.sendDayzModUninstallResult(corrID, "rm symlink failed: "+strings.TrimSpace(string(stderr)), removed)
+		return
+	}
+	d.sendDayzModUninstallResult(corrID, "", removed)
+}
+
+func (d *Dispatcher) sendDayzModUninstallResult(corrID, errMsg string, bikeys []string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_DayzModUninstallResult{
+			DayzModUninstallResult: &panelv1.DayzModUninstallResult{
+				Error:          errMsg,
+				BikeysRemoved:  bikeys,
+			},
+		},
+	})
+}

agent/internal/dispatch/decoremap_provision.go

@@ -0,0 +1,165 @@
+package dispatch
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// Auto-provision a freshly-generated clustered 7DTD world (zero-touch create),
+// AND keep a visible "setting up" status until it is actually done so an operator
+// who refreshes the dashboard sees it is not ready yet (don't restart it).
+//
+// A new cluster world bakes its decorations (decoration.7dt / multiblocks.7dt)
+// with the engine's NATIVE block ids at generation. The entrypoint stamps the
+// cluster CANONICAL .nim over the world's table every boot so shared inventory
+// (.ttp) decodes uniformly — but native-baked decorations then resolve to a null
+// Block under canonical -> DecoManager NRE -> the world loads but never reaches
+// GameStartDone ("online but unjoinable"). The entrypoint fixes this by remapping
+// the decorations native->canonical on the FIRST boot it sees them in the native
+// band — which can only be a boot AFTER the gen boot (the running server writes
+// decoration.7dt; the entrypoint can't see it until next start).
+//
+// This watcher (a) closes that 2-phase gap so create is zero-touch — it bounces
+// the container ONCE so the entrypoint's remap runs — and (b) holds a persistent
+// "Finishing cluster setup…" status detail (the dashboard renders it as a "setting
+// up" pill instead of a green "running" one) until the world carries the
+// .canon-provisioned marker, so a page refresh always shows it isn't done yet.
+//
+// Strict no-op for non-cluster servers (no /cluster bind -> the watcher exits) and
+// already-canon worlds (deco in canon band -> "CANON" on the first probe -> the
+// indicator clears immediately). Single-flight; the entrypoint is the backstop if
+// the watcher ever misses (the world still provisions on its next restart).
+// Mirrors the hang-guard single-flight container-restart pattern (hangguard.go).
+
+const (
+	provisionPollInterval = 20 * time.Second
+	provisionPollTimeout  = 15 * time.Minute
+	provisionExecTimeout  = 20 * time.Second
+	provisionRestartGrace = 30 * time.Second
+)
+
+// provisionDetail is the persistent status detail shown while a clustered world is
+// still aligning to the cluster canonical. The dashboard matches this text to paint
+// a "setting up" pill, so a refresh shows the server isn't ready yet.
+const provisionDetail = "Finishing cluster setup — aligning decorations to the cluster (it auto-restarts once); please wait, don't restart it."
+
+// provisionProbeScript runs inside the container and classifies the active cluster
+// world's provisioning state. Emits exactly one token on stdout:
+//
+//	NOWORLD     no recorded save dir (RWG world not yet picked in the Cluster tab)
+//	PROVISIONED .canon-provisioned marker present (already aligned)
+//	GENERATING  no main.ttw yet (world still generating)
+//	CANON       decorations already in the canon band (>=24000) -> nothing to do
+//	ATTEMPTED   already bounced once and still not aligned -> stop (avoid a loop)
+//	NEEDS_ALIGN gen done; decorations native-banded OR not baked yet -> bounce once
+//
+// Gated on main.ttw (= gen finished) not on decoration.7dt existing, because some
+// worlds only write decoration.7dt on the first save — the bounce's graceful save
+// writes it (native), then the entrypoint remaps it on the align boot. First deco
+// block-id = u16 @ offset 17 (hdr 5 + record id-offset 12), masked to 15 bits.
+const provisionProbeScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
+	`[ -z "$SB" ] && { echo NOWORLD; exit 0; }; ` +
+	`[ -f "$SB/.canon-provisioned" ] && { echo PROVISIONED; exit 0; }; ` +
+	`[ -f "$SB/main.ttw" ] || { echo GENERATING; exit 0; }; ` +
+	`D="$SB/decoration.7dt"; ` +
+	`if [ -f "$D" ]; then raw="$(od -An -tu2 -j17 -N2 "$D" 2>/dev/null | tr -d ' ')"; id=$(( ${raw:-0} & 0x7FFF )); [ "$id" -ge 24000 ] && { echo CANON; exit 0; }; fi; ` +
+	`[ -f "$SB/.panel-provision-attempted" ] && { echo ATTEMPTED; exit 0; }; ` +
+	`echo NEEDS_ALIGN`
+
+// provisionAttemptScript drops a durable one-bounce guard so a world that doesn't
+// align after its single bounce is never bounced again (the entrypoint remains the
+// backstop on ordinary restarts).
+const provisionAttemptScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
+	`[ -n "$SB" ] && touch "$SB/.panel-provision-attempted" 2>/dev/null; true`
+
+// autoProvisionClusterWorldOnStart starts (single-flight) the provision watcher for
+// a just-started 7DTD instance. Returns immediately; the watch runs off a background
+// goroutine bounded by provisionPollTimeout.
+func (d *Dispatcher) autoProvisionClusterWorldOnStart(rec *instanceRecord) {
+	if rec.ModuleID != "7dtd" {
+		return
+	}
+	if !rec.provisionGuard.CompareAndSwap(false, true) {
+		return // a watcher is already running for this instance
+	}
+	go func() {
+		defer rec.provisionGuard.Store(false)
+		log := d.log.With("instance_id", rec.InstanceID, "auto_provision", true)
+
+		// Only clustered worlds provision (and show the indicator). Resolve the
+		// /cluster bind like clusterplayer.go; bail out for standalone servers.
+		mctx, mcancel := context.WithTimeout(context.Background(), 15*time.Second)
+		mounts, _ := d.runtime.ContainerMounts(mctx, "panel-"+rec.InstanceID)
+		mcancel()
+		clustered := false
+		for _, m := range mounts {
+			if m.Destination == "/cluster" && m.Type == "bind" && m.Source != "" {
+				clustered = true
+				break
+			}
+		}
+		if !clustered {
+			return
+		}
+
+		deadline := time.Now().Add(provisionPollTimeout)
+		delay := 5 * time.Second // first poll soon, to override a premature green "running"
+		var lastDetail string
+		bounced := false
+		emit := func(detail string) {
+			if detail == lastDetail {
+				return
+			}
+			lastDetail = detail
+			d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING, 0, detail)
+		}
+		for {
+			time.Sleep(delay)
+			delay = provisionPollInterval
+			if time.Now().After(deadline) {
+				return // gave up; the entrypoint aligns on the next ordinary restart
+			}
+			ectx, cancel := context.WithTimeout(context.Background(), provisionExecTimeout)
+			stdout, _, code, err := d.runtime.ExecCapture(ectx, rec.ContainerID, []string{"sh", "-c", provisionProbeScript})
+			cancel()
+			if err != nil || code != 0 {
+				continue // mid-boot / exec hiccup — leave the agent's own status alone
+			}
+			switch strings.TrimSpace(string(stdout)) {
+			case "CANON", "PROVISIONED":
+				emit("running") // fully set up — clear the "setting up" indicator
+				return
+			case "NEEDS_ALIGN":
+				emit(provisionDetail)
+				if !bounced {
+					bounced = true
+					actx, acancel := context.WithTimeout(context.Background(), provisionExecTimeout)
+					_, _, _, _ = d.runtime.ExecCapture(actx, rec.ContainerID, []string{"sh", "-c", provisionAttemptScript})
+					acancel()
+					log.Info("auto-provision: aligning new cluster world to canonical (one-time bounce)")
+					d.sendEnv(&panelv1.AgentEnvelope{
+						SentAt: timestamppb.Now(),
+						Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+							InstanceId: rec.InstanceID, Stream: "stdout", At: timestamppb.Now(),
+							Line: "[panel] auto-provision: aligning the new world's decorations to the cluster (one-time restart)…",
+						}},
+					})
+					rctx, rcancel := context.WithTimeout(context.Background(), 90*time.Second)
+					if err := d.runtime.Restart(rctx, rec.ContainerID, provisionRestartGrace); err != nil {
+						log.Error("auto-provision restart failed (entrypoint aligns on the next ordinary restart)", "err", err)
+					}
+					rcancel()
+				}
+				continue // keep watching across the bounce until the align boot marks it provisioned
+			default: // GENERATING / NOWORLD / ATTEMPTED — still setting up
+				emit(provisionDetail)
+				continue
+			}
+		}
+	}()
+}

agent/internal/dispatch/delete.go

@@ -0,0 +1,172 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// handleDelete tears down the instance record. If the container is still
+// running, it's stopped with the requested grace first. Optionally
+// removes every Docker volume labeled for this instance ("purge").
+//
+// Never deletes the host data_path directory — operators may want to
+// keep configs/logs that landed on the bind mount. If they want a full
+// wipe, they can delete the dir manually after.
+func (d *Dispatcher) handleDelete(ctx context.Context, corrID string, req *panelv1.InstanceDelete) {
+	log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
+
+	d.mu.Lock()
+	rec, exists := d.instances[req.InstanceId]
+	d.mu.Unlock()
+
+	// Cancel any in-flight steamcmd / updater goroutine FIRST. The
+	// updater spins up its own sidecar container (panel-<id>-steamcmd-<n>)
+	// that mounts the same data volumes. If we don't cancel it before
+	// trying to remove volumes, the sidecar holds them open and the
+	// volume-rm fails with "volume is in use" — operator sees the
+	// instance stuck on "Deleting…" while a download finishes in the
+	// background.
+	d.cancelUpdater(req.InstanceId)
+	// Whatever state the main container is in, any Files-tab helper that
+	// was running against this instance is now orphaned. Clean it up
+	// before touching anything else so its volume-holds don't interfere
+	// with the main container's stop/remove.
+	d.teardownFsHelper(req.InstanceId)
+
+	grace := time.Duration(req.GraceSeconds) * time.Second
+	if grace == 0 {
+		grace = 15 * time.Second
+	}
+
+	// If we have a record, use its container_id directly. Otherwise try
+	// to find the container by its canonical name (may exist from a
+	// previous session that skipped rehydrate).
+	containerID := ""
+	if exists {
+		containerID = rec.ContainerID
+	} else {
+		if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil {
+			containerID = state.ContainerID
+		}
+	}
+
+	if containerID != "" {
+		// Best-effort stop; ignore errors since container may already be stopped.
+		_ = d.runtime.Stop(ctx, containerID, grace)
+		removeErr := d.runtime.Remove(ctx, containerID)
+		// d.runtime.Remove already swallows not-found, so a non-nil
+		// removeErr means the remove genuinely failed. Before we tear down
+		// any bookkeeping (in-memory record + sidecar metadata) we must be
+		// SURE the container is gone — otherwise we orphan a live container
+		// the controller still routes to, and the agent forgets it on the
+		// next rehydrate. The classic trigger: a delete racing with agent
+		// shutdown, where the Docker connection dies mid-remove
+		// ("terminated signal received"). The remove errors, and the
+		// follow-up existence check ALSO errors — fate unknown, not gone.
+		if removeErr != nil {
+			exists, existErr := d.runtime.ContainerExists(ctx, "panel-"+req.InstanceId)
+			switch {
+			case existErr != nil:
+				// Couldn't even ask (daemon down / connection dropped /
+				// ctx cancelled). Do NOT delete the sidecar — bail and let
+				// the operator retry once the daemon is reachable. Leaving
+				// the sidecar means the next rehydrate re-adopts the
+				// instance instead of orphaning it.
+				log.Error("delete: container remove failed and existence is unknown — aborting to avoid orphan",
+					"remove_err", removeErr, "exists_err", existErr)
+				d.replyError(corrID, req.InstanceId, "container_remove",
+					fmt.Sprintf("container panel-%s remove failed and existence unverifiable: %s", req.InstanceId, removeErr.Error()))
+				return
+			case exists:
+				// Container is confirmed still present. The controller must
+				// learn — otherwise the DB row gets purged while a zombie
+				// container lives on, and the next create with the same id
+				// fails on a "name already in use" docker error.
+				log.Error("delete: container still exists after remove", "err", removeErr)
+				d.replyError(corrID, req.InstanceId, "container_remove",
+					fmt.Sprintf("container panel-%s still exists after remove: %s", req.InstanceId, removeErr.Error()))
+				return
+			default:
+				// Confirmed gone (e.g. autoremove fired between Stop and
+				// Remove). Safe to proceed with bookkeeping cleanup.
+				log.Warn("delete: remove returned error but container is confirmed gone", "err", removeErr)
+			}
+		}
+	}
+
+	// Signal any live trackers / log streams to stop.
+	if exists {
+		if rec.TrackerCancel != nil {
+			rec.TrackerCancel()
+		}
+		if rec.LogCancel != nil {
+			rec.LogCancel()
+		}
+	}
+
+	// Purge volumes if requested.
+	if req.PurgeVolumes {
+		dr, ok := d.runtime.(*runtime.DockerRuntime)
+		if !ok {
+			log.Warn("delete: purge requested but runtime is not Docker; skipping volume cleanup")
+		} else {
+			// Sweep every container that mounts one of this instance's
+			// volumes — running OR exited. Without this, a stranded
+			// steamcmd / backup / fs-helper sidecar from a previous
+			// agent crash keeps the volume "in use" and volume-rm
+			// fails repeatedly. Operator sees the instance stuck
+			// "Deleting…" forever.
+			holders, err := dr.ListContainersHoldingInstanceVolumes(ctx, req.InstanceId)
+			if err != nil {
+				log.Warn("delete: scan for orphan sidecars failed", "err", err)
+			}
+			for _, hid := range holders {
+				if err := dr.ForceRemoveContainer(ctx, hid); err != nil {
+					log.Warn("delete: force-remove sidecar failed", "container_id", hid, "err", err)
+				} else {
+					log.Info("delete: removed orphan sidecar", "container_id", short(hid))
+				}
+			}
+			vols, err := dr.ListVolumesByInstance(ctx, req.InstanceId)
+			if err != nil {
+				log.Warn("delete: list volumes failed", "err", err)
+			}
+			for _, v := range vols {
+				// Force=true: there's no scenario where we want a panel
+				// volume to survive a purge=true delete after we've
+				// scrubbed every container that referenced it.
+				if err := dr.RemoveVolume(ctx, v, true); err != nil {
+					log.Warn("delete: remove volume failed", "volume", v, "err", err)
+					d.replyError(corrID, req.InstanceId, "volume_remove", fmt.Sprintf("volume %s: %s", v, err.Error()))
+					return
+				}
+				log.Info("delete: purged volume", "volume", v)
+			}
+		}
+	}
+
+	// Remove in-memory record + sidecar metadata.
+	d.mu.Lock()
+	delete(d.instances, req.InstanceId)
+	d.mu.Unlock()
+	d.deleteMeta(req.InstanceId)
+
+	// Leave host data_path alone (see doc comment); operators can rm -rf
+	// manually. We do NOT attempt os.RemoveAll here — too destructive to
+	// do automatically on a shared multi-instance directory layout.
+
+	d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "deleted")
+	d.replyOK(corrID)
+	log.Info("instance deleted", "purge_volumes", req.PurgeVolumes)
+	_ = os.Stat // kept imported for future host-data_path removal flag
+}
+
+// sendDelete-specific correlation reply uses the existing replyOK/err pattern.
+var _ = timestamppb.Now // keep import hint while ctx-based future expansions land

agent/internal/dispatch/empyrion_discoveries.go

@@ -0,0 +1,526 @@
+package dispatch
+
+// Per-player POI discovery query against the empyrion server's
+// global.db SQLite store. Spawns an alpine+sqlite sidecar with the
+// instance's saves volume mounted read-only, runs a parameterized
+// join query, returns the rows.
+//
+// Schema reference (DiscoveredPOIs / Entities / Playfields / SolarSystems
+// / LoginLogoff) was reverse-engineered from a live empyrion install on
+// 2026-04-30. Eleon's mod API does NOT expose per-player discovery —
+// Request_GlobalStructure_List returns the server-wide aggregate that
+// EAH shows. The DB has the finer per-player data the in-game map uses.
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// sqlite3's CLI doesn't support `?` parameter binding (that's a
+// prepared-statement-only feature on the C API). We string-interpolate
+// the SteamID64 — already validated as 17 digits server-side, but we
+// also belt-and-suspenders re-validate at the shell layer (digits only).
+//
+// Two query modes:
+//   "personal" — POIs the player personally first-walked-into
+//   "faction"  — POIs anyone in the player's faction has discovered
+//                (matches the in-game shared map)
+const sqlPersonalTpl = `
+SELECT
+  COALESCE(ss.name, '') AS solar_system,
+  COALESCE(pf.name, '') AS playfield,
+  COALESCE(poi.name, '') AS poi_name,
+  COALESCE(poi.etype, 0) AS poi_type,
+  COALESCE(dp.gametime, 0) AS game_time,
+  COALESCE(ll.playername, '') AS discoverer_name,
+  COALESCE(ll.playerid, '') AS discoverer_steam_id,
+  COALESCE(dp.facgroup, 0) AS facgroup,
+  COALESCE(dp.facid, 0) AS facid
+FROM DiscoveredPOIs dp
+JOIN Entities poi ON dp.poiid = poi.entityid
+LEFT JOIN Playfields pf ON dp.pfid = pf.pfid
+LEFT JOIN SolarSystems ss ON pf.ssid = ss.ssid
+LEFT JOIN LoginLogoff ll ON dp.entityid = ll.entityid
+WHERE dp.entityid IN (
+  SELECT DISTINCT entityid FROM LoginLogoff WHERE playerid='%SID%'
+)
+ORDER BY dp.gametime DESC
+LIMIT 5000;
+`
+
+const sqlFactionTpl = `
+SELECT
+  COALESCE(ss.name, '') AS solar_system,
+  COALESCE(pf.name, '') AS playfield,
+  COALESCE(poi.name, '') AS poi_name,
+  COALESCE(poi.etype, 0) AS poi_type,
+  COALESCE(dp.gametime, 0) AS game_time,
+  COALESCE(ll.playername, '') AS discoverer_name,
+  COALESCE(ll.playerid, '') AS discoverer_steam_id,
+  COALESCE(dp.facgroup, 0) AS facgroup,
+  COALESCE(dp.facid, 0) AS facid
+FROM DiscoveredPOIs dp
+JOIN Entities poi ON dp.poiid = poi.entityid
+LEFT JOIN Playfields pf ON dp.pfid = pf.pfid
+LEFT JOIN SolarSystems ss ON pf.ssid = ss.ssid
+LEFT JOIN LoginLogoff ll ON dp.entityid = ll.entityid
+WHERE (dp.facgroup, dp.facid) IN (
+  SELECT DISTINCT pe.facgroup, pe.facid
+  FROM Entities pe
+  WHERE pe.entityid IN (
+    SELECT DISTINCT entityid FROM LoginLogoff WHERE playerid='%SID%'
+  )
+)
+ORDER BY dp.gametime DESC
+LIMIT 5000;
+`
+
+const sqlPlayerNameTpl = `
+SELECT playername FROM LoginLogoff
+WHERE playerid='%SID%'
+ORDER BY loginticks DESC
+LIMIT 1;
+`
+
+func (d *Dispatcher) handleEmpyrionDiscoveries(corrID string, req *panelv1.EmpyrionDiscoveriesRequest) {
+	if req == nil || req.InstanceId == "" || req.SteamId == "" {
+		d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+			Ok: false, Error: "instance_id and steam_id required",
+		})
+		return
+	}
+	go d.runEmpyrionDiscoveriesJob(corrID, req)
+}
+
+func (d *Dispatcher) runEmpyrionDiscoveriesJob(corrID string, req *panelv1.EmpyrionDiscoveriesRequest) {
+	mode := strings.ToLower(strings.TrimSpace(req.Mode))
+	if mode == "" || mode == "personal" {
+		mode = "personal"
+	} else if mode != "faction" {
+		d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+			Ok: false, Error: "mode must be 'personal' or 'faction'", Mode: req.Mode, SteamId: req.SteamId,
+		})
+		return
+	}
+
+	dockerBin, err := exec.LookPath("docker")
+	if err != nil {
+		d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+			Ok: false, Error: "docker binary not found on agent host", Mode: mode, SteamId: req.SteamId,
+		})
+		return
+	}
+
+	// Saves volume name follows the panel convention: panel-<id>-saves.
+	savesVol := "panel-" + req.InstanceId + "-saves"
+	// Pre-flight: volume must exist (instance exists on this agent).
+	if err := exec.Command(dockerBin, "volume", "inspect", savesVol).Run(); err != nil {
+		d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+			Ok: false, Error: fmt.Sprintf("instance saves volume %q not found on this agent", savesVol),
+			Mode: mode, SteamId: req.SteamId,
+		})
+		return
+	}
+
+	// Validate steam_id is digits-only — defense in depth, controller
+	// already checks 17-digit format. Belt-and-suspenders against any
+	// future caller that bypasses the controller's validation.
+	for _, c := range req.SteamId {
+		if c < '0' || c > '9' {
+			d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+				Ok: false, Error: "steam_id must be digits only", Mode: mode, SteamId: req.SteamId,
+			})
+			return
+		}
+	}
+	tpl := sqlPersonalTpl
+	if mode == "faction" {
+		tpl = sqlFactionTpl
+	}
+	q := strings.ReplaceAll(tpl, "%SID%", req.SteamId)
+	nameQ := strings.ReplaceAll(sqlPlayerNameTpl, "%SID%", req.SteamId)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// alpine+sqlite sidecar. We don't need to discover the savegame
+	// folder name dynamically — the entrypoint creates exactly one
+	// (via dedicated.yaml.SaveDirectory which defaults to "DediGame";
+	// the panel's empyrion module pins this). We pick `*/global.db`
+	// at the shell layer so renames don't break the query.
+	//
+	// Note: -2 disables stdin which avoids the `command timed out`
+	// path that empty stdin can hit on some Alpine builds. We feed the
+	// SQL via the third positional arg.
+	rowsScript := `set -e
+apk add --no-cache sqlite >/dev/null 2>&1
+DB=$(ls /saves/Saves/Games/*/global.db 2>/dev/null | head -1)
+if [ -z "$DB" ]; then echo '{"err":"no global.db found in saves volume"}' >&2; exit 2; fi
+NAME=$(sqlite3 "$DB" "$1" 2>/dev/null || echo "")
+ROWS=$(sqlite3 -json "$DB" "$2" 2>&1)
+if [ -z "$ROWS" ]; then ROWS='[]'; fi
+NAME_JSON=$(printf '%s' "$NAME" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g')
+printf '{"player_name":"%s","rows":%s}\n' "$NAME_JSON" "$ROWS"
+`
+
+	out, err := exec.CommandContext(ctx, dockerBin,
+		"run", "--rm",
+		"-v", savesVol+":/saves:ro",
+		"--entrypoint", "sh",
+		"alpine:3.20", "-c", rowsScript, "_", nameQ, q,
+	).Output()
+	if err != nil {
+		d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+			Ok: false, Error: "sqlite query failed: " + err.Error(),
+			Mode: mode, SteamId: req.SteamId,
+		})
+		return
+	}
+
+	// Parse the wrapper json {"player_name": "...", "rows": [...]}.
+	type wireRow struct {
+		SolarSystem       string `json:"solar_system"`
+		Playfield         string `json:"playfield"`
+		PoiName           string `json:"poi_name"`
+		PoiType           int32  `json:"poi_type"`
+		GameTime          int64  `json:"game_time"`
+		DiscovererName    string `json:"discoverer_name"`
+		DiscovererSteamID string `json:"discoverer_steam_id"`
+		Facgroup          int32  `json:"facgroup"`
+		Facid             int32  `json:"facid"`
+	}
+	var wrap struct {
+		PlayerName string    `json:"player_name"`
+		Rows       []wireRow `json:"rows"`
+	}
+	// Strip leading whitespace; sqlite3 -json sometimes prepends a BOM-ish blank line.
+	if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &wrap); err != nil {
+		// Tolerate "no rows" — sqlite3 -json prints `[]` (good) or empty
+		// string when no DB (bad — caught above) or `null` for some
+		// builds. Fall back to empty-result on parse failure.
+		var rowsOnly []wireRow
+		if err2 := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &rowsOnly); err2 == nil {
+			wrap.Rows = rowsOnly
+		} else {
+			d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+				Ok: false, Error: "parse sqlite output: " + err.Error(),
+				Mode: mode, SteamId: req.SteamId,
+			})
+			return
+		}
+	}
+
+	pois := make([]*panelv1.DiscoveredPOI, 0, len(wrap.Rows))
+	for _, r := range wrap.Rows {
+		pois = append(pois, &panelv1.DiscoveredPOI{
+			SolarSystem:       r.SolarSystem,
+			Playfield:         r.Playfield,
+			PoiName:           r.PoiName,
+			PoiType:           r.PoiType,
+			GameTime:          r.GameTime,
+			DiscovererName:    r.DiscovererName,
+			DiscovererSteamId: r.DiscovererSteamID,
+			FactionGroup:      r.Facgroup,
+			FactionId:         r.Facid,
+		})
+	}
+	d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
+		Ok:         true,
+		Pois:       pois,
+		Mode:       mode,
+		SteamId:    req.SteamId,
+		PlayerName: strings.Trim(wrap.PlayerName, "\""),
+	})
+	_ = errors.New
+}
+
+func (d *Dispatcher) sendEmpyrionDiscoveriesResult(corrID string, res *panelv1.EmpyrionDiscoveriesResult) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_EmpyrionDiscoveriesResult{
+			EmpyrionDiscoveriesResult: res,
+		},
+	})
+}
+
+// ============================================================================
+// Player Summary — stats + sessions + inventory in one query
+// ============================================================================
+
+// playerSummaryScript is one big sh script run inside an alpine sidecar.
+// It opens global.db, resolves the player's entityid via LoginLogoff,
+// then runs four sqlite queries with -json output. The output is one
+// JSON object with named keys for each query — we marshal-decode it
+// directly into the proto result.
+//
+// The 4 queries:
+//   1. summary  — Entities row + LoginLogoff name lookup + last position
+//   2. stats    — PlayerStatistics row
+//   3. sessions — recent LoginLogoff rows
+//   4. inv      — most recent PlayerInventory snapshot + items
+//
+// We accept that some of these may return [] (player has no stats row
+// yet, never logged in this savegame, etc.) — JSON parser handles it.
+func playerSummaryScript() string {
+	return `set -e
+apk add --no-cache sqlite >/dev/null 2>&1
+DB=$(ls /saves/Saves/Games/*/global.db 2>/dev/null | head -1)
+if [ -z "$DB" ]; then echo '{"err":"no global.db found in saves volume"}' >&2; exit 2; fi
+SID="$1"
+LIMIT="${2:-50}"
+
+# Resolve entityid for this Steam ID. Use the most recent LoginLogoff
+# row to handle character resets that re-bind the same Steam ID to a
+# new entityid.
+EID=$(sqlite3 "$DB" "SELECT entityid FROM LoginLogoff WHERE playerid='${SID}' ORDER BY loginticks DESC LIMIT 1;")
+if [ -z "$EID" ]; then
+  printf '{"steam_id":"%s","entity_id":0,"player_name":"","stats":null,"sessions":[],"inventory":[]}\n' "$SID"
+  exit 0
+fi
+
+NAME=$(sqlite3 "$DB" "SELECT playername FROM LoginLogoff WHERE entityid=${EID} ORDER BY loginticks DESC LIMIT 1;")
+FG=$(sqlite3 "$DB" "SELECT facgroup FROM Entities WHERE entityid=${EID};")
+FI=$(sqlite3 "$DB" "SELECT facid FROM Entities WHERE entityid=${EID};")
+
+# Last position from PlayerData.
+PD=$(sqlite3 -json "$DB" "
+  SELECT
+    COALESCE(pf.name, '') AS last_playfield,
+    COALESCE(pd.posx, 0) AS last_x,
+    COALESCE(pd.posy, 0) AS last_y,
+    COALESCE(pd.posz, 0) AS last_z,
+    COALESCE(pd.credits, 0) AS credits
+  FROM PlayerData pd
+  LEFT JOIN Playfields pf ON pd.pfid = pf.pfid
+  WHERE pd.entityid=${EID};
+")
+
+STATS=$(sqlite3 -json "$DB" "
+  SELECT
+    COALESCE(killedenemies, 0) AS killed_enemies,
+    COALESCE(killedallied, 0) AS killed_allied,
+    COALESCE(killedanimals, 0) AS killed_animals,
+    COALESCE(killeddrones, 0) AS killed_drones,
+    COALESCE(killedplayers, 0) AS killed_players,
+    COALESCE(killedalliedplayers, 0) AS killed_allied_players,
+    COALESCE(died, 0) AS died,
+    COALESCE(score, 0) AS score,
+    COALESCE(blocksplaced, 0) AS blocks_placed,
+    COALESCE(blocksdigged, 0) AS blocks_digged,
+    COALESCE(walkedmeters, 0) AS walked_meters,
+    COALESCE(jetpackmeters, 0) AS jetpack_meters,
+    COALESCE(hvmeters, 0) AS hv_meters,
+    COALESCE(svmeters, 0) AS sv_meters,
+    COALESCE(cvmeters, 0) AS cv_meters,
+    COALESCE(playtime, 0.0) AS playtime,
+    COALESCE(travly, 0.0) AS light_years,
+    COALESCE(travau, 0.0) AS astronomical_units
+  FROM PlayerStatistics WHERE entityid=${EID};
+")
+
+SESS=$(sqlite3 -json "$DB" "
+  SELECT
+    COALESCE(loginticks, 0)  AS login_ticks,
+    COALESCE(logoffticks, 0) AS logoff_ticks,
+    COALESCE(playername, '') AS player_name,
+    COALESCE(buildnr, 0)     AS build_nr,
+    COALESCE(ip, '')         AS ip,
+    COALESCE(os, '')         AS os,
+    COALESCE(gfx, '')        AS gfx,
+    COALESCE(cpu, '')        AS cpu
+  FROM LoginLogoff WHERE entityid=${EID}
+  ORDER BY loginticks DESC LIMIT ${LIMIT};
+")
+
+INV_PIID=$(sqlite3 "$DB" "SELECT piid FROM PlayerInventory WHERE entityid=${EID} ORDER BY gametime DESC LIMIT 1;")
+INV_GT=0
+INV='[]'
+if [ -n "$INV_PIID" ]; then
+  INV_GT=$(sqlite3 "$DB" "SELECT gametime FROM PlayerInventory WHERE piid=${INV_PIID};")
+  INV=$(sqlite3 -json "$DB" "
+    SELECT item AS item_id, count
+    FROM PlayerInventoryItems WHERE piid=${INV_PIID};
+  ")
+  if [ -z "$INV" ]; then INV='[]'; fi
+fi
+
+if [ -z "$STATS" ]; then STATS='[]'; fi
+if [ -z "$SESS" ]; then SESS='[]'; fi
+if [ -z "$PD" ]; then PD='[]'; fi
+
+printf '{"steam_id":"%s","player_name":%s,"entity_id":%s,"faction_group":%s,"faction_id":%s,"position":%s,"stats":%s,"sessions":%s,"inventory_gametime":%s,"inventory":%s}\n' \
+  "$SID" \
+  "$(printf '%%s' "$NAME" | sqlite3 ':memory:' 'select json_quote(?1);' 2>/dev/null || echo '""')" \
+  "${EID:-0}" "${FG:-0}" "${FI:-0}" \
+  "$PD" "$STATS" "$SESS" "${INV_GT:-0}" "$INV"
+`
+}
+
+func (d *Dispatcher) handleEmpyrionPlayerSummary(corrID string, req *panelv1.EmpyrionPlayerSummaryRequest) {
+	if req == nil || req.InstanceId == "" || req.SteamId == "" {
+		d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
+			Ok: false, Error: "instance_id and steam_id required",
+		})
+		return
+	}
+	go d.runEmpyrionPlayerSummaryJob(corrID, req)
+}
+
+func (d *Dispatcher) runEmpyrionPlayerSummaryJob(corrID string, req *panelv1.EmpyrionPlayerSummaryRequest) {
+	dockerBin, err := exec.LookPath("docker")
+	if err != nil {
+		d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
+			Ok: false, Error: "docker binary not found on agent host", SteamId: req.SteamId,
+		})
+		return
+	}
+	savesVol := "panel-" + req.InstanceId + "-saves"
+	if err := exec.Command(dockerBin, "volume", "inspect", savesVol).Run(); err != nil {
+		d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
+			Ok: false, Error: fmt.Sprintf("instance saves volume %q not found on this agent", savesVol), SteamId: req.SteamId,
+		})
+		return
+	}
+	limit := req.MaxLoginRows
+	if limit <= 0 || limit > 500 {
+		limit = 50
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, dockerBin,
+		"run", "--rm",
+		"-v", savesVol+":/saves:ro",
+		"--entrypoint", "sh",
+		"alpine:3.20", "-c", playerSummaryScript(), "_",
+		req.SteamId, fmt.Sprintf("%d", limit),
+	).Output()
+	if err != nil {
+		d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
+			Ok: false, Error: "sqlite query failed: " + err.Error(), SteamId: req.SteamId,
+		})
+		return
+	}
+	type wireStats struct {
+		KilledEnemies       int64   `json:"killed_enemies"`
+		KilledAllied        int64   `json:"killed_allied"`
+		KilledAnimals       int64   `json:"killed_animals"`
+		KilledDrones        int64   `json:"killed_drones"`
+		KilledPlayers       int64   `json:"killed_players"`
+		KilledAlliedPlayers int64   `json:"killed_allied_players"`
+		Died                int64   `json:"died"`
+		Score               int64   `json:"score"`
+		BlocksPlaced        int64   `json:"blocks_placed"`
+		BlocksDigged        int64   `json:"blocks_digged"`
+		WalkedMeters        int64   `json:"walked_meters"`
+		JetpackMeters       int64   `json:"jetpack_meters"`
+		HvMeters            int64   `json:"hv_meters"`
+		SvMeters            int64   `json:"sv_meters"`
+		CvMeters            int64   `json:"cv_meters"`
+		Playtime            float64 `json:"playtime"`
+		LightYears          float64 `json:"light_years"`
+		AstroUnits          float64 `json:"astronomical_units"`
+	}
+	type wireSession struct {
+		LoginTicks  int64  `json:"login_ticks"`
+		LogoffTicks int64  `json:"logoff_ticks"`
+		PlayerName  string `json:"player_name"`
+		BuildNr     int64  `json:"build_nr"`
+		IP          string `json:"ip"`
+		OS          string `json:"os"`
+		GFX         string `json:"gfx"`
+		CPU         string `json:"cpu"`
+	}
+	type wireInvItem struct {
+		ItemID int32 `json:"item_id"`
+		Count  int32 `json:"count"`
+	}
+	type wirePos struct {
+		LastPlayfield string  `json:"last_playfield"`
+		LastX         float64 `json:"last_x"`
+		LastY         float64 `json:"last_y"`
+		LastZ         float64 `json:"last_z"`
+		Credits       int64   `json:"credits"`
+	}
+	type wire struct {
+		SteamID           string         `json:"steam_id"`
+		PlayerName        string         `json:"player_name"`
+		EntityID          int64          `json:"entity_id"`
+		FactionGroup      int32          `json:"faction_group"`
+		FactionID         int32          `json:"faction_id"`
+		Position          []wirePos      `json:"position"`
+		Stats             []wireStats    `json:"stats"`
+		Sessions          []wireSession  `json:"sessions"`
+		InventoryGametime int64          `json:"inventory_gametime"`
+		Inventory         []wireInvItem  `json:"inventory"`
+	}
+	var w wire
+	if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &w); err != nil {
+		d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
+			Ok: false, Error: "parse sqlite output: " + err.Error(), SteamId: req.SteamId,
+		})
+		return
+	}
+	res := &panelv1.EmpyrionPlayerSummaryResult{
+		Ok:                true,
+		SteamId:           w.SteamID,
+		PlayerName:        strings.Trim(w.PlayerName, "\""),
+		EntityId:          w.EntityID,
+		FactionGroup:      w.FactionGroup,
+		FactionId:         w.FactionID,
+		InventoryGametime: w.InventoryGametime,
+	}
+	if len(w.Position) > 0 {
+		res.LastPlayfield = w.Position[0].LastPlayfield
+		res.LastX = w.Position[0].LastX
+		res.LastY = w.Position[0].LastY
+		res.LastZ = w.Position[0].LastZ
+		res.Credits = w.Position[0].Credits
+	}
+	if len(w.Stats) > 0 {
+		s := w.Stats[0]
+		res.Stats = &panelv1.PlayerStatsRow{
+			KilledEnemies: s.KilledEnemies, KilledAllied: s.KilledAllied,
+			KilledAnimals: s.KilledAnimals, KilledDrones: s.KilledDrones,
+			KilledPlayers: s.KilledPlayers, KilledAlliedPlayers: s.KilledAlliedPlayers,
+			Died: s.Died, Score: s.Score,
+			BlocksPlaced: s.BlocksPlaced, BlocksDigged: s.BlocksDigged,
+			WalkedMeters: s.WalkedMeters, JetpackMeters: s.JetpackMeters,
+			HvMeters: s.HvMeters, SvMeters: s.SvMeters, CvMeters: s.CvMeters,
+			Playtime: s.Playtime, LightYears: s.LightYears, AstronomicalUnits: s.AstroUnits,
+		}
+	}
+	for _, s := range w.Sessions {
+		res.Sessions = append(res.Sessions, &panelv1.PlayerLoginRow{
+			LoginTicks: s.LoginTicks, LogoffTicks: s.LogoffTicks,
+			PlayerName: s.PlayerName, BuildNr: s.BuildNr,
+			Ip: s.IP, Os: s.OS, Gfx: s.GFX, Cpu: s.CPU,
+		})
+	}
+	for _, it := range w.Inventory {
+		res.Inventory = append(res.Inventory, &panelv1.PlayerInventoryItem{
+			ItemId: it.ItemID, Count: it.Count,
+		})
+	}
+	d.sendEmpyrionPlayerSummaryResult(corrID, res)
+}
+
+func (d *Dispatcher) sendEmpyrionPlayerSummaryResult(corrID string, res *panelv1.EmpyrionPlayerSummaryResult) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_EmpyrionPlayerSummaryResult{
+			EmpyrionPlayerSummaryResult: res,
+		},
+	})
+}

agent/internal/dispatch/empyrion_scenario.go

@@ -0,0 +1,308 @@
+package dispatch
+
+// Agent-side Empyrion workshop-scenario installer. Runs entirely on the
+// local Docker daemon, so a multi-agent panel can install Reforged Eden 2
+// (and any other workshop scenario) into an Empyrion instance regardless
+// of which agent owns it.
+//
+// Flow:
+//   1. SteamCMD sidecar mounts panel-empyrion-workshop (an agent-local
+//      cache volume, lazily created on first install) and downloads the
+//      workshop item.
+//   2. Alpine helper sidecar mounts the workshop volume + the instance's
+//      `panel-<INSTANCE>-game` volume and copies the scenario files into
+//      Content/Scenarios/<scenario_name>/.
+//   3. A marker file (.panel-installed) is touched so the empyrion
+//      entrypoint can confirm the install.
+//
+// Progress is streamed back to the controller as LogLine envelopes with
+// stream="scenario" so the panel UI's scenario job log shows what's
+// happening live.
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+func (d *Dispatcher) handleEmpyrionScenarioInstall(corrID string, req *panelv1.EmpyrionScenarioInstallRequest) {
+	if req == nil || req.JobId == "" || req.InstanceId == "" || req.WorkshopId == "" || req.ScenarioName == "" {
+		d.sendEmpyrionScenarioResult(corrID, req.GetJobId(), false, "missing required fields", req.GetScenarioName())
+		return
+	}
+	appID := req.AppId
+	if appID == "" {
+		appID = "530870"
+	}
+	go d.runEmpyrionScenarioJob(corrID, req, appID)
+}
+
+func (d *Dispatcher) runEmpyrionScenarioJob(corrID string, req *panelv1.EmpyrionScenarioInstallRequest, appID string) {
+	// Redact the password from any line we forward upstream, since
+	// SteamCMD CAN echo `+login user PASS` back in error paths.
+	redact := func(s string) string {
+		if req.SteamPassword == "" || len(req.SteamPassword) < 4 {
+			return s
+		}
+		return strings.ReplaceAll(s, req.SteamPassword, "[REDACTED]")
+	}
+	emit := func(line string) {
+		d.sendEnv(&panelv1.AgentEnvelope{
+			SentAt: timestamppb.Now(),
+			Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+				InstanceId: req.InstanceId,
+				Stream:     "scenario",
+				At:         timestamppb.Now(),
+				Line:       redact(line),
+			}},
+		})
+	}
+	finish := func(ok bool, msg string) {
+		d.sendEmpyrionScenarioResult(corrID, req.JobId, ok, msg, req.ScenarioName)
+	}
+
+	emit(fmt.Sprintf("[scenario] starting workshop download: app=%s item=%s scenario=%q", appID, req.WorkshopId, req.ScenarioName))
+
+	dockerBin, err := exec.LookPath("docker")
+	if err != nil {
+		finish(false, "docker binary not found on agent host")
+		return
+	}
+
+	// Phase 1 — workshop download. The volume name is per-agent (Docker
+	// daemons don't share volumes); it's created lazily by the first run.
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+
+	// Build the SteamCMD argv for an attempt — login portion is the only
+	// variable. `+app_license_request` refreshes Steam's entitlement
+	// cache before the workshop download (cheap fix for stale-license
+	// "Access Denied" on owned games).
+	buildArgs := func(loginArgs []string) []string {
+		base := []string{
+			"run", "--rm",
+			"-v", "panel-steamcmd-auth:/root/.local/share/Steam",
+			"-v", "panel-empyrion-workshop:/empyrion-workshop/steamapps/workshop",
+			"steamcmd/steamcmd:latest",
+			"+@sSteamCmdForcePlatformType", "windows",
+			"+force_install_dir", "/empyrion-workshop",
+		}
+		base = append(base, loginArgs...)
+		base = append(base,
+			"+app_license_request", appID,
+			"+workshop_download_item", appID, req.WorkshopId, "validate",
+			"+quit",
+		)
+		return base
+	}
+
+	var out string
+	switch {
+	case req.SteamUsername == "":
+		emit("[scenario] no Steam credentials provided — trying anonymous (will fail for app workshops that require ownership)")
+		out, err = runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", "anonymous"}), emit)
+	default:
+		// Try cached refresh-token login first (no password). If the
+		// agent's panel-steamcmd-auth volume has a valid token from a
+		// prior install, this skips Steam Guard entirely — no push
+		// notification fires. Only when the cache is missing or expired
+		// (Logon failure / no cached credentials in the SteamCMD output)
+		// do we fall back to full password auth, which DOES require
+		// approving a Steam Guard push but then re-primes the cache for
+		// future runs.
+		emit(fmt.Sprintf("[scenario] login attempt 1: %q via cached refresh token (no Steam Guard push if cache valid)", req.SteamUsername))
+		cachedOut, cachedErr := runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", req.SteamUsername}), emit)
+		needsPwd := cachedErr != nil ||
+			strings.Contains(cachedOut, "Logon failure") ||
+			strings.Contains(cachedOut, "No cached credentials") ||
+			strings.Contains(cachedOut, "FAILED login") ||
+			strings.Contains(cachedOut, "Login failure")
+		if !needsPwd {
+			out, err = cachedOut, cachedErr
+			break
+		}
+		if req.SteamPassword == "" {
+			finish(false, "Steam refresh-token cache is missing/expired on this agent and no password is available to re-prime it. Re-enter credentials via the Steam login modal.")
+			return
+		}
+		emit(fmt.Sprintf("[scenario] login attempt 2: %q with password (Steam Guard push will fire — approve on phone)", req.SteamUsername))
+		out, err = runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", req.SteamUsername, req.SteamPassword}), emit)
+	}
+	out = redact(out)
+	if err != nil {
+		finish(false, "steamcmd: "+err.Error())
+		return
+	}
+	// Steam reports "Access Denied" when the logged-in account can't
+	// pull this workshop item. Common causes (in order of likelihood):
+	//   1. Account doesn't own the app — most common
+	//   2. License cache stale — we already issued +app_license_request
+	//      above, so if it still errors that's not it
+	//   3. Workshop item is private / friends-only / restricted by author
+	//   4. Workshop item requires the operator to "subscribe" to it via
+	//      the desktop Steam client at least once before SteamCMD can
+	//      pull it (some apps gate this)
+	//   5. Family Library Sharing — the account borrows Empyrion via
+	//      sharing instead of owning a license; SteamCMD requires real
+	//      ownership, not borrowed licenses
+	if strings.Contains(out, "ERROR! Download item") && strings.Contains(out, "Access Denied") {
+		// Account confirmed-owner of the app but still denied — this is
+		// almost always a publisher-side restriction on workshop_download_item
+		// (Empyrion / Eleon is the canonical example: the desktop Steam
+		// client uses a different API path that's allowed, but SteamCMD's
+		// path is denied for this app's workshop). NOT a panel bug, NOT an
+		// account bug. Manual upload is the only reliable path.
+		ownsApp := strings.Contains(out, "already owned")
+		var msg string
+		if ownsApp {
+			msg = fmt.Sprintf(
+				"Workshop item %s denied even though account %q owns app %s (Steam confirmed: \"AppID %s already owned\"). "+
+					"This is a publisher-side restriction on SteamCMD's workshop API — common for Empyrion (Eleon disables CLI workshop downloads; the desktop Steam client uses a different allowed path). "+
+					"Workaround: subscribe to the item in the desktop Steam client (Workshop → Subscribe), let it download to %%STEAM%%/steamapps/workshop/content/%s/%s/, then upload that folder into the instance's panel-empyrion-game volume at Content/Scenarios/<scenario_name>/. The panel can't bypass this — Steam policy enforced server-side.",
+				req.WorkshopId, req.SteamUsername, appID, appID, appID, req.WorkshopId,
+			)
+		} else {
+			msg = fmt.Sprintf(
+				"Steam returned Access Denied for workshop item %s after logging in as %q. "+
+					"Likely causes: (a) account doesn't actually own app %s (Family Library Sharing borrows don't count — needs a purchase license); "+
+					"(b) workshop item is private/friends-only/restricted by author; "+
+					"(c) item requires desktop-Steam-client subscription first. "+
+					"Workaround: download via desktop Steam client and drop Content/Scenarios/<folder> into the instance volume manually.",
+				req.WorkshopId, req.SteamUsername, appID,
+			)
+		}
+		finish(false, msg)
+		return
+	}
+	if !strings.Contains(out, "Success. Downloaded item") {
+		// Don't fail outright on a missing success marker — some
+		// SteamCMD versions return without that exact string when the
+		// item is already cached. Sanity-check the file actually
+		// exists in the volume before deciding.
+		probe := []string{"run", "--rm", "-v", "panel-empyrion-workshop:/ws", "alpine:3.20",
+			"sh", "-c", fmt.Sprintf("test -d /ws/content/%s/%s && echo cached", appID, req.WorkshopId)}
+		probeOut, _ := exec.CommandContext(ctx, dockerBin, probe...).CombinedOutput()
+		if !strings.Contains(string(probeOut), "cached") {
+			finish(false, "steamcmd did not download item — see log for details")
+			return
+		}
+		emit("[scenario] workshop item already cached on this agent — skipping re-download")
+	}
+
+	// Phase 2 — copy into instance volume.
+	emit("[scenario] copying scenario files into instance volume…")
+	gameVol := "panel-" + req.InstanceId + "-game"
+	dst := "/dst/Content/Scenarios/" + req.ScenarioName
+
+	// Verify the instance volume exists; if not, the instance was
+	// never created on this agent.
+	probe := []string{"volume", "inspect", gameVol}
+	if err := exec.CommandContext(ctx, dockerBin, probe...).Run(); err != nil {
+		finish(false, fmt.Sprintf("instance volume %q not found on this agent — is the empyrion instance created here?", gameVol))
+		return
+	}
+
+	helperArgs := []string{
+		"run", "--rm",
+		"-v", "panel-empyrion-workshop:/src",
+		"-v", gameVol + ":/dst",
+		"alpine:3.20", "sh", "-c",
+		fmt.Sprintf(
+			"mkdir -p %s && rm -rf %s/* %s/.* 2>/dev/null; "+
+				"cp -a /src/content/%s/%s/. %s/ && date > %s/.panel-installed && echo OK",
+			shQuote(dst), shQuote(dst), shQuote(dst),
+			appID, req.WorkshopId, shQuote(dst), shQuote(dst),
+		),
+	}
+	out, err = runWithLineEmit(ctx, dockerBin, helperArgs, emit)
+	if err != nil {
+		finish(false, "scenario copy failed: "+err.Error())
+		return
+	}
+	if !strings.Contains(out, "OK") {
+		finish(false, "scenario copy did not confirm OK — see log for details")
+		return
+	}
+
+	emit(fmt.Sprintf("[scenario] %q installed into %s", req.ScenarioName, req.InstanceId))
+	finish(true, "")
+}
+
+// shQuote single-quotes a path for safe inclusion in a `sh -c` script.
+// Embedded single quotes are escaped via the standard '\'' trick. Used
+// for scenario_name which is operator input — even though the controller
+// already rejects path-traversal characters, defense-in-depth at the
+// agent layer is cheap.
+func shQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// runWithLineEmit spawns docker, captures combined stdout/stderr,
+// emits each line as it arrives, and returns the full combined output
+// for post-mortem checks (success markers etc.) when the process exits.
+func runWithLineEmit(ctx context.Context, bin string, args []string, emit func(string)) (string, error) {
+	cmd := exec.CommandContext(ctx, bin, args...)
+	pipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+	cmd.Stderr = cmd.Stdout
+	if err := cmd.Start(); err != nil {
+		return "", err
+	}
+	var buf strings.Builder
+	scanCh := make(chan struct{})
+	go func() {
+		defer close(scanCh)
+		readBuf := make([]byte, 4096)
+		var line strings.Builder
+		for {
+			n, err := pipe.Read(readBuf)
+			if n > 0 {
+				chunk := readBuf[:n]
+				buf.Write(chunk)
+				for _, b := range chunk {
+					if b == '\n' {
+						emit(strings.TrimRight(line.String(), "\r"))
+						line.Reset()
+					} else {
+						line.WriteByte(b)
+					}
+				}
+			}
+			if err != nil {
+				if line.Len() > 0 {
+					emit(strings.TrimRight(line.String(), "\r"))
+				}
+				return
+			}
+		}
+	}()
+	waitErr := cmd.Wait()
+	<-scanCh
+	if waitErr != nil {
+		return buf.String(), waitErr
+	}
+	return buf.String(), nil
+}
+
+func (d *Dispatcher) sendEmpyrionScenarioResult(corrID, jobID string, ok bool, errMsg, scenarioName string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_EmpyrionScenarioInstallResult{
+			EmpyrionScenarioInstallResult: &panelv1.EmpyrionScenarioInstallResult{
+				JobId:        jobID,
+				Ok:           ok,
+				Error:        errMsg,
+				ScenarioName: scenarioName,
+			},
+		},
+	})
+}

agent/internal/dispatch/files.go

@@ -0,0 +1,662 @@
+package dispatch
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	agentmodule "github.com/dbledeez/panel/agent/internal/module"
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// File management: operations are rooted at each instance's declared
+// BrowseableRoot inside the container when the container is running +
+// the runtime supports container-path ops (Docker). Otherwise we fall
+// back to the host-side DataPath bind mount. Either path is safe-joined
+// to prevent escape.
+
+const (
+	// fsMaxReadBytes caps a single file-read RPC. Big enough for full
+	// world saves and most ARK/Empyrion scenario zips (a few-hundred MB
+	// is common). True streaming would remove the cap entirely; until
+	// then 4 GiB matches the controller-side upload cap.
+	fsMaxReadBytes = 4 * 1024 * 1024 * 1024
+	fsMaxListSize  = 5000
+
+	// fsCopyTimeout bounds a single CopyToContainer / CopyFromContainer
+	// call. A multi-GB upload over the loopback Docker socket can take
+	// 30+ seconds, and an extract that ships the whole archive's
+	// contents back to the container can be larger still — give it
+	// real headroom so we don't surface spurious timeouts.
+	fsCopyTimeout = 10 * time.Minute
+)
+
+// ---- Path safety ----
+
+// safeJoinHost joins rel onto root using OS path semantics, rejecting
+// anything that canonicalizes outside of root.
+func safeJoinHost(root, rel string) (string, error) {
+	rel = strings.TrimPrefix(rel, "/")
+	rel = strings.TrimPrefix(rel, "\\")
+	if rel == "" {
+		return root, nil
+	}
+	joined := filepath.Join(root, rel)
+	absRoot, err := filepath.Abs(root)
+	if err != nil {
+		return "", err
+	}
+	absJoined, err := filepath.Abs(joined)
+	if err != nil {
+		return "", err
+	}
+	sep := string(filepath.Separator)
+	if absJoined != absRoot && !strings.HasPrefix(absJoined+sep, absRoot+sep) {
+		return "", fmt.Errorf("path escapes instance data directory")
+	}
+	return absJoined, nil
+}
+
+// safeJoinContainer joins rel onto root using Linux container semantics
+// (always forward-slash). Returns the cleaned absolute path.
+func safeJoinContainer(root, rel string) (string, error) {
+	rel = strings.TrimPrefix(rel, "/")
+	if rel == "" {
+		return root, nil
+	}
+	joined := path.Clean(path.Join(root, rel))
+	if joined != root && !strings.HasPrefix(joined+"/", root+"/") {
+		return "", fmt.Errorf("path escapes instance root")
+	}
+	return joined, nil
+}
+
+// rootPaths returns the full list of allowed root paths for a record
+// (flattens BrowseableRoots → container paths, or falls back to the
+// singular BrowseableRoot). Always returns at least one entry.
+func rootPaths(rec *instanceRecord) []string {
+	if len(rec.BrowseableRoots) > 0 {
+		out := make([]string, 0, len(rec.BrowseableRoots))
+		for _, r := range rec.BrowseableRoots {
+			if r.Path != "" {
+				out = append(out, r.Path)
+			}
+		}
+		if len(out) > 0 {
+			return out
+		}
+	}
+	if rec.BrowseableRoot != "" {
+		return []string{rec.BrowseableRoot}
+	}
+	return nil
+}
+
+// safeJoinAny resolves a caller-supplied path against ANY of the module's
+// declared roots. Semantics:
+//
+//   - If p starts with "/", it must be EQUAL TO or UNDER one of the
+//     roots; we return it cleaned as-is. Lets the UI send container-
+//     absolute paths when it knows the full layout (multi-root modules).
+//   - Otherwise p is relative, resolved against `defaultRoot`. Same
+//     safeJoinContainer behaviour as before.
+//
+// Returns an error if the path can't be mapped to any root or tries to
+// escape via `..`.
+func safeJoinAny(defaultRoot string, roots []string, p string) (string, error) {
+	if strings.HasPrefix(p, "/") {
+		cleaned := path.Clean(p)
+		for _, r := range roots {
+			if cleaned == r || strings.HasPrefix(cleaned+"/", r+"/") {
+				return cleaned, nil
+			}
+		}
+		return "", fmt.Errorf("path %q is not under any declared root", cleaned)
+	}
+	return safeJoinContainer(defaultRoot, p)
+}
+
+func (d *Dispatcher) lookupRecord(instanceID string) (*instanceRecord, error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	rec, ok := d.instances[instanceID]
+	if !ok {
+		return nil, fmt.Errorf("instance %q not on this target", instanceID)
+	}
+	return rec, nil
+}
+
+// useContainerOps decides whether to go through the container runtime
+// (true) or fall back to the host bind mount (false). Any module that
+// declares a browseable root uses container ops — if the main container
+// isn't running at the moment, we spin up a helper sidecar (see
+// getFsTargetContainerID). This lets operators browse/edit files on
+// stopped instances.
+func (d *Dispatcher) useContainerOps(rec *instanceRecord) bool {
+	return rec.BrowseableRoot != ""
+}
+
+// ---- FS helper sidecar: file ops on stopped containers ----
+
+// fsHelperImage is the base for the helper container. We use debian-slim
+// rather than alpine because our `ls --time-style=+%s` call relies on GNU
+// coreutils — busybox's ls doesn't accept --time-style. debian:12-slim is
+// already on disk for every panel-native game module, so no extra pull.
+const fsHelperImage = "debian:12-slim"
+
+// fsHelperIdleTTL is how long a helper container is kept alive after its
+// last file-op access. Balance: long enough that click-around browsing
+// reuses the same helper without respawning; short enough that a forgotten
+// Files tab doesn't pin volumes forever.
+const fsHelperIdleTTL = 5 * time.Minute
+
+// getFsTargetContainerID picks the container to route a file op through:
+// the main instance container if it's running, or a helper sidecar
+// (lazy-created + started on demand) if it isn't. The helper reuses the
+// same named volumes as the main container, so the operator sees the
+// same files they would see with the game running.
+func (d *Dispatcher) getFsTargetContainerID(ctx context.Context, rec *instanceRecord) (string, error) {
+	mainName := "panel-" + rec.InstanceID
+	st, err := d.runtime.InspectByName(ctx, mainName)
+	if err == nil && st.Status == "running" {
+		// Main is alive — tear down any lingering helper so volumes
+		// aren't held by two containers simultaneously during live play.
+		d.teardownFsHelper(rec.InstanceID)
+		return st.ContainerID, nil
+	}
+	return d.ensureFsHelper(ctx, rec)
+}
+
+func (d *Dispatcher) ensureFsHelper(ctx context.Context, rec *instanceRecord) (string, error) {
+	helperName := "panel-" + rec.InstanceID + "-fs"
+
+	// Reuse existing helper if it's still healthy.
+	d.mu.Lock()
+	h, ok := d.fsHelpers[rec.InstanceID]
+	d.mu.Unlock()
+	if ok {
+		st, err := d.runtime.InspectByName(ctx, helperName)
+		if err == nil && st.Status == "running" {
+			d.mu.Lock()
+			h.lastAccess = time.Now()
+			d.mu.Unlock()
+			return h.containerID, nil
+		}
+		// Stale record — container gone or in a bad state. Drop + rebuild.
+		d.teardownFsHelper(rec.InstanceID)
+	}
+
+	// Check for an orphaned helper container on Docker's side from a
+	// previous agent run. When the agent restarts, in-memory fsHelpers
+	// empties but the container persists. Adopt it if it's running,
+	// remove it if not (Create will then succeed).
+	if st, err := d.runtime.InspectByName(ctx, helperName); err == nil {
+		if st.Status == "running" {
+			d.log.Info("fs helper adopted from prior agent run",
+				"instance_id", rec.InstanceID, "container_id", st.ContainerID)
+			d.mu.Lock()
+			d.fsHelpers[rec.InstanceID] = &fsHelper{containerID: st.ContainerID, lastAccess: time.Now()}
+			d.mu.Unlock()
+			return st.ContainerID, nil
+		}
+		d.log.Info("removing stale fs helper before recreate",
+			"instance_id", rec.InstanceID, "container_id", st.ContainerID, "status", st.Status)
+		rmCtx, rmCancel := context.WithTimeout(context.Background(), 5*time.Second)
+		_ = d.runtime.Remove(rmCtx, st.ContainerID)
+		rmCancel()
+	}
+
+	// Resolve the instance's volumes from its manifest so the helper sees
+	// exactly what the game would.
+	manifest, ok := d.modules.Get(rec.ModuleID)
+	if !ok {
+		return "", fmt.Errorf("module %q not loaded; can't mount helper volumes", rec.ModuleID)
+	}
+	volumes := agentmodule.ResolveVolumes(manifest, rec.InstanceID, rec.DataPath)
+
+	spec := runtime.InstanceSpec{
+		InstanceID:    rec.InstanceID + "-fs",
+		IsSidecar:     true,
+		Image:         fsHelperImage,
+		Command:       []string{"sleep", "infinity"},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+	containerID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		return "", fmt.Errorf("create fs helper: %w", err)
+	}
+	if err := d.runtime.Start(ctx, containerID); err != nil {
+		// Clean up the half-created container so next call doesn't see a
+		// zombie from `Created` state.
+		rmCtx, rmCancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer rmCancel()
+		_ = d.runtime.Remove(rmCtx, containerID)
+		return "", fmt.Errorf("start fs helper: %w", err)
+	}
+	d.mu.Lock()
+	d.fsHelpers[rec.InstanceID] = &fsHelper{containerID: containerID, lastAccess: time.Now()}
+	d.mu.Unlock()
+	d.log.Info("fs helper started", "instance_id", rec.InstanceID, "container_id", containerID)
+	return containerID, nil
+}
+
+// teardownFsHelper stops and removes the helper sidecar for an instance,
+// if one exists. Safe to call at any time — no-ops on absent helpers.
+// Called on main-container start (to free volumes) and on instance
+// delete (cleanup).
+func (d *Dispatcher) teardownFsHelper(instanceID string) {
+	d.mu.Lock()
+	h, ok := d.fsHelpers[instanceID]
+	if ok {
+		delete(d.fsHelpers, instanceID)
+	}
+	d.mu.Unlock()
+	if !ok {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_ = d.runtime.Stop(ctx, h.containerID, 2*time.Second)
+	_ = d.runtime.Remove(ctx, h.containerID)
+	d.log.Info("fs helper torn down", "instance_id", instanceID)
+}
+
+// fsHelperReaper runs for the lifetime of the dispatcher and tears down
+// any helper that hasn't been touched in fsHelperIdleTTL. Scans every
+// minute — granular enough that abandoned Files tabs don't pin resources
+// for long, coarse enough that the scan itself is cheap.
+func (d *Dispatcher) fsHelperReaper() {
+	ticker := time.NewTicker(1 * time.Minute)
+	defer ticker.Stop()
+	for range ticker.C {
+		now := time.Now()
+		d.mu.Lock()
+		stale := make([]string, 0)
+		for id, h := range d.fsHelpers {
+			if now.Sub(h.lastAccess) > fsHelperIdleTTL {
+				stale = append(stale, id)
+			}
+		}
+		d.mu.Unlock()
+		for _, id := range stale {
+			d.teardownFsHelper(id)
+		}
+	}
+}
+
+// ---- FsList ----
+
+func (d *Dispatcher) handleFsList(corrID string, req *panelv1.FsListRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsListResult(corrID, nil, "", err.Error())
+		return
+	}
+	if d.useContainerOps(rec) {
+		entries, err := d.listInContainer(rec, req.Path)
+		if err != nil {
+			d.sendFsListResult(corrID, nil, req.Path, err.Error())
+			return
+		}
+		d.sendFsListResult(corrID, entries, req.Path, "")
+		return
+	}
+	// Host fallback — bind mount.
+	if rec.DataPath == "" {
+		d.sendFsListResult(corrID, nil, req.Path, "no file storage available (container not running and no host data path)")
+		return
+	}
+	absPath, err := safeJoinHost(rec.DataPath, req.Path)
+	if err != nil {
+		d.sendFsListResult(corrID, nil, req.Path, err.Error())
+		return
+	}
+	entries, err := os.ReadDir(absPath)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			d.sendFsListResult(corrID, nil, req.Path, "not found")
+			return
+		}
+		d.sendFsListResult(corrID, nil, req.Path, err.Error())
+		return
+	}
+	out := make([]*panelv1.FsEntry, 0, len(entries))
+	for i, e := range entries {
+		if i >= fsMaxListSize {
+			break
+		}
+		info, err := e.Info()
+		if err != nil {
+			continue
+		}
+		out = append(out, &panelv1.FsEntry{
+			Name:    e.Name(),
+			Path:    filepath.ToSlash(filepath.Join(req.Path, e.Name())),
+			IsDir:   e.IsDir(),
+			Size:    info.Size(),
+			ModTime: timestamppb.New(info.ModTime()),
+		})
+	}
+	d.sendFsListResult(corrID, out, filepath.ToSlash(req.Path), "")
+}
+
+// listInContainer shells out to `ls -la --time-style=+%s <root/rel>` and
+// parses the output into FsEntry records.
+func (d *Dispatcher) listInContainer(rec *instanceRecord, rel string) ([]*panelv1.FsEntry, error) {
+	abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), rel)
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	targetID, err := d.getFsTargetContainerID(ctx, rec)
+	if err != nil {
+		return nil, err
+	}
+	// -L dereferences symlinks so that listing a `@ModName` link (which
+	// points into the shared workshop tree) shows the mod's contents, not
+	// the symlink itself. Without this, `ls -l` on a symlinked dir prints
+	// the symlink as a single entry and the UI looks empty.
+	stdout, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{
+		"sh", "-c",
+		"ls -lAL --time-style=+%s -- " + shellQuote(abs),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("exec ls: %w", err)
+	}
+	if code != 0 {
+		msg := strings.TrimSpace(string(stderr))
+		if msg == "" {
+			msg = fmt.Sprintf("ls exited %d", code)
+		}
+		return nil, fmt.Errorf("%s", msg)
+	}
+	return parseLsOutput(string(stdout), rel)
+}
+
+// parseLsOutput parses `ls -lA --time-style=+%s` output. Each line:
+//
+//	drwxr-xr-x  2 root root   4096 1735000000 somefile
+//
+// We split on any whitespace and take fields. The date field is a unix
+// timestamp (because of --time-style=+%s).
+func parseLsOutput(out, relRoot string) ([]*panelv1.FsEntry, error) {
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	entries := make([]*panelv1.FsEntry, 0, len(lines))
+	for i, line := range lines {
+		if i == 0 && strings.HasPrefix(line, "total ") {
+			continue
+		}
+		// Split into at most 7 fields so filenames with spaces survive.
+		fields := strings.Fields(line)
+		if len(fields) < 7 {
+			continue
+		}
+		// fields[0]=mode 1=nlink 2=user 3=group 4=size 5=unixts 6+=name
+		size, _ := strconv.ParseInt(fields[4], 10, 64)
+		tsUnix, _ := strconv.ParseInt(fields[5], 10, 64)
+		name := strings.Join(fields[6:], " ")
+		// For symlinks "name -> target", keep just the name.
+		if strings.Contains(name, " -> ") {
+			name = strings.SplitN(name, " -> ", 2)[0]
+		}
+		if name == "" || name == "." || name == ".." {
+			continue
+		}
+		isDir := strings.HasPrefix(fields[0], "d")
+		entries = append(entries, &panelv1.FsEntry{
+			Name:    name,
+			Path:    path.Join(relRoot, name),
+			IsDir:   isDir,
+			Size:    size,
+			ModTime: timestamppb.New(time.Unix(tsUnix, 0)),
+		})
+	}
+	return entries, nil
+}
+
+// shellQuote wraps s in single quotes, escaping embedded quotes for
+// interpolation into `sh -c`.
+func shellQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+func (d *Dispatcher) sendFsListResult(corrID string, entries []*panelv1.FsEntry, p, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsListResult{
+			FsListResult: &panelv1.FsListResult{Entries: entries, Path: p, Error: errMsg},
+		},
+	})
+}
+
+// ---- FsRead ----
+
+func (d *Dispatcher) handleFsRead(corrID string, req *panelv1.FsReadRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsReadResult(corrID, req.Path, nil, err.Error())
+		return
+	}
+	if d.useContainerOps(rec) {
+		abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
+		if err != nil {
+			d.sendFsReadResult(corrID, req.Path, nil, err.Error())
+			return
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		defer cancel()
+		targetID, err := d.getFsTargetContainerID(ctx, rec)
+		if err != nil {
+			d.sendFsReadResult(corrID, req.Path, nil, err.Error())
+			return
+		}
+		data, err := d.runtime.CopyFileFromContainer(ctx, targetID, abs)
+		if err != nil {
+			d.sendFsReadResult(corrID, req.Path, nil, err.Error())
+			return
+		}
+		if len(data) > fsMaxReadBytes {
+			d.sendFsReadResult(corrID, req.Path, nil, fmt.Sprintf("file too large (%d bytes; max %d)", len(data), fsMaxReadBytes))
+			return
+		}
+		d.sendFsReadResult(corrID, req.Path, data, "")
+		return
+	}
+	// Host fallback.
+	if rec.DataPath == "" {
+		d.sendFsReadResult(corrID, req.Path, nil, "no file storage available")
+		return
+	}
+	abs, err := safeJoinHost(rec.DataPath, req.Path)
+	if err != nil {
+		d.sendFsReadResult(corrID, req.Path, nil, err.Error())
+		return
+	}
+	info, err := os.Stat(abs)
+	if err != nil {
+		d.sendFsReadResult(corrID, req.Path, nil, err.Error())
+		return
+	}
+	if info.IsDir() {
+		d.sendFsReadResult(corrID, req.Path, nil, "is a directory")
+		return
+	}
+	if info.Size() > fsMaxReadBytes {
+		d.sendFsReadResult(corrID, req.Path, nil, fmt.Sprintf("file too large (%d bytes; max %d)", info.Size(), fsMaxReadBytes))
+		return
+	}
+	data, err := os.ReadFile(abs)
+	if err != nil {
+		d.sendFsReadResult(corrID, req.Path, nil, err.Error())
+		return
+	}
+	d.sendFsReadResult(corrID, req.Path, data, "")
+}
+
+func (d *Dispatcher) sendFsReadResult(corrID, p string, content []byte, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsReadResult{
+			FsReadResult: &panelv1.FsReadResult{Path: p, Content: content, Error: errMsg},
+		},
+	})
+}
+
+// ---- FsWrite ----
+
+func (d *Dispatcher) handleFsWrite(corrID string, req *panelv1.FsWriteRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsWriteResult(corrID, 0, err.Error())
+		return
+	}
+	if d.useContainerOps(rec) {
+		abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
+		if err != nil {
+			d.sendFsWriteResult(corrID, 0, err.Error())
+			return
+		}
+		// Make sure the parent dir exists. `mkdir -p` inside container.
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		targetID, terr := d.getFsTargetContainerID(ctx, rec)
+		if terr != nil {
+			cancel()
+			d.sendFsWriteResult(corrID, 0, terr.Error())
+			return
+		}
+		_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+			"mkdir -p -- " + shellQuote(path.Dir(abs))})
+		cancel()
+		ctx2, cancel2 := context.WithTimeout(context.Background(), fsCopyTimeout)
+		defer cancel2()
+		if err := d.runtime.CopyFileToContainer(ctx2, targetID, abs, req.Content); err != nil {
+			d.sendFsWriteResult(corrID, 0, err.Error())
+			return
+		}
+		d.sendFsWriteResult(corrID, int64(len(req.Content)), "")
+		return
+	}
+	// Host fallback.
+	if rec.DataPath == "" {
+		d.sendFsWriteResult(corrID, 0, "no file storage available")
+		return
+	}
+	abs, err := safeJoinHost(rec.DataPath, req.Path)
+	if err != nil {
+		d.sendFsWriteResult(corrID, 0, err.Error())
+		return
+	}
+	if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
+		d.sendFsWriteResult(corrID, 0, err.Error())
+		return
+	}
+	if err := os.WriteFile(abs, req.Content, 0o644); err != nil {
+		d.sendFsWriteResult(corrID, 0, err.Error())
+		return
+	}
+	d.sendFsWriteResult(corrID, int64(len(req.Content)), "")
+}
+
+func (d *Dispatcher) sendFsWriteResult(corrID string, bytesWritten int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsWriteResult{
+			FsWriteResult: &panelv1.FsWriteResult{BytesWritten: bytesWritten, Error: errMsg},
+		},
+	})
+}
+
+// ---- FsDelete ----
+
+func (d *Dispatcher) handleFsDelete(corrID string, req *panelv1.FsDeleteRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsDeleteResult(corrID, err.Error())
+		return
+	}
+	if d.useContainerOps(rec) {
+		abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
+		if err != nil {
+			d.sendFsDeleteResult(corrID, err.Error())
+			return
+		}
+		if abs == rec.BrowseableRoot {
+			d.sendFsDeleteResult(corrID, "refusing to delete instance root")
+			return
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		defer cancel()
+		targetID, err := d.getFsTargetContainerID(ctx, rec)
+		if err != nil {
+			d.sendFsDeleteResult(corrID, err.Error())
+			return
+		}
+		flag := ""
+		if req.Recursive {
+			flag = "-rf"
+		} else {
+			flag = "-f"
+		}
+		_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+			"rm " + flag + " -- " + shellQuote(abs)})
+		if err != nil {
+			d.sendFsDeleteResult(corrID, err.Error())
+			return
+		}
+		if code != 0 {
+			d.sendFsDeleteResult(corrID, strings.TrimSpace(string(stderr)))
+			return
+		}
+		d.sendFsDeleteResult(corrID, "")
+		return
+	}
+	// Host fallback.
+	if rec.DataPath == "" {
+		d.sendFsDeleteResult(corrID, "no file storage available")
+		return
+	}
+	abs, err := safeJoinHost(rec.DataPath, req.Path)
+	if err != nil {
+		d.sendFsDeleteResult(corrID, err.Error())
+		return
+	}
+	if abs == rec.DataPath {
+		d.sendFsDeleteResult(corrID, "refusing to delete instance data root")
+		return
+	}
+	var delErr error
+	if req.Recursive {
+		delErr = os.RemoveAll(abs)
+	} else {
+		delErr = os.Remove(abs)
+	}
+	if delErr != nil {
+		d.sendFsDeleteResult(corrID, delErr.Error())
+		return
+	}
+	d.sendFsDeleteResult(corrID, "")
+}
+
+func (d *Dispatcher) sendFsDeleteResult(corrID, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsDeleteResult{
+			FsDeleteResult: &panelv1.FsDeleteResult{Error: errMsg},
+		},
+	})
+}

agent/internal/dispatch/files_archive.go

@@ -0,0 +1,511 @@
+package dispatch
+
+// Archive operations for the file browser:
+//
+//   - FsExtract  : expand a zip / tar / tar.gz / tar.bz2 archive that
+//                  already lives inside the instance's volume.
+//   - FsCompress : zip up one or more paths inside the instance into a
+//                  destination .zip in the same volume.
+//   - FsRename   : safe `mv` that respects the browseable_root sandbox.
+//
+// Every container-side path goes through the existing fs helper sidecar
+// (see getFsTargetContainerID) so these all work on stopped instances.
+//
+// Implementation notes:
+//   - Extract reads the archive bytes via CopyFileFromContainer, walks
+//     the entries with the appropriate stdlib reader, and re-emits a
+//     single tar stream that's piped to CopyTarToContainer in one API
+//     call. That avoids N per-file writes.
+//   - Compress walks each source via CopyDirFromContainer (tar stream
+//     from Docker), and re-encodes the entries as zip. The final zip
+//     bytes go back via CopyFileToContainer.
+//   - We support zip + tar + tar.gz + tar.bz2 + tar.xz + 7z + rar.
+//     7z and rar use pure-Go libraries (bodgit/sevenzip,
+//     nwaples/rardecode/v2) so no native deps. Encrypted rars and
+//     solid 7z volumes are surfaced as errors, not silently skipped.
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/dbledeez/panel/agent/internal/archive"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// ---- FsExtract ----
+
+func (d *Dispatcher) handleFsExtract(corrID string, req *panelv1.FsExtractRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+	if req.ArchivePath == "" {
+		d.sendFsExtractResult(corrID, 0, 0, "archive_path is required")
+		return
+	}
+	if d.useContainerOps(rec) {
+		d.extractInContainer(corrID, rec, req)
+		return
+	}
+	d.extractOnHost(corrID, rec, req)
+}
+
+func (d *Dispatcher) extractInContainer(corrID string, rec *instanceRecord, req *panelv1.FsExtractRequest) {
+	archiveAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.ArchivePath)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+	destRel := req.DestDir
+	if destRel == "" {
+		// Default: same dir as the archive.
+		destRel = path.Dir(req.ArchivePath)
+	}
+	destAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), destRel)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
+	defer cancel()
+	targetID, err := d.getFsTargetContainerID(ctx, rec)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+
+	// Pull the archive bytes back from the container. CopyFileFromContainer
+	// internally tar-walks Docker's response and returns just the file body.
+	archiveData, err := d.runtime.CopyFileFromContainer(ctx, targetID, archiveAbs)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, fmt.Errorf("read archive: %w", err).Error())
+		return
+	}
+
+	// Make sure the destination dir exists before we ship the tar stream.
+	_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		"mkdir -p -- " + shellQuote(destAbs)})
+
+	// Build a tar stream of the archive's contents and pipe it to
+	// CopyTarToContainer. Doing this in a single round-trip keeps the
+	// wall-time of "extract a 500 MB archive with 4000 entries" manageable.
+	pr, pw := io.Pipe()
+	emitErr := make(chan error, 1)
+	var entries, bytesOut int64
+	go func() {
+		n, b, err := archive.WriteAsTar(archiveData, req.ArchivePath, pw)
+		entries, bytesOut = n, b
+		_ = pw.CloseWithError(err)
+		emitErr <- err
+	}()
+	if err := d.runtime.CopyTarToContainer(ctx, targetID, destAbs, pr); err != nil {
+		// Pipe gets closed with our error from the goroutine; whichever
+		// side surfaces first wins. We prefer the goroutine error since
+		// it carries archive-format detail.
+		if encErr := <-emitErr; encErr != nil {
+			d.sendFsExtractResult(corrID, entries, bytesOut, encErr.Error())
+			return
+		}
+		d.sendFsExtractResult(corrID, entries, bytesOut, fmt.Errorf("copy to container: %w", err).Error())
+		return
+	}
+	if encErr := <-emitErr; encErr != nil {
+		d.sendFsExtractResult(corrID, entries, bytesOut, encErr.Error())
+		return
+	}
+	d.sendFsExtractResult(corrID, entries, bytesOut, "")
+}
+
+// extractOnHost handles the rare host-fallback case (no container, no
+// helper) — uses os primitives. Same path-safety contract as everywhere
+// else: must stay under the instance's data path.
+func (d *Dispatcher) extractOnHost(corrID string, rec *instanceRecord, req *panelv1.FsExtractRequest) {
+	if rec.DataPath == "" {
+		d.sendFsExtractResult(corrID, 0, 0, "no file storage available")
+		return
+	}
+	archiveAbs, err := safeJoinHost(rec.DataPath, req.ArchivePath)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+	destRel := req.DestDir
+	if destRel == "" {
+		destRel = filepath.Dir(req.ArchivePath)
+	}
+	destAbs, err := safeJoinHost(rec.DataPath, destRel)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+	if err := os.MkdirAll(destAbs, 0o755); err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+	data, err := os.ReadFile(archiveAbs)
+	if err != nil {
+		d.sendFsExtractResult(corrID, 0, 0, err.Error())
+		return
+	}
+	entries, bytesOut, err := archive.Walk(data, req.ArchivePath, func(name string, mode int64, isDir bool, body io.Reader) error {
+		safeName := archive.SafeEntryPath(name)
+		if safeName == "" {
+			return nil
+		}
+		out := filepath.Join(destAbs, filepath.FromSlash(safeName))
+		if isDir {
+			return os.MkdirAll(out, 0o755)
+		}
+		if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
+			return err
+		}
+		f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(mode&0o777))
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(f, body); err != nil {
+			_ = f.Close()
+			return err
+		}
+		return f.Close()
+	})
+	if err != nil {
+		d.sendFsExtractResult(corrID, entries, bytesOut, err.Error())
+		return
+	}
+	d.sendFsExtractResult(corrID, entries, bytesOut, "")
+}
+
+func (d *Dispatcher) sendFsExtractResult(corrID string, entries, bytesOut int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsExtractResult{
+			FsExtractResult: &panelv1.FsExtractResult{EntriesWritten: entries, BytesWritten: bytesOut, Error: errMsg},
+		},
+	})
+}
+
+// ---- FsCompress ----
+
+func (d *Dispatcher) handleFsCompress(corrID string, req *panelv1.FsCompressRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsCompressResult(corrID, 0, 0, err.Error())
+		return
+	}
+	if len(req.Sources) == 0 {
+		d.sendFsCompressResult(corrID, 0, 0, "at least one source path is required")
+		return
+	}
+	if req.DestZipPath == "" {
+		d.sendFsCompressResult(corrID, 0, 0, "dest_zip_path is required")
+		return
+	}
+	if d.useContainerOps(rec) {
+		d.compressInContainer(corrID, rec, req)
+		return
+	}
+	d.compressOnHost(corrID, rec, req)
+}
+
+func (d *Dispatcher) compressInContainer(corrID string, rec *instanceRecord, req *panelv1.FsCompressRequest) {
+	destAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.DestZipPath)
+	if err != nil {
+		d.sendFsCompressResult(corrID, 0, 0, err.Error())
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
+	defer cancel()
+	targetID, err := d.getFsTargetContainerID(ctx, rec)
+	if err != nil {
+		d.sendFsCompressResult(corrID, 0, 0, err.Error())
+		return
+	}
+
+	var zipBuf bytes.Buffer
+	zw := zip.NewWriter(&zipBuf)
+	var entries, bytesOut int64
+
+	for _, src := range req.Sources {
+		srcAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), src)
+		if err != nil {
+			d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
+			return
+		}
+		// CopyDirFromContainer returns Docker's tar stream of srcAbs.
+		// For files: a single-entry tar. For dirs: every file under it.
+		// The base name of srcAbs is the prefix Docker uses, which we
+		// keep in the zip so the user gets familiar names back.
+		rc, err := d.runtime.CopyDirFromContainer(ctx, targetID, srcAbs)
+		if err != nil {
+			d.sendFsCompressResult(corrID, entries, bytesOut, fmt.Errorf("copy from container %q: %w", src, err).Error())
+			return
+		}
+		n, b, err := pipeTarIntoZip(rc, zw)
+		_ = rc.Close()
+		if err != nil {
+			d.sendFsCompressResult(corrID, entries+n, bytesOut+b, fmt.Errorf("encode %q: %w", src, err).Error())
+			return
+		}
+		entries += n
+		bytesOut += b
+	}
+	if err := zw.Close(); err != nil {
+		d.sendFsCompressResult(corrID, entries, bytesOut, fmt.Errorf("zip close: %w", err).Error())
+		return
+	}
+	// Drop the resulting zip into the volume.
+	_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+		"mkdir -p -- " + shellQuote(path.Dir(destAbs))})
+	if err := d.runtime.CopyFileToContainer(ctx, targetID, destAbs, zipBuf.Bytes()); err != nil {
+		d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
+		return
+	}
+	d.sendFsCompressResult(corrID, entries, bytesOut, "")
+}
+
+func (d *Dispatcher) compressOnHost(corrID string, rec *instanceRecord, req *panelv1.FsCompressRequest) {
+	if rec.DataPath == "" {
+		d.sendFsCompressResult(corrID, 0, 0, "no file storage available")
+		return
+	}
+	destAbs, err := safeJoinHost(rec.DataPath, req.DestZipPath)
+	if err != nil {
+		d.sendFsCompressResult(corrID, 0, 0, err.Error())
+		return
+	}
+	if err := os.MkdirAll(filepath.Dir(destAbs), 0o755); err != nil {
+		d.sendFsCompressResult(corrID, 0, 0, err.Error())
+		return
+	}
+	f, err := os.Create(destAbs)
+	if err != nil {
+		d.sendFsCompressResult(corrID, 0, 0, err.Error())
+		return
+	}
+	zw := zip.NewWriter(f)
+	var entries, bytesOut int64
+	for _, src := range req.Sources {
+		srcAbs, err := safeJoinHost(rec.DataPath, src)
+		if err != nil {
+			zw.Close()
+			f.Close()
+			d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
+			return
+		}
+		base := filepath.Base(srcAbs)
+		err = filepath.Walk(srcAbs, func(p string, info os.FileInfo, walkErr error) error {
+			if walkErr != nil {
+				return walkErr
+			}
+			rel, err := filepath.Rel(srcAbs, p)
+			if err != nil {
+				return err
+			}
+			zipName := base
+			if rel != "." {
+				zipName = filepath.ToSlash(filepath.Join(base, rel))
+			}
+			if info.IsDir() {
+				if zipName == "" || zipName == "." {
+					return nil
+				}
+				_, err = zw.Create(zipName + "/")
+				return err
+			}
+			zf, err := zw.Create(zipName)
+			if err != nil {
+				return err
+			}
+			rf, err := os.Open(p)
+			if err != nil {
+				return err
+			}
+			defer rf.Close()
+			n, err := io.Copy(zf, rf)
+			if err != nil {
+				return err
+			}
+			entries++
+			bytesOut += n
+			return nil
+		})
+		if err != nil {
+			zw.Close()
+			f.Close()
+			d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
+			return
+		}
+	}
+	if err := zw.Close(); err != nil {
+		f.Close()
+		d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
+		return
+	}
+	if err := f.Close(); err != nil {
+		d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
+		return
+	}
+	d.sendFsCompressResult(corrID, entries, bytesOut, "")
+}
+
+// pipeTarIntoZip walks a docker-style tar stream and writes each
+// regular-file entry into the zip writer, preserving relative paths.
+// Directory entries are written as zero-byte zip directory markers.
+func pipeTarIntoZip(r io.Reader, zw *zip.Writer) (int64, int64, error) {
+	tr := tar.NewReader(r)
+	var entries, bytesOut, totalBytes int64
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return entries, bytesOut, nil
+		}
+		if err != nil {
+			return entries, bytesOut, err
+		}
+		safe := archive.SafeEntryPath(hdr.Name)
+		if safe == "" {
+			continue
+		}
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if _, err := zw.Create(safe + "/"); err != nil {
+				return entries, bytesOut, err
+			}
+			entries++
+		case tar.TypeReg, tar.TypeRegA:
+			zf, err := zw.Create(safe)
+			if err != nil {
+				return entries, bytesOut, err
+			}
+			n, err := io.Copy(zf, tr)
+			if err != nil {
+				return entries, bytesOut, err
+			}
+			entries++
+			bytesOut += n
+			totalBytes += n
+			if totalBytes > archive.MaxTotalBytes {
+				return entries, bytesOut, fmt.Errorf("compress: total uncompressed bytes exceed %d", archive.MaxTotalBytes)
+			}
+		}
+	}
+}
+
+func (d *Dispatcher) sendFsCompressResult(corrID string, entries, bytesOut int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsCompressResult{
+			FsCompressResult: &panelv1.FsCompressResult{EntriesWritten: entries, BytesWritten: bytesOut, Error: errMsg},
+		},
+	})
+}
+
+// ---- FsRename ----
+
+func (d *Dispatcher) handleFsRename(corrID string, req *panelv1.FsRenameRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendFsRenameResult(corrID, err.Error())
+		return
+	}
+	if req.FromPath == "" || req.ToPath == "" {
+		d.sendFsRenameResult(corrID, "from_path and to_path are required")
+		return
+	}
+	if d.useContainerOps(rec) {
+		fromAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.FromPath)
+		if err != nil {
+			d.sendFsRenameResult(corrID, err.Error())
+			return
+		}
+		toAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.ToPath)
+		if err != nil {
+			d.sendFsRenameResult(corrID, err.Error())
+			return
+		}
+		if fromAbs == rec.BrowseableRoot {
+			d.sendFsRenameResult(corrID, "refusing to rename instance root")
+			return
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		targetID, err := d.getFsTargetContainerID(ctx, rec)
+		if err != nil {
+			d.sendFsRenameResult(corrID, err.Error())
+			return
+		}
+		// mkdir parent so a "rename to subdir" implicitly creates it.
+		_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+			"mkdir -p -- " + shellQuote(path.Dir(toAbs))})
+		_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+			"mv -- " + shellQuote(fromAbs) + " " + shellQuote(toAbs)})
+		if err != nil {
+			d.sendFsRenameResult(corrID, err.Error())
+			return
+		}
+		if code != 0 {
+			msg := strings.TrimSpace(string(stderr))
+			if msg == "" {
+				msg = fmt.Sprintf("mv exited %d", code)
+			}
+			d.sendFsRenameResult(corrID, msg)
+			return
+		}
+		d.sendFsRenameResult(corrID, "")
+		return
+	}
+	// Host fallback.
+	if rec.DataPath == "" {
+		d.sendFsRenameResult(corrID, "no file storage available")
+		return
+	}
+	fromAbs, err := safeJoinHost(rec.DataPath, req.FromPath)
+	if err != nil {
+		d.sendFsRenameResult(corrID, err.Error())
+		return
+	}
+	toAbs, err := safeJoinHost(rec.DataPath, req.ToPath)
+	if err != nil {
+		d.sendFsRenameResult(corrID, err.Error())
+		return
+	}
+	if fromAbs == rec.DataPath {
+		d.sendFsRenameResult(corrID, "refusing to rename instance root")
+		return
+	}
+	if err := os.MkdirAll(filepath.Dir(toAbs), 0o755); err != nil {
+		d.sendFsRenameResult(corrID, err.Error())
+		return
+	}
+	if err := os.Rename(fromAbs, toAbs); err != nil {
+		d.sendFsRenameResult(corrID, err.Error())
+		return
+	}
+	d.sendFsRenameResult(corrID, "")
+}
+
+func (d *Dispatcher) sendFsRenameResult(corrID, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsRenameResult{
+			FsRenameResult: &panelv1.FsRenameResult{Error: errMsg},
+		},
+	})
+}

agent/internal/dispatch/files_chunked.go

@@ -0,0 +1,337 @@
+package dispatch
+
+// Chunked file upload — solves three problems at once:
+//
+//   1. Cloudflare's request-body cap (~100 MB free, 200 MB pro) silently
+//      stalls "single-shot" uploads of large mod / scenario archives
+//      that go through the panel's public hostname.
+//   2. The previous single-shot path buffered the entire body in memory
+//      on both the controller and the agent, peaking at ~3× file size.
+//      A 3 GB upload would thrash a smaller box.
+//   3. Single-shot uploads block the agent's bidi stream for the
+//      duration of the transfer; chunks are small and interleaved.
+//
+// Wire protocol (all on the existing AgentEnvelope):
+//
+//   FsWriteChunkRequest { upload_id, instance_id, path, offset, data,
+//                         total_size, is_final }
+//   FsWriteChunkResult  { bytes_received, total_received, finalized,
+//                         error }
+//
+// The agent owns one `uploadSession` per upload_id. Each session has an
+// `*os.File` open in its scratch dir. Chunks append in order. On
+// is_final the agent ships the file to the container via a streaming
+// CopyTarToContainer call (no full-file in-memory copy), then unlinks.
+// Stale sessions are reaped after uploadIdleTTL.
+
+import (
+	"archive/tar"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+const uploadIdleTTL = 30 * time.Minute
+
+type uploadSession struct {
+	mu         sync.Mutex
+	uploadID   string
+	instanceID string
+	path       string
+	tempPath   string
+	f          *os.File
+	bytes      int64
+	total      int64
+	lastTouch  time.Time
+}
+
+func (d *Dispatcher) uploadScratchDir() string {
+	dir := filepath.Join(d.dataRoot, ".panel-uploads")
+	_ = os.MkdirAll(dir, 0o755)
+	return dir
+}
+
+// handleFsWriteChunk processes one chunk in the streaming-upload protocol.
+// First chunk creates the session; subsequent chunks append; is_final
+// triggers a streaming copy into the container and cleans up.
+func (d *Dispatcher) handleFsWriteChunk(corrID string, req *panelv1.FsWriteChunkRequest) {
+	if req.UploadId == "" {
+		d.sendChunkResult(corrID, 0, 0, false, "upload_id is required")
+		return
+	}
+	if req.InstanceId == "" || req.Path == "" {
+		d.sendChunkResult(corrID, 0, 0, false, "instance_id and path are required")
+		return
+	}
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendChunkResult(corrID, 0, 0, false, err.Error())
+		return
+	}
+
+	d.mu.Lock()
+	sess, ok := d.uploads[req.UploadId]
+	if !ok {
+		// First chunk for this upload — create session + temp file.
+		// Use the upload_id in the filename so a recovered scratch dir
+		// is self-describing for forensic purposes.
+		base := filepath.Join(d.uploadScratchDir(), "u-"+sanitizeUploadID(req.UploadId))
+		f, err := os.OpenFile(base, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+		if err != nil {
+			d.mu.Unlock()
+			d.sendChunkResult(corrID, 0, 0, false, fmt.Errorf("open temp: %w", err).Error())
+			return
+		}
+		sess = &uploadSession{
+			uploadID:   req.UploadId,
+			instanceID: req.InstanceId,
+			path:       req.Path,
+			tempPath:   base,
+			f:          f,
+			total:      req.TotalSize,
+			lastTouch:  time.Now(),
+		}
+		d.uploads[req.UploadId] = sess
+	}
+	d.mu.Unlock()
+
+	sess.mu.Lock()
+	defer sess.mu.Unlock()
+
+	// Reject stray chunks for a session that's already closed (final
+	// chunk processed earlier — operator double-clicked, retry, etc.).
+	if sess.f == nil {
+		d.sendChunkResult(corrID, 0, sess.bytes, true, "")
+		return
+	}
+
+	// Allow operator to upload to a renamed path mid-stream? No — pin
+	// the destination from the first chunk for safety. Subsequent
+	// chunks with a different path get rejected.
+	if sess.path != req.Path {
+		d.sendChunkResult(corrID, 0, sess.bytes, false, "path differs from initial chunk")
+		return
+	}
+
+	// Append the chunk. The browser is expected to send chunks in
+	// order; if offset doesn't match the running tail we err so the
+	// frontend can fall back / retry rather than silently writing a
+	// hole.
+	if req.Offset != sess.bytes {
+		d.sendChunkResult(corrID, 0, sess.bytes, false,
+			fmt.Sprintf("chunk offset %d != expected %d (out-of-order chunk)", req.Offset, sess.bytes))
+		return
+	}
+	n, err := sess.f.Write(req.Data)
+	if err != nil {
+		d.sendChunkResult(corrID, 0, sess.bytes, false, fmt.Errorf("write chunk: %w", err).Error())
+		return
+	}
+	sess.bytes += int64(n)
+	sess.lastTouch = time.Now()
+
+	if !req.IsFinal {
+		d.sendChunkResult(corrID, int64(n), sess.bytes, false, "")
+		return
+	}
+
+	// Final chunk — close the temp file, ship it to the container.
+	if err := sess.f.Sync(); err != nil {
+		d.sendChunkResult(corrID, int64(n), sess.bytes, false, fmt.Errorf("fsync: %w", err).Error())
+		return
+	}
+	if err := sess.f.Close(); err != nil {
+		d.sendChunkResult(corrID, int64(n), sess.bytes, false, fmt.Errorf("close temp: %w", err).Error())
+		return
+	}
+	sess.f = nil
+	if sess.total > 0 && sess.bytes != sess.total {
+		d.sendChunkResult(corrID, int64(n), sess.bytes, false,
+			fmt.Sprintf("size mismatch: received %d bytes, expected %d", sess.bytes, sess.total))
+		return
+	}
+
+	// Ship to the container (or host fallback) without re-reading the
+	// whole file into memory — we hand Docker an io.Reader that
+	// streams from disk, wrapped in a tar header on the fly.
+	if err := d.deliverUploadedFile(rec, sess); err != nil {
+		d.sendChunkResult(corrID, int64(n), sess.bytes, false, err.Error())
+		return
+	}
+
+	// Tear down the session — temp file deleted, map slot freed.
+	_ = os.Remove(sess.tempPath)
+	d.mu.Lock()
+	delete(d.uploads, sess.uploadID)
+	d.mu.Unlock()
+
+	d.sendChunkResult(corrID, int64(n), sess.bytes, true, "")
+}
+
+// deliverUploadedFile streams the temp file into the destination,
+// either via container CopyTarToContainer (running or helper sidecar)
+// or via a plain os.Rename for host-fallback instances. Designed so
+// the file's bytes never sit in memory in their entirety.
+func (d *Dispatcher) deliverUploadedFile(rec *instanceRecord, sess *uploadSession) error {
+	if d.useContainerOps(rec) {
+		abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), sess.path)
+		if err != nil {
+			return err
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
+		defer cancel()
+		targetID, err := d.getFsTargetContainerID(ctx, rec)
+		if err != nil {
+			return err
+		}
+		// mkdir -p the parent so the destination always exists.
+		_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
+			"mkdir -p -- " + shellQuote(path.Dir(abs))})
+
+		// Open the temp file for read; build a tar wrapper on the fly
+		// and feed Docker. CopyTarToContainer streams the body without
+		// materializing it in memory.
+		fr, err := os.Open(sess.tempPath)
+		if err != nil {
+			return err
+		}
+		defer fr.Close()
+		fi, err := fr.Stat()
+		if err != nil {
+			return err
+		}
+
+		pr, pw := io.Pipe()
+		errCh := make(chan error, 1)
+		go func() {
+			tw := tar.NewWriter(pw)
+			err := tw.WriteHeader(&tar.Header{
+				Name: path.Base(abs),
+				Mode: 0o644,
+				Size: fi.Size(),
+			})
+			if err == nil {
+				_, err = io.Copy(tw, fr)
+			}
+			if err == nil {
+				err = tw.Close()
+			}
+			_ = pw.CloseWithError(err)
+			errCh <- err
+		}()
+		if err := d.runtime.CopyTarToContainer(ctx, targetID, path.Dir(abs), pr); err != nil {
+			if encErr := <-errCh; encErr != nil {
+				return encErr
+			}
+			return fmt.Errorf("copy to container: %w", err)
+		}
+		if encErr := <-errCh; encErr != nil {
+			return encErr
+		}
+		return nil
+	}
+
+	// Host fallback — just move the temp file into place.
+	if rec.DataPath == "" {
+		return fmt.Errorf("no file storage available")
+	}
+	abs, err := safeJoinHost(rec.DataPath, sess.path)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
+		return err
+	}
+	// os.Rename is atomic on the same filesystem; if scratch lives on
+	// a different fs we fall back to copy + delete.
+	if err := os.Rename(sess.tempPath, abs); err == nil {
+		return nil
+	}
+	src, err := os.Open(sess.tempPath)
+	if err != nil {
+		return err
+	}
+	defer src.Close()
+	dst, err := os.OpenFile(abs, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+	if _, err := io.Copy(dst, src); err != nil {
+		return err
+	}
+	return nil
+}
+
+// uploadReaper closes + removes any session that hasn't been touched
+// in uploadIdleTTL. Runs forever; cheap enough to scan once a minute.
+func (d *Dispatcher) uploadReaper() {
+	t := time.NewTicker(time.Minute)
+	defer t.Stop()
+	for range t.C {
+		now := time.Now()
+		d.mu.Lock()
+		stale := []*uploadSession{}
+		for id, s := range d.uploads {
+			if now.Sub(s.lastTouch) > uploadIdleTTL {
+				stale = append(stale, s)
+				delete(d.uploads, id)
+			}
+		}
+		d.mu.Unlock()
+		for _, s := range stale {
+			s.mu.Lock()
+			if s.f != nil {
+				_ = s.f.Close()
+				s.f = nil
+			}
+			_ = os.Remove(s.tempPath)
+			s.mu.Unlock()
+			d.log.Info("upload session reaped", "upload_id", s.uploadID, "bytes", s.bytes)
+		}
+	}
+}
+
+// sanitizeUploadID strips characters that could escape the scratch dir.
+// upload_id is generated by the browser (UUID-like) — defense in depth.
+func sanitizeUploadID(s string) string {
+	out := make([]byte, 0, len(s))
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		switch {
+		case (c >= 'a' && c <= 'z'), (c >= 'A' && c <= 'Z'), (c >= '0' && c <= '9'), c == '-', c == '_':
+			out = append(out, c)
+		}
+		if len(out) > 64 {
+			break
+		}
+	}
+	if len(out) == 0 {
+		out = []byte("anon")
+	}
+	return string(out)
+}
+
+func (d *Dispatcher) sendChunkResult(corrID string, n, total int64, finalized bool, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_FsWriteChunkResult{
+			FsWriteChunkResult: &panelv1.FsWriteChunkResult{
+				BytesReceived: n,
+				TotalReceived: total,
+				Finalized:     finalized,
+				Error:         errMsg,
+			},
+		},
+	})
+}

agent/internal/dispatch/hangguard.go

@@ -0,0 +1,180 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// hangGuard implements a belt-and-suspenders hang-detection guardrail for
+// ARK Survival Ascended (ark-sa) instances running under Wine 9.
+// The Wine 9 RCON listener can wedge under sustained traffic; the engine
+// also sometimes emits a "!!!HANG DETECTED!!!" line on its own.
+// This guard auto-restarts the docker container in either case, with a
+// cooldown to prevent flapping.
+// The root-cause fix (docker_exec_rcon adapter) shipped 2026-05-03; this
+// guard handles the rare residual cases.
+// It replaces the bash watchdog that previously ran on princess via systemd timer.
+
+const (
+	hangGuardWindow        = 10 * time.Minute
+	hangGuardFailThreshold = 5
+	hangGuardCooldown      = 10 * time.Minute
+	hangGuardEngineMarker  = "!!!HANG DETECTED!!!"
+)
+
+// hangGuardState holds per-instance hang-detection state.
+// Embed by value in instanceRecord; zero value is ready to use.
+type hangGuardState struct {
+	mu              sync.Mutex
+	consecutiveFails int
+	windowStart     time.Time
+	lastRestart     time.Time
+	restarting      atomic.Bool
+}
+
+// arkHangGuardOnLogLine checks for engine-side hang detection tokens
+// in console log lines. Only applies to ark-sa instances.
+func (d *Dispatcher) arkHangGuardOnLogLine(rec *instanceRecord, line string) {
+	if rec.ModuleID != "ark-sa" {
+		return
+	}
+	if !strings.Contains(line, hangGuardEngineMarker) {
+		return
+	}
+	// Engine self-detected a hang; trigger restart asynchronously.
+	go d.arkHangGuardRestart(rec, fmt.Sprintf("engine emitted '%s'", hangGuardEngineMarker))
+}
+
+// arkHangGuardOnRconResult tracks RCON call successes/failures for
+// hang detection. A threshold of consecutive failures triggers a restart.
+func (d *Dispatcher) arkHangGuardOnRconResult(rec *instanceRecord, success bool) {
+	if rec.ModuleID != "ark-sa" {
+		return
+	}
+
+	rec.hangGuard.mu.Lock()
+	defer rec.hangGuard.mu.Unlock()
+
+	if success {
+		rec.hangGuard.consecutiveFails = 0
+		rec.hangGuard.windowStart = time.Time{}
+		return
+	}
+
+	now := time.Now()
+	if rec.hangGuard.windowStart.IsZero() || now.Sub(rec.hangGuard.windowStart) > hangGuardWindow {
+		// Start a new window
+		rec.hangGuard.windowStart = now
+		rec.hangGuard.consecutiveFails = 1
+	} else {
+		rec.hangGuard.consecutiveFails++
+	}
+
+	if rec.hangGuard.consecutiveFails >= hangGuardFailThreshold {
+		go d.arkHangGuardRestart(rec, fmt.Sprintf("%d RCON failures in last %v", rec.hangGuard.consecutiveFails, hangGuardWindow))
+	}
+}
+
+// arkHangGuardRestart performs the actual container restart, respecting a
+// cooldown and using a single-flight pattern to prevent concurrent restarts.
+func (d *Dispatcher) arkHangGuardRestart(rec *instanceRecord, reason string) {
+	// Single-flight: only one restart at a time per instance.
+	if !rec.hangGuard.restarting.CompareAndSwap(false, true) {
+		return
+	}
+	defer rec.hangGuard.restarting.Store(false)
+
+	// Lock only for cooldown check and counter reset. We do NOT set
+	// lastRestart here — only after a successful runtime.Restart, so a
+	// failed restart attempt doesn't lock out retries for the full
+	// cooldown (would block the next legitimate hang signal). Resetting
+	// consecutiveFails / windowStart up front is fine: the bounce is
+	// either about to happen (success path will set lastRestart) or has
+	// failed (caller logged + emitted; future signals will re-arm).
+	rec.hangGuard.mu.Lock()
+	if !rec.hangGuard.lastRestart.IsZero() && time.Since(rec.hangGuard.lastRestart) < hangGuardCooldown {
+		remaining := hangGuardCooldown - time.Since(rec.hangGuard.lastRestart)
+		d.log.Debug("hang-guard skipped (cooldown)",
+			"instance_id", rec.InstanceID,
+			"cooldown_remaining", remaining,
+		)
+		rec.hangGuard.mu.Unlock()
+		return
+	}
+	rec.hangGuard.consecutiveFails = 0
+	rec.hangGuard.windowStart = time.Time{}
+	rec.hangGuard.mu.Unlock()
+
+	d.log.Warn("hang-guard restarting container",
+		"instance_id", rec.InstanceID,
+		"container_id", rec.ContainerID,
+		"reason", reason,
+	)
+
+	// Emit a panel log line announcing the restart.
+	d.sendEnv(&panelv1.AgentEnvelope{
+		SentAt: timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+			InstanceId: rec.InstanceID,
+			Stream:     "stderr",
+			At:         timestamppb.Now(),
+			Line:       "[panel] HANG-GUARD: " + reason,
+		}},
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// 2026-05-25: bumped grace 5s → 30s. The hang-guard's bounce can
+	// fire mid-SaveWorld; the 5s window made it a near-guarantee that the
+	// SIGKILL would land inside the SQLite-backed save's flush — same
+	// torn-save pattern that the memory guardrail was producing. 30s lets
+	// most in-flight saves finish before the engine dies; if the engine is
+	// truly wedged it'll get SIGKILL'd after the 30s anyway. See
+	// stats.go's emergencyStopGracePeriod for the matching rationale.
+	if err := d.runtime.Restart(ctx, rec.ContainerID, 30*time.Second); err != nil {
+		d.log.Error("hang-guard restart failed",
+			"instance_id", rec.InstanceID,
+			"err", err,
+		)
+		d.sendEnv(&panelv1.AgentEnvelope{
+			SentAt: timestamppb.Now(),
+			Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+				InstanceId: rec.InstanceID,
+				Stream:     "stderr",
+				At:         timestamppb.Now(),
+				Line:       "[panel] HANG-GUARD restart failed: " + err.Error(),
+			}},
+		})
+		return
+	}
+
+	// Successful restart — start the cooldown clock so we don't bounce
+	// the same instance again for hangGuardCooldown. A failed restart
+	// (above branch returned early) intentionally leaves lastRestart
+	// untouched so the next hang signal can retry.
+	rec.hangGuard.mu.Lock()
+	rec.hangGuard.lastRestart = time.Now()
+	rec.hangGuard.mu.Unlock()
+
+	d.log.Info("hang-guard restart issued",
+		"instance_id", rec.InstanceID,
+	)
+	d.sendEnv(&panelv1.AgentEnvelope{
+		SentAt: timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+			InstanceId: rec.InstanceID,
+			Stream:     "stderr",
+			At:         timestamppb.Now(),
+			Line:       "[panel] HANG-GUARD restart issued",
+		}},
+	})
+}

agent/internal/dispatch/heartbeat_exitwatch_test.go

@@ -0,0 +1,220 @@
+package dispatch
+
+// Tests for WI-05: heartbeats routed through the send queue (sendLoop as the
+// sole owner of stream.Send) and the single-flight, ctx-bound exit watcher.
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// recordingSender captures every envelope the sendLoop delivers.
+type recordingSender struct {
+	mu   sync.Mutex
+	envs []*panelv1.AgentEnvelope
+}
+
+func (r *recordingSender) Send(e *panelv1.AgentEnvelope) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.envs = append(r.envs, e)
+	return nil
+}
+
+func (r *recordingSender) snapshot() []*panelv1.AgentEnvelope {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	out := make([]*panelv1.AgentEnvelope, len(r.envs))
+	copy(out, r.envs)
+	return out
+}
+
+// TestSendHeartbeatGoesThroughQueue verifies SendHeartbeat enqueues onto
+// sendCh and the envelope reaches the sender via sendLoop — i.e. heartbeats
+// no longer need a direct stream.Send from the session goroutine.
+func TestSendHeartbeatGoesThroughQueue(t *testing.T) {
+	d := newTestDispatcher()
+	rs := &recordingSender{}
+	d.SetSender(rs)
+
+	now := time.Now()
+	d.SendHeartbeat(now)
+
+	deadline := time.After(2 * time.Second)
+	for {
+		for _, e := range rs.snapshot() {
+			if hb := e.GetHeartbeat(); hb != nil {
+				if got := hb.At.AsTime(); !got.Equal(now.UTC().Truncate(time.Nanosecond)) && got.Unix() != now.Unix() {
+					t.Fatalf("heartbeat timestamp mismatch: got %v want %v", got, now)
+				}
+				return // delivered through the queue — pass
+			}
+		}
+		select {
+		case <-deadline:
+			t.Fatal("heartbeat never delivered through sendLoop")
+		case <-time.After(10 * time.Millisecond):
+		}
+	}
+}
+
+// TestHeartbeatIsDroppable pins the backpressure classification: a heartbeat
+// must be shed (not block the producer) when the buffer is full.
+func TestHeartbeatIsDroppable(t *testing.T) {
+	env := &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_Heartbeat{Heartbeat: &panelv1.Heartbeat{}}}
+	if !isDroppableEnvelope(env) {
+		t.Fatal("heartbeat envelope must be droppable telemetry")
+	}
+}
+
+// fakeExitRuntime implements just enough of runtime.Runtime for watchExit.
+// The embedded nil interface panics on any unexpected call — that's a test
+// failure signal, not a hazard.
+type fakeExitRuntime struct {
+	runtime.Runtime
+	mu          sync.Mutex
+	activeWaits int
+	maxWaits    int
+	release     chan struct{} // close to make Wait return exit code 0
+}
+
+func (f *fakeExitRuntime) Wait(ctx context.Context, id string) (int, error) {
+	f.mu.Lock()
+	f.activeWaits++
+	if f.activeWaits > f.maxWaits {
+		f.maxWaits = f.activeWaits
+	}
+	f.mu.Unlock()
+	defer func() {
+		f.mu.Lock()
+		f.activeWaits--
+		f.mu.Unlock()
+	}()
+	select {
+	case <-ctx.Done():
+		return -1, ctx.Err()
+	case <-f.release:
+		return 0, nil
+	}
+}
+
+func (f *fakeExitRuntime) InspectByName(ctx context.Context, name string) (runtime.RuntimeInstanceState, error) {
+	return runtime.RuntimeInstanceState{Status: "exited"}, nil
+}
+
+func (f *fakeExitRuntime) counts() (active, max int) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.activeWaits, f.maxWaits
+}
+
+func newExitWatchDispatcher(f *fakeExitRuntime) *Dispatcher {
+	d := &Dispatcher{
+		log:       slog.New(slog.NewTextHandler(io.Discard, nil)),
+		runtime:   f,
+		instances: map[string]*instanceRecord{},
+		sendCh:    make(chan *panelv1.AgentEnvelope, 256),
+	}
+	go d.sendLoop()
+	return d
+}
+
+// waitFor polls cond until true or the deadline elapses.
+func waitFor(t *testing.T, d time.Duration, cond func() bool, msg string) {
+	t.Helper()
+	deadline := time.Now().Add(d)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatal(msg)
+}
+
+// TestStartExitWatchSingleFlight re-arms the watcher many times (the shape of
+// Announce firing on every controller reconnect) and asserts only ONE
+// ContainerWait is ever in flight — the pre-fix behavior leaked one blocked
+// Wait (and its docker connection) per reconnect.
+func TestStartExitWatchSingleFlight(t *testing.T) {
+	f := &fakeExitRuntime{release: make(chan struct{})}
+	d := newExitWatchDispatcher(f)
+	rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
+
+	for i := 0; i < 8; i++ {
+		d.startExitWatch(rec)
+	}
+	// All superseded watchers must exit; exactly one survivor remains.
+	waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 },
+		"expected exactly 1 in-flight ContainerWait after re-arms")
+	// The peak may briefly exceed 1 while a cancelled watcher unwinds, but it
+	// must never approach the number of re-arms (pre-fix: 8 concurrent Waits).
+	if _, max := f.counts(); max >= 8 {
+		t.Fatalf("watchExit not single-flighted: peak concurrent waits = %d", max)
+	}
+
+	// Cancelling the record's watcher (handleStop / Shutdown path) must
+	// release the last Wait.
+	d.mu.Lock()
+	cancel := rec.ExitWatchCancel
+	rec.ExitWatchCancel = nil
+	d.mu.Unlock()
+	cancel()
+	waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 0 },
+		"cancelled exit watcher did not release ContainerWait")
+}
+
+// TestWatchExitCancelledEmitsNothing: a ctx-cancelled watcher must return
+// silently — no spurious stopped/crashed InstanceState for a container that
+// is still running.
+func TestWatchExitCancelledEmitsNothing(t *testing.T) {
+	f := &fakeExitRuntime{release: make(chan struct{})}
+	d := newExitWatchDispatcher(f)
+	rs := &recordingSender{}
+	d.SetSender(rs)
+	rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
+
+	d.startExitWatch(rec)
+	waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 }, "watcher never started")
+	d.mu.Lock()
+	cancel := rec.ExitWatchCancel
+	d.mu.Unlock()
+	cancel()
+	waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 0 }, "watcher never exited")
+	time.Sleep(50 * time.Millisecond) // allow any (wrong) emission to flush
+	for _, e := range rs.snapshot() {
+		if e.GetInstanceState() != nil {
+			t.Fatalf("cancelled watchExit emitted InstanceState: %v", e)
+		}
+	}
+}
+
+// TestWatchExitTerminalExit: when the container really exits (Wait returns
+// 0 and inspect says "exited"), the watcher emits a STOPPED state.
+func TestWatchExitTerminalExit(t *testing.T) {
+	f := &fakeExitRuntime{release: make(chan struct{})}
+	d := newExitWatchDispatcher(f)
+	rs := &recordingSender{}
+	d.SetSender(rs)
+	rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
+	d.instances["x"] = rec
+
+	d.startExitWatch(rec)
+	waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 }, "watcher never started")
+	close(f.release)
+	waitFor(t, 2*time.Second, func() bool {
+		for _, e := range rs.snapshot() {
+			if st := e.GetInstanceState(); st != nil && st.Status == panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED {
+				return true
+			}
+		}
+		return false
+	}, "terminal exit never produced a STOPPED InstanceState")
+}

agent/internal/dispatch/logcoalesce.go

@@ -0,0 +1,159 @@
+package dispatch
+
+import (
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/dbledeez/panel/agent/internal/runtime"
+)
+
+// logCoalescer throttles repetitive progress-style log lines while letting
+// genuine content through unchanged.
+//
+// Why this exists: containers under install/update pressure (SteamCMD
+// download ticks, preallocation, LGSM step banners) can emit 40+ lines per
+// second. Forwarding every one to the controller saturates the gRPC send
+// window, fills the event bus's per-subscriber buffer (which silently drops
+// on overflow), and burns CPU re-rendering DOM. The browser only needs to
+// see the *latest* progress value, not every interstitial frame.
+//
+// Behaviour:
+//   - Lines with progressKey() == "": emit immediately.
+//   - Lines with a key: if it's been less than `interval` since the last
+//     emit of that key, replace the currently-buffered line for that key.
+//     A single background timer flushes any pending lines every `interval`.
+type logCoalescer struct {
+	interval time.Duration
+	emit     func(stream runtime.LogStream, line string, at time.Time)
+
+	mu      sync.Mutex
+	pending map[string]*pendingLine
+	lastAt  map[string]time.Time
+	timer   *time.Timer
+	stopped bool
+}
+
+type pendingLine struct {
+	stream runtime.LogStream
+	line   string
+	at     time.Time
+}
+
+func newLogCoalescer(interval time.Duration, emit func(stream runtime.LogStream, line string, at time.Time)) *logCoalescer {
+	c := &logCoalescer{
+		interval: interval,
+		emit:     emit,
+		pending:  map[string]*pendingLine{},
+		lastAt:   map[string]time.Time{},
+	}
+	return c
+}
+
+// offer delivers a line to the coalescer. Safe to call concurrently, though
+// in practice the caller is the single log-scan goroutine.
+func (c *logCoalescer) offer(stream runtime.LogStream, line string, at time.Time) {
+	key := progressKey(line)
+	if key == "" {
+		// Non-progress — emit immediately, but first flush any buffered
+		// progress lines so they don't appear AFTER a later non-progress line.
+		c.mu.Lock()
+		c.flushLocked()
+		c.mu.Unlock()
+		c.emit(stream, line, at)
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.stopped {
+		return
+	}
+	last := c.lastAt[key]
+	if last.IsZero() || at.Sub(last) >= c.interval {
+		// Enough quiet time — emit right away.
+		c.emit(stream, line, at)
+		c.lastAt[key] = at
+		delete(c.pending, key)
+		return
+	}
+	// Replace the buffered line for this key (we only care about the newest).
+	c.pending[key] = &pendingLine{stream: stream, line: line, at: at}
+	// Make sure the timer is armed. Single timer drains all pending keys.
+	if c.timer == nil {
+		c.timer = time.AfterFunc(c.interval, c.tick)
+	}
+}
+
+func (c *logCoalescer) tick() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.flushLocked()
+	// If more pending arrived during flush (or we expect more), re-arm the timer.
+	if len(c.pending) > 0 && !c.stopped {
+		c.timer = time.AfterFunc(c.interval, c.tick)
+	} else {
+		c.timer = nil
+	}
+}
+
+func (c *logCoalescer) flushLocked() {
+	now := time.Now()
+	for key, p := range c.pending {
+		c.emit(p.stream, p.line, p.at)
+		c.lastAt[key] = now
+		delete(c.pending, key)
+	}
+}
+
+// stop flushes any pending lines and prevents further work. Call from the
+// streamLogs goroutine's defer so a stopping instance doesn't leave buffered
+// progress frames undelivered.
+func (c *logCoalescer) stop() {
+	c.mu.Lock()
+	c.stopped = true
+	if c.timer != nil {
+		c.timer.Stop()
+		c.timer = nil
+	}
+	c.flushLocked()
+	c.mu.Unlock()
+}
+
+// Regexes for lines we treat as part of a repeating progress stream.
+// Each group returns a stable key — same key = same progress counter.
+var (
+	// SteamCMD: "Update state (0x61) downloading, progress: 47.95 (...)"
+	// Also covers "verifying", "preallocating", "committing" stages.
+	reSteamcmdProgress = regexp.MustCompile(`Update state \([^)]+\) (\w+), progress:`)
+	// Generic "Progress: 64%" or "[ 64%] Installing ...".
+	reGenericProgress = regexp.MustCompile(`^\s*(?:Progress:|\[\s*\d+%\s*\])`)
+	// LGSM step banner: "Starting sdtdserver: <step>". vinanrra emits
+	// several per step as the banner animates ([ .... ] → [ INFO ] → [  OK  ]).
+	reLGSMStarting = regexp.MustCompile(`^\s*Starting sdtdserver:`)
+	reLGSMStopping = regexp.MustCompile(`^\s*Stopping sdtdserver:`)
+	// Docker pull: "Downloading [=>] 12.3MB / 100MB"
+	reDockerPull = regexp.MustCompile(`^\s*(?:Downloading|Extracting|Pulling)\s+[\[(]`)
+)
+
+// progressKey returns a stable identifier for a progress-style line, or ""
+// if the line should be forwarded unchanged. The returned key collapses
+// every frame of the same counter onto a single slot so we keep only the
+// latest within each coalesce window.
+func progressKey(text string) string {
+	if m := reSteamcmdProgress.FindStringSubmatch(text); m != nil {
+		return "steamcmd:" + m[1]
+	}
+	if reGenericProgress.MatchString(text) {
+		return "progress"
+	}
+	if reLGSMStarting.MatchString(text) {
+		return "lgsm-starting"
+	}
+	if reLGSMStopping.MatchString(text) {
+		return "lgsm-stopping"
+	}
+	if reDockerPull.MatchString(text) {
+		return "docker-pull"
+	}
+	return ""
+}

agent/internal/dispatch/meta.go

@@ -0,0 +1,112 @@
+package dispatch
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// instanceMeta is the on-disk sidecar written per instance after successful
+// Create. It lets the agent rehydrate its in-memory records after a restart,
+// so operator commands (stop, etc.) continue to work against instances that
+// outlived the agent process.
+//
+// Stored at <metaDir>/<instance_id>.json. Mode 0600 because it carries the
+// RCON password. Deleted on stop.
+type instanceMeta struct {
+	InstanceID     string            `json:"instance_id"`
+	ContainerID    string            `json:"container_id"`
+	ModuleID       string            `json:"module_id"`
+	// Branch is the normalized Steam branch the install came from (warm-seed
+	// matching key). Empty on sidecars written before this feature.
+	Branch         string            `json:"branch,omitempty"`
+	DataPath       string            `json:"data_path"`
+	BrowseableRoot string            `json:"browseable_root,omitempty"`
+	RCONAddr       string            `json:"rcon_addr,omitempty"`
+	RCONPassword   string            `json:"rcon_password,omitempty"`
+	ConfigValues   map[string]string `json:"config_values,omitempty"`
+	CreatedAt      time.Time         `json:"created_at"`
+}
+
+func (d *Dispatcher) metaPath(instanceID string) string {
+	return filepath.Join(d.metaDir, instanceID+".json")
+}
+
+func (d *Dispatcher) writeMeta(rec *instanceRecord) error {
+	if d.metaDir == "" {
+		return nil
+	}
+	if err := os.MkdirAll(d.metaDir, 0o755); err != nil {
+		return fmt.Errorf("mkdir meta dir: %w", err)
+	}
+	m := instanceMeta{
+		InstanceID:     rec.InstanceID,
+		ContainerID:    rec.ContainerID,
+		ModuleID:       rec.ModuleID,
+		Branch:         rec.Branch,
+		DataPath:       rec.DataPath,
+		BrowseableRoot: rec.BrowseableRoot,
+		RCONAddr:       rec.RCONAddr,
+		RCONPassword:   rec.RCONPassword,
+		CreatedAt:      time.Now(),
+	}
+	data, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal meta: %w", err)
+	}
+	if err := os.WriteFile(d.metaPath(rec.InstanceID), data, 0o600); err != nil {
+		return fmt.Errorf("write meta: %w", err)
+	}
+	return nil
+}
+
+func (d *Dispatcher) deleteMeta(instanceID string) {
+	if d.metaDir == "" {
+		return
+	}
+	if err := os.Remove(d.metaPath(instanceID)); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		d.log.Warn("delete meta failed", "instance_id", instanceID, "err", err)
+	}
+}
+
+func (d *Dispatcher) loadAllMeta() ([]instanceMeta, error) {
+	if d.metaDir == "" {
+		return nil, nil
+	}
+	entries, err := os.ReadDir(d.metaDir)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var out []instanceMeta
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		name := e.Name()
+		if filepath.Ext(name) != ".json" {
+			continue
+		}
+		data, err := os.ReadFile(filepath.Join(d.metaDir, name))
+		if err != nil {
+			d.log.Warn("read meta failed", "file", name, "err", err)
+			continue
+		}
+		var m instanceMeta
+		if err := json.Unmarshal(data, &m); err != nil {
+			d.log.Warn("bad meta file", "file", name, "err", err)
+			continue
+		}
+		if m.InstanceID == "" {
+			continue
+		}
+		out = append(out, m)
+	}
+	return out, nil
+}

agent/internal/dispatch/modseed.go

@@ -0,0 +1,195 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	agentmodule "github.com/dbledeez/panel/agent/internal/module"
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	modulepkg "github.com/dbledeez/panel/pkg/module"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// seedModsFromInstance clones the Mods/ folder of srcInstanceID (the cluster
+// master) onto dstInstanceID's install volume, REPLACING the destination's mod
+// set with the master's. Runs a throwaway debian sidecar that mounts both
+// instances' install volumes and tar-copies /src-game/Mods → /dst-game/Mods.
+//
+// This is the inverse of tryWarmSeedFromSibling (which copies the install but
+// EXCLUDES Mods): here we copy ONLY Mods, so a freshly-joined cluster member
+// ends up running exactly the master's mod set without an operator re-uploading
+// every mod by hand.
+//
+// Per-server STATE is preserved/excluded so a clone never leaks one server's
+// player data onto another:
+//   - The master's RefugeBot homes.json (teleport homes, keyed by SteamID) is
+//     EXCLUDED from the copy — it's per-server state, not mod config. This is
+//     the exact file that leaked when an operator rsync'd Mods by hand.
+//   - The destination's OWN homes.json is snapshotted before the wipe and
+//     restored after, so the joining server keeps its own teleports.
+//
+// Both instances must live on THIS agent (same Docker host) — the sidecar
+// mounts local named volumes. If the master isn't found here the seed returns
+// an error and the caller treats it as best-effort (the server still boots with
+// whatever mods it already had).
+func (d *Dispatcher) seedModsFromInstance(ctx context.Context, srcInstanceID, dstInstanceID, dstDataPath string, manifest *modulepkg.Manifest) (int, error) {
+	if manifest == nil {
+		return 0, fmt.Errorf("mod-seed: nil manifest")
+	}
+	if srcInstanceID == dstInstanceID {
+		return 0, fmt.Errorf("mod-seed: refusing to seed instance %q from itself", dstInstanceID)
+	}
+	installPath := ""
+	if len(manifest.UpdateProviders) > 0 {
+		installPath = manifest.UpdateProviders[0].InstallPath
+	}
+	if installPath == "" {
+		installPath = manifest.Runtime.Docker.BrowseableRoot
+	}
+	if installPath == "" {
+		return 0, fmt.Errorf("mod-seed: no install path for module %q", manifest.ID)
+	}
+
+	// Resolve the master's data path from our local instance table.
+	d.mu.Lock()
+	srcRec, ok := d.instances[srcInstanceID]
+	d.mu.Unlock()
+	if !ok {
+		return 0, fmt.Errorf("mod-seed: master instance %q is not on this agent", srcInstanceID)
+	}
+	if srcRec.ModuleID != manifest.ID {
+		return 0, fmt.Errorf("mod-seed: master %q is module %q, not %q", srcInstanceID, srcRec.ModuleID, manifest.ID)
+	}
+
+	srcVolumes := agentmodule.ResolveVolumes(manifest, srcInstanceID, srcRec.DataPath)
+	dstVolumes := agentmodule.ResolveVolumes(manifest, dstInstanceID, dstDataPath)
+	srcGameVol := volumeNameForPath(srcVolumes, installPath)
+	dstGameVol := volumeNameForPath(dstVolumes, installPath)
+	if srcGameVol == "" || dstGameVol == "" {
+		return 0, fmt.Errorf("mod-seed: install_path %q not a named volume on both instances", installPath)
+	}
+
+	// tar, not rsync — debian:12-slim ships no rsync. Replace /dst-game/Mods
+	// from /src-game/Mods, excluding the master's per-server homes.json, and
+	// preserving the destination's own homes.json across the wipe.
+	const seedScript = `set -e
+echo "mod-seed: cloning master Mods set (excl per-server homes.json)"
+HOMES=/dst-game/Mods/zzzzzzzRefugeBot/homes.json
+if [ -f "$HOMES" ]; then cp -a "$HOMES" /tmp/dst-homes.json; echo "mod-seed: preserved destination homes.json"; fi
+mkdir -p /dst-game/Mods
+find /dst-game/Mods -mindepth 1 -delete
+( cd /src-game/Mods && tar --exclude='./zzzzzzzRefugeBot/homes.json' -cf - . ) | ( cd /dst-game/Mods && tar -xf - )
+if [ -f /tmp/dst-homes.json ]; then mkdir -p /dst-game/Mods/zzzzzzzRefugeBot; cp -a /tmp/dst-homes.json "$HOMES"; echo "mod-seed: restored destination homes.json"; fi
+echo "mod-seed: $(ls /dst-game/Mods 2>/dev/null | wc -l) mods now present"
+echo SEED_OK
+`
+
+	sidecarID := fmt.Sprintf("%s-modseed", dstInstanceID)
+	spec := runtime.InstanceSpec{
+		InstanceID:    sidecarID,
+		IsSidecar:     true,
+		Image:         "debian:12-slim",
+		Command:       []string{"bash", "-c", seedScript},
+		RestartPolicy: "no",
+		Volumes: []runtime.VolumeSpec{
+			{Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true},
+			{Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"},
+		},
+	}
+
+	d.log.Info("mod-seed: starting sidecar", "src", srcInstanceID, "dst", dstInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol)
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		return 0, fmt.Errorf("mod-seed: create sidecar: %w", err)
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		return 0, fmt.Errorf("mod-seed: start sidecar: %w", err)
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	defer cancelLogs()
+	logDone := make(chan struct{})
+	var modCount int
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.log.Info("mod-seed log", "instance_id", dstInstanceID, "line", line)
+			// Parse the sidecar's "mod-seed: N mods now present" echo so the
+			// caller (and the cluster-sync UI) can report how many mods landed.
+			if strings.Contains(line, "mods now present") {
+				fields := strings.Fields(line)
+				for i, f := range fields {
+					if f == "mods" && i > 0 {
+						if n, err := strconv.Atoi(fields[i-1]); err == nil {
+							modCount = n
+						}
+					}
+				}
+			}
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		return 0, fmt.Errorf("mod-seed: wait sidecar: %w", err)
+	}
+	if exitCode != 0 {
+		return 0, fmt.Errorf("mod-seed: sidecar exited %d", exitCode)
+	}
+	d.log.Info("mod-seed: success", "src", srcInstanceID, "dst", dstInstanceID, "mods", modCount)
+	return modCount, nil
+}
+
+// handleSeedMods runs the mod-seed sidecar on demand — no container recreate,
+// no restart. Clones src_instance_id's Mods/ onto instance_id (excluding the
+// master's per-server homes.json, preserving the member's own). The member
+// keeps running on its currently-loaded mods; the freshly-copied files take
+// effect on its NEXT restart. Backs the cluster "Sync mods to members" action,
+// where the master pushes its current mod set to every other member at once.
+func (d *Dispatcher) handleSeedMods(ctx context.Context, corrID string, req *panelv1.SeedModsRequest) {
+	log := d.log.With("instance_id", req.InstanceId, "src", req.SrcInstanceId, "correlation_id", corrID)
+	reply := func(ok bool, modCount int, errMsg string) {
+		d.sendEnv(&panelv1.AgentEnvelope{
+			CorrelationId: corrID,
+			SentAt:        timestamppb.Now(),
+			Payload: &panelv1.AgentEnvelope_SeedModsResult{
+				SeedModsResult: &panelv1.SeedModsResult{
+					InstanceId: req.InstanceId,
+					Ok:         ok,
+					ModCount:   int32(modCount),
+					Error:      errMsg,
+				},
+			},
+		})
+		if !ok {
+			log.Warn("mod-seed RPC failed", "err", errMsg)
+		}
+	}
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		reply(false, 0, err.Error())
+		return
+	}
+	manifest, ok := d.modules.Get(rec.ModuleID)
+	if !ok {
+		reply(false, 0, "module "+rec.ModuleID+" not in registry")
+		return
+	}
+	modCount, err := d.seedModsFromInstance(ctx, req.SrcInstanceId, req.InstanceId, rec.DataPath, manifest)
+	if err != nil {
+		reply(false, 0, err.Error())
+		return
+	}
+	log.Info("mod-seed RPC complete", "mods", modCount)
+	reply(true, modCount, "")
+}

agent/internal/dispatch/region.go

@@ -0,0 +1,264 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/dbledeez/panel/pkg/regionmedic"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+// Region Medic agent handlers — scan a 7DTD instance's active-world Region
+// directory for .7rg corruption, and heal one region from the newest clean
+// panel backup. Volume files are reached through the runtime's container-path
+// ops (the agent runs unprivileged, so it cannot touch /var/lib/docker/volumes
+// directly). The controller orchestrates stop→heal→start; handleRegionHeal
+// itself only performs the file swap and refuses to run on a live container.
+
+// activeRegionDir resolves the container-absolute Region directory of the
+// instance's active world by reading GameWorld/GameName from the live
+// serverconfig.xml. Works on a stopped container (CopyFileFromContainer does).
+func (d *Dispatcher) activeRegionDir(ctx context.Context, container string) (string, error) {
+	raw, err := d.runtime.CopyFileFromContainer(ctx, container, "/game-saves/serverconfig.xml")
+	if err != nil {
+		return "", fmt.Errorf("read serverconfig.xml: %w", err)
+	}
+	world := xmlProp(string(raw), "GameWorld")
+	game := xmlProp(string(raw), "GameName")
+	if world == "" || game == "" {
+		return "", fmt.Errorf("serverconfig.xml missing GameWorld/GameName (got %q/%q)", world, game)
+	}
+	// HOME is pinned to /game-saves by the 7dtd entrypoint.
+	return path.Join("/game-saves/.local/share/7DaysToDie/Saves", world, game, "Region"), nil
+}
+
+// xmlProp pulls the value from a 7DTD serverconfig line:
+//
+//	<property name="GameWorld" value="Navezgane" />
+//
+// Tolerant of attribute order and single/double quotes; skips commented lines.
+func xmlProp(content, name string) string {
+	for _, raw := range strings.Split(content, "\n") {
+		line := strings.TrimSpace(raw)
+		if line == "" || strings.HasPrefix(line, "<!--") {
+			continue
+		}
+		if !strings.Contains(line, `name="`+name+`"`) && !strings.Contains(line, `name='`+name+`'`) {
+			continue
+		}
+		i := strings.Index(line, "value=")
+		if i < 0 {
+			continue
+		}
+		rest := line[i+len("value="):]
+		if rest == "" {
+			continue
+		}
+		q := rest[0]
+		rest = rest[1:]
+		if end := strings.IndexByte(rest, q); end >= 0 {
+			return strings.TrimSpace(rest[:end])
+		}
+	}
+	return ""
+}
+
+// listRegionDirNames returns the file names in the container's Region dir.
+// Uses ExecCapture when the container is running; nil + error otherwise (the
+// scan path runs while the instance is up).
+func (d *Dispatcher) listRegionDirNames(ctx context.Context, container, regionDir string) ([]string, error) {
+	stdout, _, _, err := d.runtime.ExecCapture(ctx, container, []string{"ls", "-1", regionDir})
+	if err != nil {
+		return nil, err
+	}
+	var names []string
+	for _, n := range strings.Split(string(stdout), "\n") {
+		if n = strings.TrimSpace(n); n != "" {
+			names = append(names, n)
+		}
+	}
+	return names, nil
+}
+
+// handleRegionScan reports corruption evidence for an instance's active world:
+// error_backup salvage files clustered into regions, plus structural validation
+// of each affected region's .7rg file.
+func (d *Dispatcher) handleRegionScan(ctx context.Context, corrID string, req *panelv1.RegionScanRequest) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "not_found", Message: err.Error()}})
+		return
+	}
+	container := "panel-" + rec.InstanceID
+	regionDir, err := d.activeRegionDir(ctx, container)
+	if err != nil {
+		d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "resolve_region_dir", Message: err.Error()}})
+		return
+	}
+	names, err := d.listRegionDirNames(ctx, container, regionDir)
+	if err != nil {
+		d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{RegionDir: regionDir, Error: &panelv1.Error{Code: "list_region_dir", Message: err.Error()}})
+		return
+	}
+
+	// Cluster error_backups by region; note which region files exist.
+	type agg struct {
+		files   int
+		present bool
+	}
+	clusters := map[regionmedic.RegionCoord]*agg{}
+	fileSet := map[regionmedic.RegionCoord]bool{}
+	total := 0
+	for _, n := range names {
+		if cx, cz, ok := regionmedic.ParseErrorBackup(n); ok {
+			total++
+			rc := regionmedic.RegionForChunk(cx, cz)
+			a := clusters[rc]
+			if a == nil {
+				a = &agg{}
+				clusters[rc] = a
+			}
+			a.files++
+			continue
+		}
+		if rc, ok := regionmedic.ParseRegionFileName(n); ok {
+			fileSet[rc] = true
+		}
+	}
+
+	var affected []*panelv1.AffectedRegionMsg
+	for rc, a := range clusters {
+		msg := &panelv1.AffectedRegionMsg{
+			Region:       rc.String(),
+			ErrorBackups: int32(a.files),
+			FilePresent:  fileSet[rc],
+		}
+		// Validate the affected region file (read via container-path op).
+		if fileSet[rc] {
+			if b, rerr := d.runtime.CopyFileFromContainer(ctx, container, path.Join(regionDir, rc.FileName())); rerr == nil {
+				rep := regionmedic.ValidateRegionBytes(b, false)
+				msg.FileCorrupt = !rep.Healthy()
+				msg.BadChunks = int32(len(rep.BadChunks))
+			}
+		}
+		affected = append(affected, msg)
+	}
+	sort.Slice(affected, func(i, j int) bool { return affected[i].ErrorBackups > affected[j].ErrorBackups })
+
+	d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{
+		ErrorBackups: int32(total),
+		Affected:     affected,
+		RegionDir:    regionDir,
+	})
+}
+
+// handleRegionHeal swaps one region file for the newest clean backup copy.
+// It REQUIRES the instance to be stopped (the controller stops it first) so the
+// running engine can't overwrite the swap. It snapshots the corrupt original
+// next to the region file before overwriting. Orchestration (stop/start) is the
+// controller's job.
+func (d *Dispatcher) handleRegionHeal(ctx context.Context, corrID string, req *panelv1.RegionHealRequest) {
+	fail := func(code, msg string) {
+		d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{Region: req.Region, Error: &panelv1.Error{Code: code, Message: msg}})
+	}
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		fail("not_found", err.Error())
+		return
+	}
+	region, ok := regionmedic.ParseRegionFileName(regionFileArg(req.Region))
+	if !ok {
+		fail("bad_region", fmt.Sprintf("invalid region %q", req.Region))
+		return
+	}
+	container := "panel-" + rec.InstanceID
+
+	// Refuse to swap a region on a LIVE container (it would be overwritten on
+	// the next save). ExecCapture succeeds only when the container is running.
+	if _, _, _, exErr := d.runtime.ExecCapture(ctx, container, []string{"true"}); exErr == nil {
+		fail("instance_running", "instance must be stopped before healing a region")
+		return
+	}
+
+	regionDir, err := d.activeRegionDir(ctx, container)
+	if err != nil {
+		fail("resolve_region_dir", err.Error())
+		return
+	}
+	liveFile := path.Join(regionDir, region.FileName())
+	liveBytes, err := d.runtime.CopyFileFromContainer(ctx, container, liveFile)
+	if err != nil {
+		fail("read_live_region", err.Error())
+		return
+	}
+
+	// Find the newest clean backup copy of this region (backups are local to
+	// the agent at <backupDir>/<instance>/).
+	backups, err := regionmedic.ListPanelBackups(path.Join(d.backupDir, rec.InstanceID))
+	if err != nil {
+		fail("list_backups", err.Error())
+		return
+	}
+	entry := regionmedic.SaveRelEntry(regionDir, region.FileName())
+	best, _, err := regionmedic.FindLastGoodRegion(backups, entry, true, time.Time{})
+	if err != nil {
+		fail("find_backup", err.Error())
+		return
+	}
+	if !best.Found || !best.Clean() {
+		fail("no_clean_backup", "no backup contains a clean copy of this region")
+		return
+	}
+
+	// Snapshot the corrupt original alongside the region file (no new subdir, so
+	// CopyFileToContainer needs no parent-dir creation on a stopped container).
+	stamp := time.Now().UTC().Format("20060102-150405")
+	snapName := region.FileName() + ".medic-corrupt-" + stamp
+	snapPath := path.Join(regionDir, snapName)
+	if err := d.runtime.CopyFileToContainer(ctx, container, snapPath, liveBytes); err != nil {
+		fail("snapshot", err.Error())
+		return
+	}
+	// Swap in the clean copy (overwrites the live region file in place).
+	if err := d.runtime.CopyFileToContainer(ctx, container, liveFile, best.Bytes); err != nil {
+		fail("write_region", err.Error())
+		return
+	}
+
+	d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{
+		Region:       region.String(),
+		SourceBackup: best.Backup.ID,
+		BytesWritten: int64(len(best.Bytes)),
+		SnapshotPath: snapPath,
+		Healed:       true,
+	})
+}
+
+// regionFileArg normalizes "r.X.Z" or "r.X.Z.7rg" to a .7rg filename.
+func regionFileArg(s string) string {
+	if strings.HasSuffix(s, ".7rg") {
+		return s
+	}
+	return s + ".7rg"
+}
+
+func (d *Dispatcher) sendRegionScanResult(corrID string, res *panelv1.RegionScanResult) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload:       &panelv1.AgentEnvelope_RegionScanResult{RegionScanResult: res},
+	})
+}
+
+func (d *Dispatcher) sendRegionHealResult(corrID string, res *panelv1.RegionHealResult) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload:       &panelv1.AgentEnvelope_RegionHealResult{RegionHealResult: res},
+	})
+}

agent/internal/dispatch/regionmedic_auto.go

@@ -0,0 +1,144 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"github.com/dbledeez/panel/agent/internal/runtime"
+)
+
+func medicEnvOr(k, def string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return def
+}
+
+// Medic host paths on the agent box (figaro, for 7DTD). Overridable via env so
+// the same agent binary works if 7DTD ever moves hosts.
+func regionRollingRoot() string {
+	return medicEnvOr("PANEL_REGION_ROLLING_DIR", "/home/refuge/region-rolling")
+}
+func regionMedicBin() string {
+	return medicEnvOr("PANEL_REGION_MEDIC_BIN", "/home/refuge/panel/bin/region-medic")
+}
+func discordTokenFile() string {
+	return medicEnvOr("PANEL_DISCORD_TOKEN_FILE", "/home/refuge/panel/discord-bot-token")
+}
+
+func medicFileExists(p string) bool {
+	st, err := os.Stat(p)
+	return err == nil && !st.IsDir()
+}
+
+// runRegionMedicOnStart runs the autoheal CLI in a throwaway debian sidecar while
+// the 7DTD container is STILL STOPPED, just before the agent starts it:
+//   - validate every .7rg in the active world's Region dir,
+//   - heal corrupt ones from the newest clean rolling snapshot, then the panel
+//     tar backups as a fallback,
+//   - snapshot the now-clean Region dir to a fresh rolling slot,
+//   - post a Discord summary (naming the server) if anything healed.
+//
+// The per-server medic config (enable / rolling-keep / Discord channel) lives in
+// the SAVES volume at region-medic.json — written by the panel's medic-config
+// endpoint via FsWrite (NO container recreate) and read by the CLI here in the
+// sidecar. The agent therefore always launches the sidecar for 7DTD and lets the
+// CLI decide; an absent/empty config just means defaults + no Discord.
+//
+// Best-effort by contract: any failure is logged + swallowed, never blocking the
+// start. 10-min hard cap so a wedged sidecar/daemon can't stall a boot.
+func (d *Dispatcher) runRegionMedicOnStart(ctx context.Context, rec *instanceRecord) {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+	log := d.log.With("instance_id", rec.InstanceID, "medic", true)
+	medicBin := regionMedicBin()
+	if !medicFileExists(medicBin) {
+		log.Warn("region-medic: binary missing, skipping pre-start heal", "bin", medicBin)
+		return
+	}
+	savesVol := fmt.Sprintf("panel-%s-saves", rec.InstanceID)
+	rollingDir := path.Join(regionRollingRoot(), rec.InstanceID)
+	_ = os.MkdirAll(rollingDir, 0o755)
+	backupsDir := path.Join(d.backupDir, rec.InstanceID)
+
+	cmd := []string{"/rm", "autoheal",
+		"-saves-root", "/saves",
+		"-rolling-dir", "/rolling",
+		"-keep", "2", // default; overridden by region-medic.json in the saves volume
+		"-label", rec.InstanceID,
+		"-apply",
+	}
+	vols := []runtime.VolumeSpec{
+		{Type: "volume", VolumeName: savesVol, ContainerPath: "/saves"},
+		{Type: "bind", HostPath: rollingDir, ContainerPath: "/rolling"},
+		{Type: "bind", HostPath: medicBin, ContainerPath: "/rm", ReadOnly: true},
+	}
+	// debian:12-slim ships no CA bundle, so the CLI's HTTPS POST to Discord can't
+	// verify the TLS cert ("x509: certificate signed by unknown authority"). Mount
+	// the host's CA certs read-only — Go's x509 finds ca-certificates.crt there.
+	if _, err := os.Stat("/etc/ssl/certs"); err == nil {
+		vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: "/etc/ssl/certs", ContainerPath: "/etc/ssl/certs", ReadOnly: true})
+	}
+	// Panel tar backups are the heal fallback when no rolling slot has a clean
+	// copy (e.g. very first boot, before any snapshot). Only mount if present.
+	if st, err := os.Stat(backupsDir); err == nil && st.IsDir() {
+		cmd = append(cmd, "-backups-dir", "/backups")
+		vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: backupsDir, ContainerPath: "/backups", ReadOnly: true})
+	}
+	// The token is mounted whenever it exists; the CLI only posts if the saves
+	// config names a channel. No -discord-channel flag — the CLI reads it from
+	// /saves/region-medic.json.
+	if tf := discordTokenFile(); medicFileExists(tf) {
+		cmd = append(cmd, "-discord-token-file", "/discord-token")
+		vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: tf, ContainerPath: "/discord-token", ReadOnly: true})
+	}
+
+	spec := runtime.InstanceSpec{
+		InstanceID:    rec.InstanceID + "-medic",
+		IsSidecar:     true,
+		Image:         "debian:12-slim",
+		Command:       cmd,
+		RestartPolicy: "no",
+		Volumes:       vols,
+	}
+
+	log.Info("region-medic: pre-start scan+heal")
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		log.Warn("region-medic: create sidecar failed", "err", err)
+		return
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		log.Warn("region-medic: start sidecar failed", "err", err)
+		return
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	defer cancelLogs()
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			log.Info("region-medic", "line", line)
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		log.Warn("region-medic: wait sidecar failed", "err", err)
+		return
+	}
+	if exitCode != 0 {
+		log.Warn("region-medic: sidecar exited non-zero", "code", exitCode)
+		return
+	}
+	log.Info("region-medic: pre-start heal complete")
+}

agent/internal/dispatch/renderconfig.go

@@ -0,0 +1,125 @@
+package dispatch
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	agentmodule "github.com/dbledeez/panel/agent/internal/module"
+	modulepkg "github.com/dbledeez/panel/pkg/module"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// handleRenderConfig re-renders the module's declared config_files into the
+// instance's host DataPath — the same modulepkg.Render call handleCreate
+// makes, minus the container recreate. For modules that bind-mount the
+// rendered output into the container (7dtd's serverconfig.xml.rendered),
+// os.WriteFile rewrites the host file in place (same inode), so the running
+// container sees the new content immediately and the entrypoint applies it
+// on the NEXT boot. The container itself is never restarted.
+//
+// Containers created BEFORE a config_file's bind-mount was added to the
+// module manifest don't have the bind at all (docker pins mounts at create)
+// — their /game-saves/<file> is a plain file inside the saves volume. For
+// those we self-verify: read the file back through the container, and if it
+// doesn't match the fresh render, copy the rendered bytes in directly
+// (works running or stopped; the game only reads config at boot).
+func (d *Dispatcher) handleRenderConfig(ctx context.Context, corrID string, req *panelv1.InstanceRenderConfigRequest) {
+	log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
+	fail := func(msg string) {
+		d.sendEnv(&panelv1.AgentEnvelope{
+			CorrelationId: corrID,
+			SentAt:        timestamppb.Now(),
+			Payload: &panelv1.AgentEnvelope_InstanceRenderConfigResult{
+				InstanceRenderConfigResult: &panelv1.InstanceRenderConfigResult{Error: msg},
+			},
+		})
+		log.Warn("render config failed", "err", msg)
+	}
+
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		fail(err.Error())
+		return
+	}
+	if rec.DataPath == "" {
+		fail("instance has no data path (pre-feature sidecar?) — recreate once to adopt")
+		return
+	}
+	manifest, ok := d.modules.Get(rec.ModuleID)
+	if !ok {
+		fail("module " + rec.ModuleID + " not in registry")
+		return
+	}
+
+	// Same merge handleCreate uses: generated secrets first, user values on
+	// top (empties dropped → template `or` defaults kick in).
+	values, err := mergeValuesAndSecrets(manifest, req.ConfigValues)
+	if err != nil {
+		fail("secrets: " + err.Error())
+		return
+	}
+	// Branch-aware: a 3.0 instance renders serverconfig.v3.xml.tmpl, ≤2.6
+	// keeps the default. Resolve from the instance's persisted branch (set at
+	// create), falling back to the merged config_values' provider_id so a
+	// durable config-render still picks the right template if the record
+	// predates branch persistence.
+	branch := rec.Branch
+	if branch == "" {
+		branch = resolveCreateBranch(manifest, values)
+	}
+	if err := modulepkg.RenderForBranch(manifest, rec.DataPath, values, branch); err != nil {
+		fail("render: " + err.Error())
+		return
+	}
+
+	// Self-verify through the container; fall back to a direct copy into
+	// the volume for pre-bind containers. The mapping host-path → container
+	// path comes from the manifest's volume specs.
+	volumes := agentmodule.ResolveVolumes(manifest, rec.InstanceID, rec.DataPath)
+	container := "panel-" + rec.InstanceID
+	var rendered []string
+	for _, cf := range manifest.ConfigFiles {
+		if cf.TemplateForBranch(branch) == "" {
+			continue
+		}
+		rendered = append(rendered, cf.Path)
+		hostPath := filepath.Join(rec.DataPath, cf.Path)
+		ctrPath := ""
+		for _, v := range volumes {
+			if v.Type == "bind" && v.HostPath == hostPath {
+				ctrPath = v.ContainerPath
+				break
+			}
+		}
+		if ctrPath == "" {
+			continue // file isn't mounted into the container; host render is all there is
+		}
+		want, rerr := os.ReadFile(hostPath)
+		if rerr != nil {
+			fail("read rendered " + cf.Path + ": " + rerr.Error())
+			return
+		}
+		got, gerr := d.runtime.CopyFileFromContainer(ctx, container, ctrPath)
+		if gerr == nil && bytes.Equal(got, want) {
+			continue // live bind-mount delivered the new render
+		}
+		// Pre-bind container (or unreadable): write the volume copy directly.
+		if cerr := d.runtime.CopyFileToContainer(ctx, container, ctrPath, want); cerr != nil {
+			fail("container has no live bind for " + cf.Path + " and direct copy failed: " + cerr.Error())
+			return
+		}
+		log.Info("rendered config copied into pre-bind container volume", "file", cf.Path, "container_path", ctrPath)
+	}
+	log.Info("re-rendered config files (durable, applies next boot)", "files", rendered)
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_InstanceRenderConfigResult{
+			InstanceRenderConfigResult: &panelv1.InstanceRenderConfigResult{RenderedFiles: rendered},
+		},
+	})
+}

agent/internal/dispatch/sendbuffer_test.go

@@ -0,0 +1,131 @@
+package dispatch
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// blockingSender simulates a controller that has stopped draining the stream:
+// every Send blocks until released. This is the backpressure condition that
+// used to wedge the whole agent when sendEnv called Send inline.
+type blockingSender struct {
+	release chan struct{}
+	mu      sync.Mutex
+	sent    int
+}
+
+func (b *blockingSender) Send(*panelv1.AgentEnvelope) error {
+	<-b.release // block until the test lets it through
+	b.mu.Lock()
+	b.sent++
+	b.mu.Unlock()
+	return nil
+}
+
+func logEnv() *panelv1.AgentEnvelope {
+	return &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{Line: "x"}}}
+}
+
+func stateEnv() *panelv1.AgentEnvelope {
+	return &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_InstanceState{InstanceState: &panelv1.InstanceStateUpdate{InstanceId: "i"}}}
+}
+
+// newTestDispatcher builds a Dispatcher with just the send machinery wired —
+// enough to exercise sendEnv / sendLoop without a real runtime.
+func newTestDispatcher() *Dispatcher {
+	d := &Dispatcher{sendCh: make(chan *panelv1.AgentEnvelope, 4096)}
+	go d.sendLoop()
+	return d
+}
+
+// TestSendEnvDropsTelemetryUnderBackpressure is the core regression guard for
+// the VEIN-exposed wedge: with the sender blocked, a flood of log envelopes far
+// exceeding the buffer must NOT block the producer. Pre-fix, sendEnv called
+// Send inline and the producer (and the whole agent) froze here.
+func TestSendEnvDropsTelemetryUnderBackpressure(t *testing.T) {
+	d := newTestDispatcher()
+	bs := &blockingSender{release: make(chan struct{})}
+	d.SetSender(bs)
+
+	done := make(chan struct{})
+	go func() {
+		// 50k log frames — 12x the 4096 buffer. Must all return promptly.
+		for i := 0; i < 50000; i++ {
+			d.sendEnv(logEnv())
+		}
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		// Producer completed without blocking on the stalled sender. Pass.
+	case <-time.After(5 * time.Second):
+		t.Fatal("sendEnv blocked under backpressure — the wedge regressed")
+	}
+}
+
+// TestCriticalEnvelopeNotDropped verifies an instance-state envelope is queued
+// (not silently dropped) even when telemetry is being shed. We fill the buffer
+// with the sender blocked, then confirm a critical send blocks until drained
+// (i.e. it's preserved, not discarded).
+func TestCriticalEnvelopeNotDropped(t *testing.T) {
+	d := &Dispatcher{sendCh: make(chan *panelv1.AgentEnvelope, 2)}
+	// No drainer started yet — fill the buffer.
+	d.sendEnv(logEnv())
+	d.sendEnv(logEnv())
+	// Buffer is full (cap 2). A telemetry frame must drop (non-blocking).
+	dropDone := make(chan struct{})
+	go func() { d.sendEnv(logEnv()); close(dropDone) }()
+	select {
+	case <-dropDone:
+	case <-time.After(time.Second):
+		t.Fatal("telemetry send blocked on full buffer — should have dropped")
+	}
+	// A critical frame must NOT drop: it blocks until space frees. Start a
+	// drainer to free space and confirm the critical send then completes.
+	critDone := make(chan struct{})
+	go func() { d.sendEnv(stateEnv()); close(critDone) }()
+	select {
+	case <-critDone:
+		t.Fatal("critical send returned before buffer had space — it was dropped")
+	case <-time.After(100 * time.Millisecond):
+		// Correctly still blocking. Now drain and it should complete.
+	}
+	go d.sendLoop()
+	select {
+	case <-critDone:
+		// Drained and delivered. Pass.
+	case <-time.After(time.Second):
+		t.Fatal("critical send never completed after drain")
+	}
+}
+
+// TestConcurrentProducersNoRace runs many producers through sendEnv at once.
+// Pre-fix, each called stream.Send directly — concurrent Sends on one gRPC
+// stream are a data race. The single-drainer design serializes them. Run with
+// -race to catch a regression.
+func TestConcurrentProducersNoRace(t *testing.T) {
+	d := newTestDispatcher()
+	bs := &blockingSender{release: make(chan struct{})}
+	close(bs.release) // never block — just count
+	d.SetSender(bs)
+
+	var wg sync.WaitGroup
+	for p := 0; p < 32; p++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 1000; i++ {
+				d.sendEnv(logEnv())
+			}
+		}()
+	}
+	wg.Wait()
+	// Give the drainer a moment to flush.
+	time.Sleep(200 * time.Millisecond)
+	// No assertion on exact count (some may still be in flight / dropped);
+	// the point is the -race detector finds no concurrent Send.
+}

agent/internal/dispatch/stats.go

@@ -0,0 +1,398 @@
+package dispatch
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// Per-instance stats polling. Docker streams a JSON frame ~once per
+// second; we throttle to one upstream emit per statsEmitInterval so the
+// event bus isn't flooded. CPU% uses the frame's cur−previous delta
+// directly (Docker includes precpu_stats on streamed frames).
+
+// 3s is snappy enough to watch RAM climb during ARK world-load (12-15 GB
+// in under a minute) without flooding the SSE bus. Older 10s value made
+// the dashboard feel laggy when memory was actually moving fast.
+const statsEmitInterval = 3 * time.Second
+
+// ARK SA memory guardrails — two-stage so we can flush saves cleanly
+// before having to SIGKILL.
+//
+// Healthy ARK SA world: 10-15 GB resident. 16-30 GB during save-flush
+// peaks is normal on a populated cluster. 45 GB is the warning level;
+// 60 GB is the hard kill that protects the host from OOM (which would
+// take down all sibling ARKs + postgres + the panel itself).
+//
+// 2026-05-25 history: the previous single-threshold 40 GB → 5s SIGKILL
+// path corrupted at least three SQLite-backed `.ark` saves over the
+// 2026-05-19 → 2026-05-25 window because the SIGKILL hit while a
+// SaveWorld was mid-flush (FAtlasSaveManager::LoadOperationSql crash on
+// next boot — null page read in the half-written SQLite store).
+//
+// The two-stage guard:
+//
+//   At arkRAMWarnLimit (45 GB) — fired AT MOST ONCE per container life:
+//     1. Broadcast a 60s heads-up to in-game players via RCON
+//     2. SaveWorld (lets ARK flush cleanly)
+//     3. Wait 60s for the save + warning window
+//     4. runtime.Stop with 120s grace (clean shutdown path)
+//   The container then stays Exited; operator restarts when ready, no
+//   torn save, no progress loss beyond what the SaveWorld captured.
+//
+//   At arkRAMHardLimit (60 GB) — fires if the warn path didn't help:
+//     SIGKILL via runtime.Stop with 30s grace. Save MAY be torn here;
+//     this is the host-protection layer, accepted as last-resort.
+//
+// The warn level fires at most once per container life via the
+// rec.memGuardWarned atomic — once we've committed to a graceful stop,
+// the hard limit isn't allowed to also fire and interrupt the SaveWorld.
+const arkRAMWarnLimit uint64 = 45 * 1024 * 1024 * 1024
+const arkRAMHardLimit uint64 = 60 * 1024 * 1024 * 1024
+
+// gracefulStopGracePeriod is the docker-stop grace given to the
+// graceful warn path. ARK's SaveWorld can take 30-60s on a populated
+// world; 120s is enough headroom for the flush + the watchdog's DoExit.
+const gracefulStopGracePeriod = 120 * time.Second
+
+// emergencyStopGracePeriod is the docker-stop grace for the 60 GB hard
+// path. 30s (was 5s) gives the engine a real shot at flushing in-flight
+// state before SIGKILL — the host-OOM risk that motivated the original
+// 5s grace doesn't materialize this fast in practice.
+const emergencyStopGracePeriod = 30 * time.Second
+
+// dockerStatsFrame is the minimum subset of Docker's /containers/<id>/stats
+// response we need. Kept as a local shape so we don't depend on which
+// version of github.com/docker/docker renamed the struct this week.
+type dockerStatsFrame struct {
+	CPUStats struct {
+		// IMPORTANT: every nested field needs an explicit json tag.
+		// Go's unmarshal is case-insensitive but does NOT translate
+		// CamelCase ↔ snake_case. Without `json:"total_usage"`, the
+		// field reads as 0 because Docker emits the snake_case key
+		// and we'd be looking for "TotalUsage" / "totalusage". This
+		// bug shipped for months — every container reported 0% CPU.
+		CPUUsage struct {
+			TotalUsage uint64 `json:"total_usage"`
+		} `json:"cpu_usage"`
+		SystemCPUUsage uint64 `json:"system_cpu_usage"`
+		OnlineCPUs     uint32 `json:"online_cpus"`
+	} `json:"cpu_stats"`
+	PreCPUStats struct {
+		CPUUsage struct {
+			TotalUsage uint64 `json:"total_usage"`
+		} `json:"cpu_usage"`
+		SystemCPUUsage uint64 `json:"system_cpu_usage"`
+	} `json:"precpu_stats"`
+	MemoryStats struct {
+		Usage uint64 `json:"usage"`
+		Limit uint64 `json:"limit"`
+		// Docker's `usage` includes file-system page cache, which can
+		// balloon to multi-GB on read-heavy workloads (ARK SA's mod +
+		// asset reads inflate it to 25 GB on a 12 GB RSS workload).
+		// `docker stats` CLI subtracts cache/inactive_file to match
+		// what operators expect. We mirror that.
+		//   - cgroups v1: usage - stats.cache
+		//   - cgroups v2: usage - stats.inactive_file
+		// Both keys are in the same nested map; whichever is present
+		// is the one our cgroup version exposes.
+		Stats struct {
+			Cache        uint64 `json:"cache"`         // cgroups v1
+			InactiveFile uint64 `json:"inactive_file"` // cgroups v2
+			RSS          uint64 `json:"rss"`           // cgroups v1
+			Anon         uint64 `json:"anon"`          // cgroups v2 (active anon)
+		} `json:"stats"`
+	} `json:"memory_stats"`
+	Networks map[string]struct {
+		RxBytes uint64 `json:"rx_bytes"`
+		TxBytes uint64 `json:"tx_bytes"`
+	} `json:"networks"`
+	PidsStats struct {
+		Current uint32 `json:"current"`
+	} `json:"pids_stats"`
+}
+
+func (d *Dispatcher) streamStats(ctx context.Context, rec *instanceRecord) {
+	rc, err := d.runtime.StatsStream(ctx, rec.ContainerID)
+	if err != nil {
+		d.log.Warn("stats stream open failed", "instance_id", rec.InstanceID, "err", err)
+		return
+	}
+	defer rc.Close()
+	d.log.Info("stats stream opened", "instance_id", rec.InstanceID)
+	emitCount := 0
+	defer func() { d.log.Info("stats stream closed", "instance_id", rec.InstanceID, "emitted", emitCount) }()
+
+	dec := json.NewDecoder(rc)
+	lastEmit := time.Time{}
+	for {
+		if ctx.Err() != nil {
+			return
+		}
+		var f dockerStatsFrame
+		if err := dec.Decode(&f); err != nil {
+			if err != io.EOF && ctx.Err() == nil {
+				d.log.Debug("stats decode ended", "instance_id", rec.InstanceID, "err", err)
+			}
+			return
+		}
+		if time.Since(lastEmit) < statsEmitInterval {
+			continue
+		}
+		memUsed := d.sendStats(rec.InstanceID, &f)
+		lastEmit = time.Now()
+		emitCount++
+		// ARK runaway-memory guardrails. Two stages: warn (45 GB) tries a
+		// graceful flush; hard (60 GB) SIGKILLs to protect the host.
+		// Both paths set rec.stopping inside their handler so subsequent
+		// samples short-circuit there.
+		if rec.ModuleID == "ark-sa" {
+			if memUsed > arkRAMHardLimit {
+				go d.emergencyRestart(rec, fmt.Sprintf(
+					"ARK RAM hit %.1f GB (hard limit %d GB) — bouncing to protect host",
+					float64(memUsed)/1024/1024/1024, arkRAMHardLimit/1024/1024/1024))
+				return
+			}
+			if memUsed > arkRAMWarnLimit && rec.memGuardWarned.CompareAndSwap(false, true) {
+				go d.gracefulMemoryRestart(rec, fmt.Sprintf(
+					"ARK RAM hit %.1f GB (warn limit %d GB) — flushing save + graceful restart",
+					float64(memUsed)/1024/1024/1024, arkRAMWarnLimit/1024/1024/1024))
+				// Don't return — keep streaming so the hard limit can still
+				// fire if RAM keeps climbing past 60 GB during the 60s flush
+				// window. The memGuardWarned latch prevents the warn path
+				// from re-arming, but the hard path is the host-protection
+				// backstop and must remain live.
+			}
+		}
+	}
+}
+
+func (d *Dispatcher) sendStats(instanceID string, f *dockerStatsFrame) uint64 {
+	// CPU%: delta total / delta system * online_cpus * 100.
+	// First frame's precpu_stats is zero so cpuPct comes out 0 — fine.
+	var cpuPct float64
+	cpuDelta := int64(f.CPUStats.CPUUsage.TotalUsage) - int64(f.PreCPUStats.CPUUsage.TotalUsage)
+	sysDelta := int64(f.CPUStats.SystemCPUUsage) - int64(f.PreCPUStats.SystemCPUUsage)
+	hostCpus := f.CPUStats.OnlineCPUs
+	if sysDelta > 0 && cpuDelta > 0 {
+		cpus := float64(hostCpus)
+		if cpus < 1 {
+			cpus = 1
+		}
+		cpuPct = float64(cpuDelta) / float64(sysDelta) * cpus * 100.0
+	}
+
+	var rx, tx uint64
+	for _, n := range f.Networks {
+		rx += n.RxBytes
+		tx += n.TxBytes
+	}
+
+	// "Real" memory: subtract file cache (cgroups v1) or inactive_file
+	// (cgroups v2). Falls through to raw usage when neither is reported
+	// (Windows containers, exotic runtimes). This is what `docker stats`
+	// shows and what operators expect — the inflated ARK numbers (~25 GB
+	// vs the actual ~12 GB RSS) were the cache-included variant.
+	memUsed := f.MemoryStats.Usage
+	if f.MemoryStats.Stats.InactiveFile > 0 && f.MemoryStats.Stats.InactiveFile <= memUsed {
+		memUsed -= f.MemoryStats.Stats.InactiveFile
+	} else if f.MemoryStats.Stats.Cache > 0 && f.MemoryStats.Stats.Cache <= memUsed {
+		memUsed -= f.MemoryStats.Stats.Cache
+	}
+
+	d.sendEnv(&panelv1.AgentEnvelope{
+		SentAt: timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_InstanceStats{
+			InstanceStats: &panelv1.InstanceStatsUpdate{
+				InstanceId:    instanceID,
+				CpuPercent:    cpuPct,
+				MemUsedBytes:  memUsed,
+				MemLimitBytes: f.MemoryStats.Limit,
+				NetRxBytes:    rx,
+				NetTxBytes:    tx,
+				Pids:          f.PidsStats.Current,
+				HostCpus:      hostCpus,
+				At:            timestamppb.Now(),
+			},
+		},
+	})
+	return memUsed
+}
+
+// gracefulMemoryRestart is the 45 GB warn path. Tries to land a clean
+// SaveWorld before bouncing the container so the next boot doesn't face
+// a torn SQLite save. Sequence:
+//
+//   1. Latch rec.stopping so other paths back off.
+//   2. Emit a panel log line + INSTANCE_STATUS_STOPPING state.
+//   3. RCON Broadcast: "[Panel] high memory — saving and restarting in 60s"
+//   4. RCON SaveWorld
+//   5. Sleep 60s (gives players warning + engine time to write the save)
+//   6. Cancel tracker/log goroutines, runtime.Restart with 120s grace.
+//
+// Uses Restart NOT Stop: a plain Stop on an `unless-stopped` container
+// is honored by docker as an operator stop — the container then stays
+// Exited until someone manually restarts it. That regression took
+// Ragnarok offline for 17 hours after the 02:02 PDT warn fire on
+// 2026-05-25 (see HANDOFF "Last meaningfully updated"). Restart bounces
+// the container as a transient event so it comes back up automatically;
+// the operator just sees a brief blip + the "[panel] MEMORY-WARN" line.
+//
+// If any RCON step fails (engine wedged, RCON socket dead), fall through
+// to the restart anyway — we still want the engine down + back up. The
+// 120s docker grace gives the engine a real shot at a clean DoExit even
+// when RCON is the broken thing.
+func (d *Dispatcher) gracefulMemoryRestart(rec *instanceRecord, reason string) {
+	if !rec.stopping.CompareAndSwap(false, true) {
+		return
+	}
+	d.log.Warn("ark memory warn — graceful restart", "instance_id", rec.InstanceID, "reason", reason)
+
+	d.sendEnv(&panelv1.AgentEnvelope{
+		SentAt: timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+			InstanceId: rec.InstanceID,
+			Stream:     "stderr",
+			At:         timestamppb.Now(),
+			Line:       "[panel] MEMORY-WARN: " + reason,
+		}},
+	})
+	d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPING, 0, "graceful memory restart: "+reason)
+
+	// Best-effort RCON broadcast + SaveWorld. Use a short per-call
+	// timeout so a wedged RCON socket doesn't block the whole graceful
+	// path. tracker.Exec redials internally, so a closed socket from
+	// idle-disconnect won't burn the attempt.
+	if rec.Tracker != nil {
+		bctx, bcancel := context.WithTimeout(context.Background(), 5*time.Second)
+		if _, err := rec.Tracker.Exec(bctx, "Broadcast [Panel] High memory detected. Server saving and restarting in 60 seconds."); err != nil {
+			d.log.Warn("rcon broadcast failed during graceful memory restart", "instance_id", rec.InstanceID, "err", err)
+		}
+		bcancel()
+
+		sctx, scancel := context.WithTimeout(context.Background(), 10*time.Second)
+		if _, err := rec.Tracker.Exec(sctx, "SaveWorld"); err != nil {
+			d.log.Warn("rcon SaveWorld failed during graceful memory restart", "instance_id", rec.InstanceID, "err", err)
+		}
+		scancel()
+	}
+
+	// Wait the warning window — players see the broadcast, engine
+	// finishes writing the save. Sleeping in the goroutine is fine;
+	// the stats loop continues to monitor for the 60 GB hard limit
+	// in case RAM keeps climbing during the wait.
+	time.Sleep(60 * time.Second)
+
+	// Cancel ancillary goroutines so they don't fight the bounce.
+	// They'll be re-armed by watchExit when the restart settles.
+	d.mu.Lock()
+	if rec.TrackerCancel != nil {
+		rec.TrackerCancel()
+		rec.TrackerCancel = nil
+	}
+	if rec.LogCancel != nil {
+		rec.LogCancel()
+		rec.LogCancel = nil
+	}
+	d.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), gracefulStopGracePeriod+30*time.Second)
+	defer cancel()
+	if err := d.runtime.Restart(ctx, rec.ContainerID, gracefulStopGracePeriod); err != nil {
+		d.log.Error("graceful memory restart runtime failure", "instance_id", rec.InstanceID, "err", err)
+		// Restart failed — surface to operator so they know to investigate.
+		d.sendEnv(&panelv1.AgentEnvelope{
+			SentAt: timestamppb.Now(),
+			Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+				InstanceId: rec.InstanceID,
+				Stream:     "stderr",
+				At:         timestamppb.Now(),
+				Line:       "[panel] MEMORY-WARN restart failed: " + err.Error(),
+			}},
+		})
+	}
+	// Clear both the stopping and memGuardWarned latches so future
+	// memory-warn fires can re-arm on this container's next life.
+	rec.stopping.Store(false)
+	rec.memGuardWarned.Store(false)
+}
+
+// emergencyRestart bounces a runaway container outside the normal
+// handleStop path. Difference from handleStop: 30s grace (not 60s),
+// runs even when no operator request is in flight, and emits a clear
+// log line so the dashboard's Console tab shows WHY this happened.
+//
+// Uses Restart NOT Stop — see gracefulMemoryRestart for the rationale.
+// A plain Stop on `unless-stopped` containers parks them Exited until
+// manually started; the panel operator has no way to know they need to
+// click Start. Bouncing the container keeps the service available, lets
+// the engine recover into a fresh process with no leak state, and the
+// "[panel] AUTO-KILL" log line is preserved for after-the-fact diagnosis.
+// If the underlying leak is persistent, RAM will climb again and the
+// guard will fire again — that's the operator's signal to investigate.
+//
+// 2026-05-25: bumped from 5s → 30s grace. The 5s window guaranteed
+// SIGKILL mid-save when this fired during a SaveWorld, and that's
+// what was producing the FAtlasSaveManager corruption pattern. 30s
+// is enough for the engine to finish a save in flight without giving
+// up host-OOM protection.
+//
+// Idempotent via rec.stopping: the stats loop fires this from a goroutine
+// every 3s while RAM is over the limit, but the second+ invocations
+// short-circuit at the stopping check so we don't pile up Stop calls
+// against a container that's already heading down.
+func (d *Dispatcher) emergencyRestart(rec *instanceRecord, reason string) {
+	if !rec.stopping.CompareAndSwap(false, true) {
+		return
+	}
+	d.log.Warn("emergency restart", "instance_id", rec.InstanceID, "reason", reason)
+
+	// Surface to operator via log line + state update.
+	d.sendEnv(&panelv1.AgentEnvelope{
+		SentAt: timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+			InstanceId: rec.InstanceID,
+			Stream:     "stderr",
+			At:         timestamppb.Now(),
+			Line:       "[panel] AUTO-KILL: " + reason,
+		}},
+	})
+	d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPING, 0, "auto-kill: "+reason)
+
+	// Cancel ancillary goroutines so they don't fight the bounce.
+	// They'll be re-armed by watchExit when the restart settles.
+	d.mu.Lock()
+	if rec.TrackerCancel != nil {
+		rec.TrackerCancel()
+		rec.TrackerCancel = nil
+	}
+	if rec.LogCancel != nil {
+		rec.LogCancel()
+		rec.LogCancel = nil
+	}
+	d.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), emergencyStopGracePeriod+30*time.Second)
+	defer cancel()
+	if err := d.runtime.Restart(ctx, rec.ContainerID, emergencyStopGracePeriod); err != nil {
+		d.log.Error("emergency restart runtime failure", "instance_id", rec.InstanceID, "err", err)
+		d.sendEnv(&panelv1.AgentEnvelope{
+			SentAt: timestamppb.Now(),
+			Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+				InstanceId: rec.InstanceID,
+				Stream:     "stderr",
+				At:         timestamppb.Now(),
+				Line:       "[panel] AUTO-KILL restart failed: " + err.Error(),
+			}},
+		})
+	}
+	// Clear both latches so future signals can re-arm on the bounced container.
+	rec.stopping.Store(false)
+	rec.memGuardWarned.Store(false)
+}

agent/internal/dispatch/updatecheck.go

@@ -0,0 +1,212 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/dbledeez/panel/agent/internal/updater"
+	"github.com/dbledeez/panel/pkg/steamvdf"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// Update-available check (WI-14). CHECK-ONLY: reads the installed
+// buildid from the instance's Steam appmanifest ACF and the latest
+// buildid for its branch via `steamcmd +app_info_print` — it never
+// starts an update or touches the install.
+
+// appInfoCacheTTL is how long a fetched branch map is reused before a
+// fresh steamcmd run. app_info runs cost a sidecar spin-up (~10-30 s),
+// so checks across many instances of the same game share one fetch.
+const appInfoCacheTTL = 15 * time.Minute
+
+type appInfoCacheEntry struct {
+	branches  map[string]string
+	fetchedAt time.Time
+}
+
+var (
+	appInfoCacheMu sync.Mutex
+	appInfoCache   = map[string]appInfoCacheEntry{} // key: app_id
+	// appInfoFetchMu single-flights concurrent fetches per process; a
+	// coarse lock is fine — checks are rare and operator-initiated.
+	appInfoFetchMu sync.Mutex
+)
+
+func (d *Dispatcher) handleUpdateCheck(corrID string, req *panelv1.UpdateCheckRequest) {
+	fail := func(msg string) {
+		d.sendUpdateCheckResult(corrID, &panelv1.UpdateCheckResult{
+			InstanceId: req.InstanceId, Error: msg,
+		})
+	}
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		fail(err.Error())
+		return
+	}
+	manifest, ok := d.modules.Get(rec.ModuleID)
+	if !ok {
+		fail("module not in registry")
+		return
+	}
+	// Same provider selection as handleUpdate: by id when supplied, else
+	// first in the manifest.
+	var spec *modulepkg_UpdateProvider
+	for i := range manifest.UpdateProviders {
+		p := &manifest.UpdateProviders[i]
+		if req.ProviderId == "" || p.ID == req.ProviderId {
+			spec = p
+			break
+		}
+	}
+	if spec == nil {
+		fail(fmt.Sprintf("no provider %q on module %s", req.ProviderId, rec.ModuleID))
+		return
+	}
+	if spec.Kind != "steamcmd" {
+		fail(fmt.Sprintf("update check supports steamcmd providers only (provider %q is %q)", spec.ID, spec.Kind))
+		return
+	}
+	if spec.AppID == "" {
+		fail("provider has no app_id")
+		return
+	}
+
+	// The check spins a steamcmd sidecar (tens of seconds) — never block
+	// the dispatch read loop.
+	go func() {
+		branch := spec.Beta
+		branchKey := branch
+		if branchKey == "" {
+			branchKey = "public"
+		}
+		res := &panelv1.UpdateCheckResult{
+			InstanceId: req.InstanceId,
+			AppId:      spec.AppID,
+			Branch:     branchKey,
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+		defer cancel()
+
+		// Installed side: appmanifest ACF under the provider's install
+		// root. Prefer betakey/buildid straight from the manifest file.
+		if acf, rerr := d.readInstalledACF(ctx, rec, spec.InstallPath, spec.AppID); rerr != nil {
+			res.Error = "read appmanifest: " + rerr.Error()
+		} else if m, perr := steamvdf.ParseAppManifest(string(acf)); perr != nil {
+			res.Error = "parse appmanifest: " + perr.Error()
+		} else {
+			res.InstalledBuildid = m.BuildID
+			if m.BetaKey != "" {
+				// Trust the on-disk truth over the manifest's declared
+				// beta — this is exactly the "bounced between branches"
+				// case the check exists to expose.
+				res.Branch = m.BetaKey
+				branchKey = m.BetaKey
+			}
+		}
+
+		// Latest side: branch map via steamcmd app_info (15-min cache).
+		branches, cached, ferr := d.latestBranches(ctx, rec.InstanceID, spec.AppID, req.Refresh)
+		if ferr != nil {
+			if res.Error != "" {
+				res.Error += "; "
+			}
+			res.Error += "fetch latest: " + ferr.Error()
+		} else {
+			res.Cached = cached
+			if bid, ok := branches[branchKey]; ok {
+				res.LatestBuildid = bid
+			} else if res.Error == "" {
+				res.Error = fmt.Sprintf("branch %q not in Steam branches map (%d branches known)", branchKey, len(branches))
+			}
+		}
+
+		res.UpdateAvailable = res.InstalledBuildid != "" && res.LatestBuildid != "" &&
+			res.InstalledBuildid != res.LatestBuildid
+		d.log.Info("update check",
+			"instance_id", req.InstanceId, "app_id", spec.AppID, "branch", branchKey,
+			"installed", res.InstalledBuildid, "latest", res.LatestBuildid,
+			"update_available", res.UpdateAvailable, "cached", res.Cached, "err", res.Error)
+		d.sendUpdateCheckResult(corrID, res)
+	}()
+}
+
+// latestBranches returns the branch→buildid map for appID, from cache
+// when fresh (unless refresh), otherwise via a steamcmd sidecar run.
+func (d *Dispatcher) latestBranches(ctx context.Context, instanceID, appID string, refresh bool) (map[string]string, bool, error) {
+	if !refresh {
+		appInfoCacheMu.Lock()
+		e, ok := appInfoCache[appID]
+		appInfoCacheMu.Unlock()
+		if ok && time.Since(e.fetchedAt) < appInfoCacheTTL {
+			return e.branches, true, nil
+		}
+	}
+	appInfoFetchMu.Lock()
+	defer appInfoFetchMu.Unlock()
+	// Re-check under the fetch lock — a concurrent check may have just
+	// filled the cache while we waited.
+	if !refresh {
+		appInfoCacheMu.Lock()
+		e, ok := appInfoCache[appID]
+		appInfoCacheMu.Unlock()
+		if ok && time.Since(e.fetchedAt) < appInfoCacheTTL {
+			return e.branches, true, nil
+		}
+	}
+	branches, err := updater.FetchAppInfoBranches(ctx, d.runtime, instanceID, appID, func(line string) {
+		d.log.Debug("app_info", "instance_id", instanceID, "line", line)
+	})
+	if err != nil {
+		return nil, false, err
+	}
+	appInfoCacheMu.Lock()
+	appInfoCache[appID] = appInfoCacheEntry{branches: branches, fetchedAt: time.Now()}
+	appInfoCacheMu.Unlock()
+	return branches, false, nil
+}
+
+// readInstalledACF reads steamapps/appmanifest_<appID>.acf under the
+// provider's install root, using the same container-first/host-fallback
+// mechanics as the FsRead RPC (fs helper sidecar works on stopped
+// instances too).
+func (d *Dispatcher) readInstalledACF(ctx context.Context, rec *instanceRecord, installPath, appID string) ([]byte, error) {
+	rel := path.Join("steamapps", fmt.Sprintf("appmanifest_%s.acf", appID))
+	if d.useContainerOps(rec) {
+		root := installPath
+		if root == "" {
+			root = rec.BrowseableRoot
+		}
+		abs, err := safeJoinAny(root, append(rootPaths(rec), root), path.Join(root, rel))
+		if err != nil {
+			return nil, err
+		}
+		targetID, err := d.getFsTargetContainerID(ctx, rec)
+		if err != nil {
+			return nil, err
+		}
+		return d.runtime.CopyFileFromContainer(ctx, targetID, abs)
+	}
+	if rec.DataPath == "" {
+		return nil, fmt.Errorf("no file storage available")
+	}
+	abs, err := safeJoinHost(rec.DataPath, rel)
+	if err != nil {
+		return nil, err
+	}
+	return os.ReadFile(abs)
+}
+
+func (d *Dispatcher) sendUpdateCheckResult(corrID string, res *panelv1.UpdateCheckResult) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload:       &panelv1.AgentEnvelope_UpdateCheckResult{UpdateCheckResult: res},
+	})
+}

agent/internal/dispatch/updates.go

@@ -0,0 +1,261 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/dbledeez/panel/agent/internal/updater"
+	modulepkg "github.com/dbledeez/panel/pkg/module"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// Use a type alias so the file's handler signatures don't bleed the
+// modulepkg import into their exposed type names.
+type modulepkg_UpdateProvider = modulepkg.UpdateProvider
+
+// registerUpdater stores the cancel func for the in-flight updater of
+// an instance. Returns (slot, true) when registration succeeds, or
+// (nil, false) when another updater is already running for the same
+// instance.
+//
+// We refuse instead of preempting because preempting races the docker
+// daemon's name cleanup: cancelling the prior context returns instantly,
+// but the prior steamcmd sidecar (e.g. panel-<id>-steamcmd-1) takes a
+// few seconds to actually exit and be removed. A new updater that
+// charges in immediately hits a "name already in use" Docker error and,
+// worse, leaves SteamCMD's staging directory mid-finalize (state 0x602)
+// — files stranded under <install>/steamapps/downloading/<appid>/.
+//
+// Concrete failure mode that prompted this guard:
+//   - panelctl create palworld-test → auto-triggers first update.
+//   - Operator (impatiently) runs panelctl update palworld-test 13s later.
+//   - Old behavior cancelled the auto-update mid-validate, then the new
+//     run failed name-conflict, then SteamCMD looped on exit-8 / 0x602.
+//
+// The delete path uses cancelUpdater (different entry), so this guard
+// doesn't block teardown.
+func (d *Dispatcher) registerUpdater(instanceID string, cancel context.CancelFunc) (*updaterSlot, bool) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if existing := d.updaters[instanceID]; existing != nil {
+		return nil, false
+	}
+	slot := &updaterSlot{cancel: cancel}
+	d.updaters[instanceID] = slot
+	return slot, true
+}
+
+// unregisterUpdater clears the registration ONLY IF the current entry
+// is still our slot — pointer compare guards against a later updater
+// having taken over.
+func (d *Dispatcher) unregisterUpdater(instanceID string, slot *updaterSlot) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.updaters[instanceID] == slot {
+		delete(d.updaters, instanceID)
+	}
+}
+
+// cancelUpdater stops any in-flight updater for this instance and
+// removes the registration. Called from the delete path so the sidecar
+// gets torn down before we try to remove the volume.
+func (d *Dispatcher) cancelUpdater(instanceID string) {
+	d.mu.Lock()
+	slot := d.updaters[instanceID]
+	delete(d.updaters, instanceID)
+	d.mu.Unlock()
+	if slot != nil && slot.cancel != nil {
+		slot.cancel()
+	}
+}
+
+// Update handler. Controller sends ControllerEnvelope_Update; agent:
+//
+//  1. Looks up the instance + module manifest.
+//  2. Picks the update provider by id (or defaults to the first one).
+//  3. Emits a synchronous UpdateResult{accepted=true} so the panel can
+//     un-gray the button and show "update started".
+//  4. Spawns a goroutine that runs the provider, streaming each progress
+//     line as a LogLine{stream="update"}. Final sentinel line
+//     "update:ok" or "update:error:<msg>" tells the UI we're done.
+//
+// Progress is not correlation-routed because the UI subscribes to the
+// normal event bus and picks up LogLines directly.
+
+func (d *Dispatcher) handleUpdate(corrID string, req *panelv1.UpdateRequest) {
+	d.handleUpdateWithHook(corrID, req, nil)
+}
+
+// handleUpdateWithHook is the internal form that optionally fires onComplete
+// once the provider run finishes (success or failure). Used by the auto-
+// update-on-create path to chain a Start after the first-time download and
+// to clear UI state if the update errors out. The normal controller-driven
+// Update flow passes nil so the hook stays opt-in.
+func (d *Dispatcher) handleUpdateWithHook(corrID string, req *panelv1.UpdateRequest, onComplete func(err error)) {
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendUpdateResult(corrID, false, err.Error(), "", "")
+		return
+	}
+	manifest, ok := d.modules.Get(rec.ModuleID)
+	if !ok {
+		d.sendUpdateResult(corrID, false, "module not in registry", "", "")
+		return
+	}
+
+	// Special pseudo-provider "_basemods": short-circuits to the third-
+	// party base-mod installer (Allocs + PrismaCore) instead of running
+	// the module's normal updater. Reuses the Update RPC plumbing so the
+	// UI gets the same "updating" pulse + error surface as a real update.
+	// Reasoning for the underscore prefix: real provider ids never start
+	// with one (manifest schema convention), so collisions are impossible.
+	if req.ProviderId == "_basemods" {
+		d.sendUpdateResult(corrID, true, "", "_basemods", "basemods")
+		d.log.Info("install-base-mods: kicked", "instance_id", req.InstanceId)
+		go func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+			defer cancel()
+			err := d.installBaseMods(ctx, req.InstanceId, manifest, rec.DataPath)
+			if err != nil {
+				d.log.Error("install-base-mods: failed", "instance_id", req.InstanceId, "err", err)
+			} else {
+				d.log.Info("install-base-mods: done", "instance_id", req.InstanceId)
+			}
+			if onComplete != nil {
+				onComplete(err)
+			}
+		}()
+		return
+	}
+
+	// Pick provider: by id if supplied, else first in manifest.
+	var spec *modulepkg_UpdateProvider
+	for i := range manifest.UpdateProviders {
+		p := &manifest.UpdateProviders[i]
+		if req.ProviderId == "" || p.ID == req.ProviderId {
+			spec = providerAdapter(p)
+			break
+		}
+	}
+	if spec == nil {
+		d.sendUpdateResult(corrID, false, fmt.Sprintf("no provider %q on module %s", req.ProviderId, rec.ModuleID), "", "")
+		return
+	}
+
+	prov, err := updater.Get(spec.Kind)
+	if err != nil {
+		d.sendUpdateResult(corrID, false, err.Error(), spec.ID, spec.Kind)
+		return
+	}
+
+	// Reject if another updater is already running on this instance —
+	// see registerUpdater for why we refuse instead of preempting.
+	//
+	// Timeout bumped 30 → 90 min (2026-05-04) — Steam CDN can rate-limit
+	// anonymous-login pulls down to 1-3 MB/s, and a 17 GB game (7DTD,
+	// ARK SA) at 5 MB/s = 56 min, easily over 30. Killing the sidecar
+	// at 30 min leaves a partial install + a confused operator. 90 min
+	// covers the realistic worst case with margin; truly hung steamcmd
+	// still gets killed eventually so we don't burn an updater slot
+	// forever.
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Minute)
+	slot, ok := d.registerUpdater(req.InstanceId, cancel)
+	if !ok {
+		cancel()
+		d.sendUpdateResult(corrID, false, "another update is already in progress for this instance", spec.ID, spec.Kind)
+		d.log.Info("update rejected: already running", "instance_id", req.InstanceId, "provider_id", spec.ID)
+		return
+	}
+
+	// Accept synchronously — run async.
+	d.sendUpdateResult(corrID, true, "", spec.ID, spec.Kind)
+	d.log.Info("update started", "instance_id", req.InstanceId, "provider_id", spec.ID, "kind", spec.Kind)
+
+	// Surface the in-flight update as a server-authoritative status.
+	// Without this, the panel's status pill stayed at whatever it was
+	// before the update (often "stopped" or worse, "crashed" if the
+	// previous run had hard-exited) — operators saw "crashed" while
+	// SteamCMD was actually downloading several GB. Emitting UPDATING
+	// here gives every dashboard viewer the same accurate "installing"
+	// pulse, regardless of which client kicked off the update.
+	d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, "updating:"+spec.Kind)
+
+	go func() {
+		defer cancel()
+		defer d.unregisterUpdater(req.InstanceId, slot)
+		sink := func(line string) {
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: rec.InstanceID,
+					Stream:     "update",
+					At:         timestamppb.Now(),
+					Line:       line,
+				}},
+			})
+		}
+		sink(fmt.Sprintf("---- update started: provider=%s kind=%s ----", spec.ID, spec.Kind))
+		uc := &updater.Context{
+			InstanceID:     rec.InstanceID,
+			Manifest:       manifest,
+			ContainerID:    rec.ContainerID,
+			BrowseableRoot: rec.BrowseableRoot,
+			DataPath:       rec.DataPath,
+			Runtime:        d.runtime,
+			Log:            sink,
+			// Steam login (if any) is forwarded by the controller for
+			// modules that set `requires_steam_login`. Fields are empty
+			// for anonymous-login apps — steamcmd.go falls back to that.
+			SteamUsername: req.SteamUsername,
+			SteamPassword: req.SteamPassword,
+		}
+		// ctx + cancel + slot were registered before the goroutine launched
+		// (see registerUpdater above). The defer at the top of the goroutine
+		// handles cleanup; the dispatcher's delete path can still abort us
+		// via cancelUpdater.
+		if err := prov.Update(ctx, uc, spec); err != nil {
+			sink(fmt.Sprintf("update:error: %s", err.Error()))
+			d.log.Warn("update failed", "instance_id", req.InstanceId, "err", err)
+			// Clear the UPDATING pill — surface the failure as CRASHED with
+			// a structured detail so operators see "install_failed: …" in
+			// the UI rather than the pill being stuck on UPDATING forever.
+			d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, -1, "install_failed: "+err.Error())
+			if onComplete != nil {
+				onComplete(err)
+			}
+			return
+		}
+		sink("update:ok")
+		d.log.Info("update complete", "instance_id", req.InstanceId, "provider_id", spec.ID)
+		// Update succeeded — drop UPDATING back to STOPPED. handleStart's
+		// own state emission supersedes if an auto-start follows. Without
+		// this, a manually-triggered update left the pill on UPDATING
+		// indefinitely until the next start/stop cycle.
+		d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "installed")
+		if onComplete != nil {
+			onComplete(nil)
+		}
+	}()
+}
+
+func (d *Dispatcher) sendUpdateResult(corrID string, accepted bool, errMsg, providerID, providerKind string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_UpdateResult{
+			UpdateResult: &panelv1.UpdateResult{
+				Accepted:     accepted,
+				Error:        errMsg,
+				ProviderId:   providerID,
+				ProviderKind: providerKind,
+			},
+		},
+	})
+}
+
+// providerAdapter is a pass-through that pins us to the modulepkg type
+// without needing to add another import alias at the top of this file.
+func providerAdapter(p *modulepkg_UpdateProvider) *modulepkg_UpdateProvider { return p }

agent/internal/dispatch/warmseed.go

@@ -0,0 +1,331 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	agentmodule "github.com/dbledeez/panel/agent/internal/module"
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	modulepkg "github.com/dbledeez/panel/pkg/module"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// tryWarmSeedFromSibling looks for another instance of the same module
+// already on this agent with a populated install volume; if found, copies
+// that volume verbatim into the new instance's install volume via a
+// throwaway debian sidecar. Saves operators 30+ minutes of redundant
+// SteamCMD downloads when standing up a second instance of a game.
+//
+// Returns (true, nil) on a successful seed — caller should skip the
+// updater and treat the install as already-complete.
+// Returns (false, nil) when there's no candidate sibling — caller falls
+// through to the normal updater.
+// Returns (false, err) on operational failure of the seed itself —
+// caller should also fall through (we'd rather over-download than leave
+// the user without a working install).
+//
+// Symlink handling: install volumes typically contain symlinks back into
+// the per-instance saves volume (e.g. /game/serverconfig.xml ->
+// /game-saves/serverconfig.xml on 7DTD, set up by the entrypoint at first
+// boot). Those symlinks would be DANGLING in the new instance because
+// the new saves volume is empty. We strip broken symlinks after copy
+// (`find -xtype l -delete`) so the entrypoint sees a clean state and
+// recreates them on its first boot.
+//
+// Saves bootstrap: if the sibling's saves volume contains a
+// serverconfig.xml-style file at a known location, we copy it to the
+// new saves volume so the entrypoint has a starting config (operator
+// can edit afterwards). Otherwise the entrypoint falls back to the
+// game's shipped default. Done per-module by the seedSavesPaths list.
+func (d *Dispatcher) tryWarmSeedFromSibling(ctx context.Context, newInstanceID string, manifest *modulepkg.Manifest, newDataPath string) (bool, error) {
+	if manifest == nil || len(manifest.UpdateProviders) == 0 {
+		return false, nil
+	}
+	provider := manifest.UpdateProviders[0]
+	installPath := provider.InstallPath
+	if installPath == "" {
+		installPath = manifest.Runtime.Docker.BrowseableRoot
+	}
+	if installPath == "" {
+		return false, nil
+	}
+
+	// Find a sibling: same agent, same module, NOT this instance, with a
+	// container that exists (so we know the volume was at least populated
+	// once). Prefer a stopped or running instance over a half-baked one.
+	//
+	// VERSION SAFETY: only reuse an install from a sibling on the SAME Steam
+	// branch as this new instance. Copying e.g. a 3.0 (latest_experimental)
+	// install into a server the operator created as 2.6 — or vice versa —
+	// produces a binary/world mismatch that corrupts the save. The new
+	// instance's record was tagged with its branch at Create (resolveCreateBranch),
+	// before this runs. Siblings predating the feature have an empty branch and
+	// are treated as the module's DEFAULT branch (branchOrDefault) — for 7DTD
+	// that's v2.6, so the existing 2.6 fleet is still a valid seed source for a
+	// new 2.6 server.
+	d.mu.Lock()
+	wantBranch := branchOrDefault("", manifest)
+	if nr, ok := d.instances[newInstanceID]; ok {
+		wantBranch = branchOrDefault(nr.Branch, manifest)
+	}
+	var srcID, srcDataPath string
+	for id, rec := range d.instances {
+		if id == newInstanceID {
+			continue
+		}
+		if rec.ModuleID != manifest.ID {
+			continue
+		}
+		if branchOrDefault(rec.Branch, manifest) != wantBranch {
+			continue // different game version — not a safe seed source
+		}
+		// Best-effort: if we have multiple on this branch, the first one wins.
+		// Could be improved later by picking the most-recently-updated.
+		srcID = id
+		srcDataPath = rec.DataPath
+		break
+	}
+	d.mu.Unlock()
+	if srcID == "" {
+		d.log.Debug("warm-seed: no same-branch sibling instance found", "module", manifest.ID, "instance", newInstanceID, "want_branch", wantBranch)
+		return false, nil
+	}
+	d.log.Info("warm-seed: matched sibling on branch", "module", manifest.ID, "instance", newInstanceID, "src", srcID, "branch", wantBranch)
+
+	srcVolumes := agentmodule.ResolveVolumes(manifest, srcID, srcDataPath)
+	dstVolumes := agentmodule.ResolveVolumes(manifest, newInstanceID, newDataPath)
+
+	srcGameVol := volumeNameForPath(srcVolumes, installPath)
+	dstGameVol := volumeNameForPath(dstVolumes, installPath)
+	if srcGameVol == "" || dstGameVol == "" {
+		return false, fmt.Errorf("warm-seed: install_path %q not in module volumes", installPath)
+	}
+
+	// Find a "saves-style" sibling volume to bootstrap from. Heuristic:
+	// any non-install named volume whose container path contains "save"
+	// or "data". Most modules have one. If none, we still seed the install
+	// volume; the entrypoint may still need to bootstrap saves itself.
+	srcSavesVol, srcSavesPath := findSavesVolume(srcVolumes, installPath)
+	dstSavesVol, _ := findSavesVolume(dstVolumes, installPath)
+
+	// Build the seed sidecar. debian:12-slim is small, has bash + find +
+	// cp + mkdir, and is already cached on every panel-agent host (the
+	// fs-helper sidecar uses it).
+	// Why a tar pipe instead of cp -a: we want fine-grained exclusions for
+	// per-instance state that shouldn't be inherited from the sibling.
+	//
+	// Mods/ split:
+	//   BASE mods (TFP_*, 0_TFP_*, Allocs_*, PrismaCore, xMarkers) are
+	//   load-bearing — TFP_Harmony is the runtime patcher, the server
+	//   won't even boot without it; Allocs is the live map renderer the
+	//   panel maps tab depends on; PrismaCore is the staff command surface
+	//   refugebotserver expects. We KEEP these.
+	//
+	//   CUSTOM mods (AGF-VP-*, RefugeBot, AsylumRoboticInbox, anything
+	//   else) are server-specific — operators usually want a clean slate
+	//   on a new server so the panel's mod manager / their own upload
+	//   decides what's installed. We DROP these.
+	//
+	// steamapps/userdata, steamapps/downloading: per-Steam-account state
+	// + half-fetched chunks. Always excluded.
+	//
+	// Approach: tar everything EXCEPT Mods + downloading + userdata, then
+	// selectively copy back the base-mod entries from the sibling.
+	// The Mods/ split + base-mod completeness gate below are 7DTD-SPECIFIC:
+	// TFP_Harmony/Allocs/PrismaCore are load-bearing 7DTD server mods, and the
+	// "missing base mods → exit 42" check re-triggers a full SteamCMD validate
+	// to refetch them. Every OTHER module (palworld, soulmask, valheim, …) has
+	// no Mods/ dir at all, so running this block against them ALWAYS hit the
+	// completeness gate and exit-42'd — turning a valid sibling seed into a
+	// spurious "incomplete sibling" failure. For non-7DTD modules we copy the
+	// whole install verbatim (minus per-instance Steam state) and skip the gate.
+	//
+	// Root cause of the 2026-07-10 gamehost outage: Palworld + Soulmask checkouts
+	// warm-seeded from a sibling, hit this 7DTD gate, exit-42'd, and fell through
+	// to a fresh download — which was then canceled by a racing env-config
+	// recreate (fixed separately in the controller). See changelog 2026-07-10.
+	seedScript := `set -e
+echo "warm-seed: copying install volume contents (excl. Mods/, steamapps/userdata, steamapps/downloading)"
+find /dst-game -mindepth 1 -delete
+( cd /src-game && tar --exclude='./Mods' --exclude='./steamapps/userdata' --exclude='./steamapps/downloading' -cf - . ) | ( cd /dst-game && tar -xf - )
+`
+	if manifest.ID == "7dtd" {
+		seedScript += `mkdir -p /dst-game/Mods
+echo "warm-seed: copying BASE mods (TFP_*, Allocs_*, PrismaCore, Xample_MarkersMod) — load-bearing for the server"
+for mod in /src-game/Mods/TFP_* /src-game/Mods/0_TFP_* /src-game/Mods/Allocs_* /src-game/Mods/PrismaCore /src-game/Mods/Xample_* /src-game/Mods/*Markers* /src-game/Mods/xMarkers /src-game/Mods/xmarkers; do
+  case "$mod" in *"*"*) continue ;; esac
+  if [ -e "$mod" ]; then
+    cp -a "$mod" /dst-game/Mods/
+    echo "  + $(basename $mod)"
+  fi
+done
+`
+	}
+	seedScript += `echo "warm-seed: stripping dangling symlinks (saves-volume references)"
+find /dst-game -xtype l -delete
+`
+	if manifest.ID == "7dtd" {
+		seedScript += `# Completeness check (7DTD only) — fail the warmseed if any LOAD-BEARING base
+# mod is missing on dst. The caller falls back to a full steamcmd validate which
+# will re-fetch the TFP/xMarkers mods from the dedicated-server depot.
+missing=""
+for required in 0_TFP_Harmony TFP_WebServer; do
+  [ -d "/dst-game/Mods/$required" ] || missing="$missing $required"
+done
+if [ -n "$missing" ]; then
+  echo "warm-seed: incomplete sibling — missing base mods:$missing"
+  exit 42
+fi
+`
+	}
+	if srcSavesVol != "" && dstSavesVol != "" {
+		// Copy ONLY the rendered config file (and the .local/share parent
+		// for HOME-based games). Don't copy the world data — the user wants
+		// a fresh save on this new instance, not the sibling's.
+		seedScript += `echo "warm-seed: bootstrapping saves volume from sibling"
+[ -s /src-saves/serverconfig.xml ] && cp /src-saves/serverconfig.xml /dst-saves/serverconfig.xml || true
+mkdir -p /dst-saves/.local/share 2>/dev/null || true
+# 7DTD HOME tree — entrypoint mkdir's the deeper path on boot, but creating
+# the parent here lets the install symlink resolve to a valid empty dir
+# even before the first start.
+for d in /src-saves/.local/share/7DaysToDie /src-saves/.local/share/Empyrion; do
+  if [ -d "$d" ]; then
+    mkdir -p "/dst-saves/$(echo $d | sed 's|/src-saves/||')"
+  fi
+done
+`
+	}
+	seedScript += `echo SEED_OK`
+
+	sidecarID := fmt.Sprintf("%s-warmseed", newInstanceID)
+	volumes := []runtime.VolumeSpec{
+		{Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true},
+		{Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"},
+	}
+	if srcSavesVol != "" && dstSavesVol != "" {
+		volumes = append(volumes,
+			runtime.VolumeSpec{Type: "volume", VolumeName: srcSavesVol, ContainerPath: "/src-saves", ReadOnly: true},
+			runtime.VolumeSpec{Type: "volume", VolumeName: dstSavesVol, ContainerPath: "/dst-saves"},
+		)
+	}
+	spec := runtime.InstanceSpec{
+		InstanceID:    sidecarID,
+		IsSidecar:     true,
+		Image:         "debian:12-slim",
+		Command:       []string{"bash", "-c", seedScript},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+
+	// Surface what's happening — without this the card just shows a static
+	// "installing" pulse with no console output during a multi-GB sibling copy,
+	// which looks hung.
+	d.sendInstanceState(newInstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, fmt.Sprintf("pulling game files from existing install (%s) — no SteamCMD download needed", srcID))
+	d.log.Info("warm-seed: starting sidecar", "src", srcID, "dst", newInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol, "saves_seed", srcSavesPath != "")
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		return false, fmt.Errorf("warm-seed: create sidecar: %w", err)
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		return false, fmt.Errorf("warm-seed: start sidecar: %w", err)
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	defer cancelLogs()
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.log.Info("warm-seed log", "instance_id", newInstanceID, "line", line)
+			// Mirror the sidecar's friendly progress echoes ("warm-seed: copying
+			// install volume…", "+ <mod>") onto the instance console so the
+			// operator can watch the "pulling from existing install" progress.
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: newInstanceID, Stream: "install", At: timestamppb.Now(), Line: line,
+				}},
+			})
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		return false, fmt.Errorf("warm-seed: wait sidecar: %w", err)
+	}
+	if exitCode != 0 {
+		return false, fmt.Errorf("warm-seed: sidecar exited %d", exitCode)
+	}
+	d.log.Info("warm-seed: success", "src", srcID, "dst", newInstanceID)
+	return true, nil
+}
+
+// volumeNameForPath returns the named-volume name whose container path
+// matches `path` exactly, or "" if no match. Bind mounts are skipped —
+// we only seed named volumes (host path is operator-managed).
+func volumeNameForPath(volumes []runtime.VolumeSpec, path string) string {
+	for _, v := range volumes {
+		if v.Type == "volume" && v.VolumeName != "" && v.ContainerPath == path {
+			return v.VolumeName
+		}
+	}
+	return ""
+}
+
+// findSavesVolume returns the (volumeName, containerPath) of the first
+// named-volume whose container path looks like a saves/data dir, ignoring
+// the install volume. Used to bootstrap the new instance's saves with a
+// rendered config from the sibling.
+func findSavesVolume(volumes []runtime.VolumeSpec, installPath string) (string, string) {
+	for _, v := range volumes {
+		if v.Type != "volume" || v.VolumeName == "" {
+			continue
+		}
+		if v.ContainerPath == installPath {
+			continue
+		}
+		lower := strings.ToLower(v.ContainerPath)
+		if strings.Contains(lower, "save") || strings.Contains(lower, "data") {
+			return v.VolumeName, v.ContainerPath
+		}
+	}
+	return "", ""
+}
+
+// normalizeBranch collapses a provider `beta` value into a stable warm-seed
+// matching key: the empty beta (the moving public/stable branch) becomes
+// "public"; any explicit beta (e.g. "v2.6", "latest_experimental") is returned
+// trimmed as-is. Two installs are warm-seed compatible iff their normalized
+// branches are equal.
+func normalizeBranch(beta string) string {
+	b := strings.TrimSpace(beta)
+	if b == "" {
+		return "public"
+	}
+	return b
+}
+
+// branchOrDefault returns the normalized branch for a record's Branch value,
+// treating an empty value (instances created before branch-tagging existed) as
+// the module's DEFAULT branch — UpdateProviders[0].Beta normalized. For 7DTD
+// the default is v2.6, so the pre-existing 2.6 fleet remains a valid warm-seed
+// source for a new 2.6 server. Returns "public" when the manifest has no
+// providers.
+func branchOrDefault(branch string, m *modulepkg.Manifest) string {
+	if strings.TrimSpace(branch) != "" {
+		return normalizeBranch(branch)
+	}
+	if m != nil && len(m.UpdateProviders) > 0 {
+		return normalizeBranch(m.UpdateProviders[0].Beta)
+	}
+	return "public"
+}