panel/agent/internal/dispatch/warmseed.go

package dispatch

import (
	"context"
	"fmt"
	"strings"
	"time"

	"google.golang.org/protobuf/types/known/timestamppb"

	agentmodule "github.com/dbledeez/panel/agent/internal/module"
	"github.com/dbledeez/panel/agent/internal/runtime"
	modulepkg "github.com/dbledeez/panel/pkg/module"
	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
)

// tryWarmSeedFromSibling looks for another instance of the same module
// already on this agent with a populated install volume; if found, copies
// that volume verbatim into the new instance's install volume via a
// throwaway debian sidecar. Saves operators 30+ minutes of redundant
// SteamCMD downloads when standing up a second instance of a game.
//
// Returns (true, nil) on a successful seed — caller should skip the
// updater and treat the install as already-complete.
// Returns (false, nil) when there's no candidate sibling — caller falls
// through to the normal updater.
// Returns (false, err) on operational failure of the seed itself —
// caller should also fall through (we'd rather over-download than leave
// the user without a working install).
//
// Symlink handling: install volumes typically contain symlinks back into
// the per-instance saves volume (e.g. /game/serverconfig.xml ->
// /game-saves/serverconfig.xml on 7DTD, set up by the entrypoint at first
// boot). Those symlinks would be DANGLING in the new instance because
// the new saves volume is empty. We strip broken symlinks after copy
// (`find -xtype l -delete`) so the entrypoint sees a clean state and
// recreates them on its first boot.
//
// Saves bootstrap: if the sibling's saves volume contains a
// serverconfig.xml-style file at a known location, we copy it to the
// new saves volume so the entrypoint has a starting config (operator
// can edit afterwards). Otherwise the entrypoint falls back to the
// game's shipped default. Done per-module by the seedSavesPaths list.
func (d *Dispatcher) tryWarmSeedFromSibling(ctx context.Context, newInstanceID string, manifest *modulepkg.Manifest, newDataPath string) (bool, error) {
	if manifest == nil || len(manifest.UpdateProviders) == 0 {
		return false, nil
	}
	provider := manifest.UpdateProviders[0]
	installPath := provider.InstallPath
	if installPath == "" {
		installPath = manifest.Runtime.Docker.BrowseableRoot
	}
	if installPath == "" {
		return false, nil
	}

	// Find a sibling: same agent, same module, NOT this instance, with a
	// container that exists (so we know the volume was at least populated
	// once). Prefer a stopped or running instance over a half-baked one.
	//
	// VERSION SAFETY: only reuse an install from a sibling on the SAME Steam
	// branch as this new instance. Copying e.g. a 3.0 (latest_experimental)
	// install into a server the operator created as 2.6 — or vice versa —
	// produces a binary/world mismatch that corrupts the save. The new
	// instance's record was tagged with its branch at Create (resolveCreateBranch),
	// before this runs. Siblings predating the feature have an empty branch and
	// are treated as the module's DEFAULT branch (branchOrDefault) — for 7DTD
	// that's v2.6, so the existing 2.6 fleet is still a valid seed source for a
	// new 2.6 server.
	d.mu.Lock()
	wantBranch := branchOrDefault("", manifest)
	if nr, ok := d.instances[newInstanceID]; ok {
		wantBranch = branchOrDefault(nr.Branch, manifest)
	}
	var srcID, srcDataPath string
	for id, rec := range d.instances {
		if id == newInstanceID {
			continue
		}
		if rec.ModuleID != manifest.ID {
			continue
		}
		if branchOrDefault(rec.Branch, manifest) != wantBranch {
			continue // different game version — not a safe seed source
		}
		// Best-effort: if we have multiple on this branch, the first one wins.
		// Could be improved later by picking the most-recently-updated.
		srcID = id
		srcDataPath = rec.DataPath
		break
	}
	d.mu.Unlock()
	if srcID == "" {
		d.log.Debug("warm-seed: no same-branch sibling instance found", "module", manifest.ID, "instance", newInstanceID, "want_branch", wantBranch)
		return false, nil
	}
	d.log.Info("warm-seed: matched sibling on branch", "module", manifest.ID, "instance", newInstanceID, "src", srcID, "branch", wantBranch)

	srcVolumes := agentmodule.ResolveVolumes(manifest, srcID, srcDataPath)
	dstVolumes := agentmodule.ResolveVolumes(manifest, newInstanceID, newDataPath)

	srcGameVol := volumeNameForPath(srcVolumes, installPath)
	dstGameVol := volumeNameForPath(dstVolumes, installPath)
	if srcGameVol == "" || dstGameVol == "" {
		return false, fmt.Errorf("warm-seed: install_path %q not in module volumes", installPath)
	}

	// Find a "saves-style" sibling volume to bootstrap from. Heuristic:
	// any non-install named volume whose container path contains "save"
	// or "data". Most modules have one. If none, we still seed the install
	// volume; the entrypoint may still need to bootstrap saves itself.
	srcSavesVol, srcSavesPath := findSavesVolume(srcVolumes, installPath)
	dstSavesVol, _ := findSavesVolume(dstVolumes, installPath)

	// Build the seed sidecar. debian:12-slim is small, has bash + find +
	// cp + mkdir, and is already cached on every panel-agent host (the
	// fs-helper sidecar uses it).
	// Why a tar pipe instead of cp -a: we want fine-grained exclusions for
	// per-instance state that shouldn't be inherited from the sibling.
	//
	// Mods/ split:
	//   BASE mods (TFP_*, 0_TFP_*, Allocs_*, PrismaCore, xMarkers) are
	//   load-bearing — TFP_Harmony is the runtime patcher, the server
	//   won't even boot without it; Allocs is the live map renderer the
	//   panel maps tab depends on; PrismaCore is the staff command surface
	//   refugebotserver expects. We KEEP these.
	//
	//   CUSTOM mods (AGF-VP-*, RefugeBot, AsylumRoboticInbox, anything
	//   else) are server-specific — operators usually want a clean slate
	//   on a new server so the panel's mod manager / their own upload
	//   decides what's installed. We DROP these.
	//
	// steamapps/userdata, steamapps/downloading: per-Steam-account state
	// + half-fetched chunks. Always excluded.
	//
	// Approach: tar everything EXCEPT Mods + downloading + userdata, then
	// selectively copy back the base-mod entries from the sibling.
	// The Mods/ split + base-mod completeness gate below are 7DTD-SPECIFIC:
	// TFP_Harmony/Allocs/PrismaCore are load-bearing 7DTD server mods, and the
	// "missing base mods → exit 42" check re-triggers a full SteamCMD validate
	// to refetch them. Every OTHER module (palworld, soulmask, valheim, …) has
	// no Mods/ dir at all, so running this block against them ALWAYS hit the
	// completeness gate and exit-42'd — turning a valid sibling seed into a
	// spurious "incomplete sibling" failure. For non-7DTD modules we copy the
	// whole install verbatim (minus per-instance Steam state) and skip the gate.
	//
	// Root cause of the 2026-07-10 gamehost outage: Palworld + Soulmask checkouts
	// warm-seeded from a sibling, hit this 7DTD gate, exit-42'd, and fell through
	// to a fresh download — which was then canceled by a racing env-config
	// recreate (fixed separately in the controller). See changelog 2026-07-10.
	seedScript := `set -e
echo "warm-seed: copying install volume contents (excl. Mods/, steamapps/userdata, steamapps/downloading)"
find /dst-game -mindepth 1 -delete
( cd /src-game && tar --exclude='./Mods' --exclude='./steamapps/userdata' --exclude='./steamapps/downloading' -cf - . ) | ( cd /dst-game && tar -xf - )
`
	if manifest.ID == "7dtd" {
		seedScript += `mkdir -p /dst-game/Mods
echo "warm-seed: copying BASE mods (TFP_*, Allocs_*, PrismaCore, Xample_MarkersMod) — load-bearing for the server"
for mod in /src-game/Mods/TFP_* /src-game/Mods/0_TFP_* /src-game/Mods/Allocs_* /src-game/Mods/PrismaCore /src-game/Mods/Xample_* /src-game/Mods/*Markers* /src-game/Mods/xMarkers /src-game/Mods/xmarkers; do
  case "$mod" in *"*"*) continue ;; esac
  if [ -e "$mod" ]; then
    cp -a "$mod" /dst-game/Mods/
    echo "  + $(basename $mod)"
  fi
done
`
	}
	seedScript += `echo "warm-seed: stripping dangling symlinks (saves-volume references)"
find /dst-game -xtype l -delete
`
	if manifest.ID == "7dtd" {
		seedScript += `# Completeness check (7DTD only) — fail the warmseed if any LOAD-BEARING base
# mod is missing on dst. The caller falls back to a full steamcmd validate which
# will re-fetch the TFP/xMarkers mods from the dedicated-server depot.
missing=""
for required in 0_TFP_Harmony TFP_WebServer; do
  [ -d "/dst-game/Mods/$required" ] || missing="$missing $required"
done
if [ -n "$missing" ]; then
  echo "warm-seed: incomplete sibling — missing base mods:$missing"
  exit 42
fi
`
	}
	if srcSavesVol != "" && dstSavesVol != "" {
		// Copy ONLY the rendered config file (and the .local/share parent
		// for HOME-based games). Don't copy the world data — the user wants
		// a fresh save on this new instance, not the sibling's.
		seedScript += `echo "warm-seed: bootstrapping saves volume from sibling"
[ -s /src-saves/serverconfig.xml ] && cp /src-saves/serverconfig.xml /dst-saves/serverconfig.xml || true
mkdir -p /dst-saves/.local/share 2>/dev/null || true
# 7DTD HOME tree — entrypoint mkdir's the deeper path on boot, but creating
# the parent here lets the install symlink resolve to a valid empty dir
# even before the first start.
for d in /src-saves/.local/share/7DaysToDie /src-saves/.local/share/Empyrion; do
  if [ -d "$d" ]; then
    mkdir -p "/dst-saves/$(echo $d | sed 's|/src-saves/||')"
  fi
done
`
	}
	seedScript += `echo SEED_OK`

	sidecarID := fmt.Sprintf("%s-warmseed", newInstanceID)
	volumes := []runtime.VolumeSpec{
		{Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true},
		{Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"},
	}
	if srcSavesVol != "" && dstSavesVol != "" {
		volumes = append(volumes,
			runtime.VolumeSpec{Type: "volume", VolumeName: srcSavesVol, ContainerPath: "/src-saves", ReadOnly: true},
			runtime.VolumeSpec{Type: "volume", VolumeName: dstSavesVol, ContainerPath: "/dst-saves"},
		)
	}
	spec := runtime.InstanceSpec{
		InstanceID:    sidecarID,
		IsSidecar:     true,
		Image:         "debian:12-slim",
		Command:       []string{"bash", "-c", seedScript},
		Volumes:       volumes,
		RestartPolicy: "no",
	}

	// Surface what's happening — without this the card just shows a static
	// "installing" pulse with no console output during a multi-GB sibling copy,
	// which looks hung.
	d.sendInstanceState(newInstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, fmt.Sprintf("pulling game files from existing install (%s) — no SteamCMD download needed", srcID))
	d.log.Info("warm-seed: starting sidecar", "src", srcID, "dst", newInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol, "saves_seed", srcSavesPath != "")
	contID, err := d.runtime.Create(ctx, spec)
	if err != nil {
		return false, fmt.Errorf("warm-seed: create sidecar: %w", err)
	}
	defer func() {
		rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
		defer cancel()
		_ = d.runtime.Remove(rmCtx, contID)
	}()
	if err := d.runtime.Start(ctx, contID); err != nil {
		return false, fmt.Errorf("warm-seed: start sidecar: %w", err)
	}
	logCtx, cancelLogs := context.WithCancel(ctx)
	defer cancelLogs()
	logDone := make(chan struct{})
	go func() {
		defer close(logDone)
		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
			d.log.Info("warm-seed log", "instance_id", newInstanceID, "line", line)
			// Mirror the sidecar's friendly progress echoes ("warm-seed: copying
			// install volume…", "+ <mod>") onto the instance console so the
			// operator can watch the "pulling from existing install" progress.
			d.sendEnv(&panelv1.AgentEnvelope{
				SentAt: timestamppb.Now(),
				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
					InstanceId: newInstanceID, Stream: "install", At: timestamppb.Now(), Line: line,
				}},
			})
		})
	}()
	exitCode, err := d.runtime.Wait(ctx, contID)
	cancelLogs()
	<-logDone
	if err != nil {
		return false, fmt.Errorf("warm-seed: wait sidecar: %w", err)
	}
	if exitCode != 0 {
		return false, fmt.Errorf("warm-seed: sidecar exited %d", exitCode)
	}
	d.log.Info("warm-seed: success", "src", srcID, "dst", newInstanceID)
	return true, nil
}

// volumeNameForPath returns the named-volume name whose container path
// matches `path` exactly, or "" if no match. Bind mounts are skipped —
// we only seed named volumes (host path is operator-managed).
func volumeNameForPath(volumes []runtime.VolumeSpec, path string) string {
	for _, v := range volumes {
		if v.Type == "volume" && v.VolumeName != "" && v.ContainerPath == path {
			return v.VolumeName
		}
	}
	return ""
}

// findSavesVolume returns the (volumeName, containerPath) of the first
// named-volume whose container path looks like a saves/data dir, ignoring
// the install volume. Used to bootstrap the new instance's saves with a
// rendered config from the sibling.
func findSavesVolume(volumes []runtime.VolumeSpec, installPath string) (string, string) {
	for _, v := range volumes {
		if v.Type != "volume" || v.VolumeName == "" {
			continue
		}
		if v.ContainerPath == installPath {
			continue
		}
		lower := strings.ToLower(v.ContainerPath)
		if strings.Contains(lower, "save") || strings.Contains(lower, "data") {
			return v.VolumeName, v.ContainerPath
		}
	}
	return "", ""
}

// normalizeBranch collapses a provider `beta` value into a stable warm-seed
// matching key: the empty beta (the moving public/stable branch) becomes
// "public"; any explicit beta (e.g. "v2.6", "latest_experimental") is returned
// trimmed as-is. Two installs are warm-seed compatible iff their normalized
// branches are equal.
func normalizeBranch(beta string) string {
	b := strings.TrimSpace(beta)
	if b == "" {
		return "public"
	}
	return b
}

// branchOrDefault returns the normalized branch for a record's Branch value,
// treating an empty value (instances created before branch-tagging existed) as
// the module's DEFAULT branch — UpdateProviders[0].Beta normalized. For 7DTD
// the default is v2.6, so the pre-existing 2.6 fleet remains a valid warm-seed
// source for a new 2.6 server. Returns "public" when the manifest has no
// providers.
func branchOrDefault(branch string, m *modulepkg.Manifest) string {
	if strings.TrimSpace(branch) != "" {
		return normalizeBranch(branch)
	}
	if m != nil && len(m.UpdateProviders) > 0 {
		return normalizeBranch(m.UpdateProviders[0].Beta)
	}
	return "public"
}