package dispatch import ( "context" "fmt" "strings" "time" "google.golang.org/protobuf/types/known/timestamppb" agentmodule "github.com/dbledeez/panel/agent/internal/module" "github.com/dbledeez/panel/agent/internal/runtime" modulepkg "github.com/dbledeez/panel/pkg/module" panelv1 "github.com/dbledeez/panel/proto/panel/v1" ) // tryWarmSeedFromSibling looks for another instance of the same module // already on this agent with a populated install volume; if found, copies // that volume verbatim into the new instance's install volume via a // throwaway debian sidecar. Saves operators 30+ minutes of redundant // SteamCMD downloads when standing up a second instance of a game. // // Returns (true, nil) on a successful seed — caller should skip the // updater and treat the install as already-complete. // Returns (false, nil) when there's no candidate sibling — caller falls // through to the normal updater. // Returns (false, err) on operational failure of the seed itself — // caller should also fall through (we'd rather over-download than leave // the user without a working install). // // Symlink handling: install volumes typically contain symlinks back into // the per-instance saves volume (e.g. /game/serverconfig.xml -> // /game-saves/serverconfig.xml on 7DTD, set up by the entrypoint at first // boot). Those symlinks would be DANGLING in the new instance because // the new saves volume is empty. We strip broken symlinks after copy // (`find -xtype l -delete`) so the entrypoint sees a clean state and // recreates them on its first boot. // // Saves bootstrap: if the sibling's saves volume contains a // serverconfig.xml-style file at a known location, we copy it to the // new saves volume so the entrypoint has a starting config (operator // can edit afterwards). Otherwise the entrypoint falls back to the // game's shipped default. Done per-module by the seedSavesPaths list. func (d *Dispatcher) tryWarmSeedFromSibling(ctx context.Context, newInstanceID string, manifest *modulepkg.Manifest, newDataPath string) (bool, error) { if manifest == nil || len(manifest.UpdateProviders) == 0 { return false, nil } provider := manifest.UpdateProviders[0] installPath := provider.InstallPath if installPath == "" { installPath = manifest.Runtime.Docker.BrowseableRoot } if installPath == "" { return false, nil } // Find a sibling: same agent, same module, NOT this instance, with a // container that exists (so we know the volume was at least populated // once). Prefer a stopped or running instance over a half-baked one. // // VERSION SAFETY: only reuse an install from a sibling on the SAME Steam // branch as this new instance. Copying e.g. a 3.0 (latest_experimental) // install into a server the operator created as 2.6 — or vice versa — // produces a binary/world mismatch that corrupts the save. The new // instance's record was tagged with its branch at Create (resolveCreateBranch), // before this runs. Siblings predating the feature have an empty branch and // are treated as the module's DEFAULT branch (branchOrDefault) — for 7DTD // that's v2.6, so the existing 2.6 fleet is still a valid seed source for a // new 2.6 server. d.mu.Lock() wantBranch := branchOrDefault("", manifest) if nr, ok := d.instances[newInstanceID]; ok { wantBranch = branchOrDefault(nr.Branch, manifest) } var srcID, srcDataPath string for id, rec := range d.instances { if id == newInstanceID { continue } if rec.ModuleID != manifest.ID { continue } if branchOrDefault(rec.Branch, manifest) != wantBranch { continue // different game version — not a safe seed source } // Best-effort: if we have multiple on this branch, the first one wins. // Could be improved later by picking the most-recently-updated. srcID = id srcDataPath = rec.DataPath break } d.mu.Unlock() if srcID == "" { d.log.Debug("warm-seed: no same-branch sibling instance found", "module", manifest.ID, "instance", newInstanceID, "want_branch", wantBranch) return false, nil } d.log.Info("warm-seed: matched sibling on branch", "module", manifest.ID, "instance", newInstanceID, "src", srcID, "branch", wantBranch) srcVolumes := agentmodule.ResolveVolumes(manifest, srcID, srcDataPath) dstVolumes := agentmodule.ResolveVolumes(manifest, newInstanceID, newDataPath) srcGameVol := volumeNameForPath(srcVolumes, installPath) dstGameVol := volumeNameForPath(dstVolumes, installPath) if srcGameVol == "" || dstGameVol == "" { return false, fmt.Errorf("warm-seed: install_path %q not in module volumes", installPath) } // Find a "saves-style" sibling volume to bootstrap from. Heuristic: // any non-install named volume whose container path contains "save" // or "data". Most modules have one. If none, we still seed the install // volume; the entrypoint may still need to bootstrap saves itself. srcSavesVol, srcSavesPath := findSavesVolume(srcVolumes, installPath) dstSavesVol, _ := findSavesVolume(dstVolumes, installPath) // Build the seed sidecar. debian:12-slim is small, has bash + find + // cp + mkdir, and is already cached on every panel-agent host (the // fs-helper sidecar uses it). // Why a tar pipe instead of cp -a: we want fine-grained exclusions for // per-instance state that shouldn't be inherited from the sibling. // // Mods/ split: // BASE mods (TFP_*, 0_TFP_*, Allocs_*, PrismaCore, xMarkers) are // load-bearing — TFP_Harmony is the runtime patcher, the server // won't even boot without it; Allocs is the live map renderer the // panel maps tab depends on; PrismaCore is the staff command surface // refugebotserver expects. We KEEP these. // // CUSTOM mods (AGF-VP-*, RefugeBot, AsylumRoboticInbox, anything // else) are server-specific — operators usually want a clean slate // on a new server so the panel's mod manager / their own upload // decides what's installed. We DROP these. // // steamapps/userdata, steamapps/downloading: per-Steam-account state // + half-fetched chunks. Always excluded. // // Approach: tar everything EXCEPT Mods + downloading + userdata, then // selectively copy back the base-mod entries from the sibling. // The Mods/ split + base-mod completeness gate below are 7DTD-SPECIFIC: // TFP_Harmony/Allocs/PrismaCore are load-bearing 7DTD server mods, and the // "missing base mods → exit 42" check re-triggers a full SteamCMD validate // to refetch them. Every OTHER module (palworld, soulmask, valheim, …) has // no Mods/ dir at all, so running this block against them ALWAYS hit the // completeness gate and exit-42'd — turning a valid sibling seed into a // spurious "incomplete sibling" failure. For non-7DTD modules we copy the // whole install verbatim (minus per-instance Steam state) and skip the gate. // // Root cause of the 2026-07-10 gamehost outage: Palworld + Soulmask checkouts // warm-seeded from a sibling, hit this 7DTD gate, exit-42'd, and fell through // to a fresh download — which was then canceled by a racing env-config // recreate (fixed separately in the controller). See changelog 2026-07-10. seedScript := `set -e echo "warm-seed: copying install volume contents (excl. Mods/, steamapps/userdata, steamapps/downloading)" find /dst-game -mindepth 1 -delete ( cd /src-game && tar --exclude='./Mods' --exclude='./steamapps/userdata' --exclude='./steamapps/downloading' -cf - . ) | ( cd /dst-game && tar -xf - ) ` if manifest.ID == "7dtd" { seedScript += `mkdir -p /dst-game/Mods echo "warm-seed: copying BASE mods (TFP_*, Allocs_*, PrismaCore, Xample_MarkersMod) — load-bearing for the server" for mod in /src-game/Mods/TFP_* /src-game/Mods/0_TFP_* /src-game/Mods/Allocs_* /src-game/Mods/PrismaCore /src-game/Mods/Xample_* /src-game/Mods/*Markers* /src-game/Mods/xMarkers /src-game/Mods/xmarkers; do case "$mod" in *"*"*) continue ;; esac if [ -e "$mod" ]; then cp -a "$mod" /dst-game/Mods/ echo " + $(basename $mod)" fi done ` } seedScript += `echo "warm-seed: stripping dangling symlinks (saves-volume references)" find /dst-game -xtype l -delete ` if manifest.ID == "7dtd" { seedScript += `# Completeness check (7DTD only) — fail the warmseed if any LOAD-BEARING base # mod is missing on dst. The caller falls back to a full steamcmd validate which # will re-fetch the TFP/xMarkers mods from the dedicated-server depot. missing="" for required in 0_TFP_Harmony TFP_WebServer; do [ -d "/dst-game/Mods/$required" ] || missing="$missing $required" done if [ -n "$missing" ]; then echo "warm-seed: incomplete sibling — missing base mods:$missing" exit 42 fi ` } if srcSavesVol != "" && dstSavesVol != "" { // Copy ONLY the rendered config file (and the .local/share parent // for HOME-based games). Don't copy the world data — the user wants // a fresh save on this new instance, not the sibling's. seedScript += `echo "warm-seed: bootstrapping saves volume from sibling" [ -s /src-saves/serverconfig.xml ] && cp /src-saves/serverconfig.xml /dst-saves/serverconfig.xml || true mkdir -p /dst-saves/.local/share 2>/dev/null || true # 7DTD HOME tree — entrypoint mkdir's the deeper path on boot, but creating # the parent here lets the install symlink resolve to a valid empty dir # even before the first start. for d in /src-saves/.local/share/7DaysToDie /src-saves/.local/share/Empyrion; do if [ -d "$d" ]; then mkdir -p "/dst-saves/$(echo $d | sed 's|/src-saves/||')" fi done ` } seedScript += `echo SEED_OK` sidecarID := fmt.Sprintf("%s-warmseed", newInstanceID) volumes := []runtime.VolumeSpec{ {Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true}, {Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"}, } if srcSavesVol != "" && dstSavesVol != "" { volumes = append(volumes, runtime.VolumeSpec{Type: "volume", VolumeName: srcSavesVol, ContainerPath: "/src-saves", ReadOnly: true}, runtime.VolumeSpec{Type: "volume", VolumeName: dstSavesVol, ContainerPath: "/dst-saves"}, ) } spec := runtime.InstanceSpec{ InstanceID: sidecarID, IsSidecar: true, Image: "debian:12-slim", Command: []string{"bash", "-c", seedScript}, Volumes: volumes, RestartPolicy: "no", } // Surface what's happening — without this the card just shows a static // "installing" pulse with no console output during a multi-GB sibling copy, // which looks hung. d.sendInstanceState(newInstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, fmt.Sprintf("pulling game files from existing install (%s) — no SteamCMD download needed", srcID)) d.log.Info("warm-seed: starting sidecar", "src", srcID, "dst", newInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol, "saves_seed", srcSavesPath != "") contID, err := d.runtime.Create(ctx, spec) if err != nil { return false, fmt.Errorf("warm-seed: create sidecar: %w", err) } defer func() { rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() _ = d.runtime.Remove(rmCtx, contID) }() if err := d.runtime.Start(ctx, contID); err != nil { return false, fmt.Errorf("warm-seed: start sidecar: %w", err) } logCtx, cancelLogs := context.WithCancel(ctx) defer cancelLogs() logDone := make(chan struct{}) go func() { defer close(logDone) _ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) { d.log.Info("warm-seed log", "instance_id", newInstanceID, "line", line) // Mirror the sidecar's friendly progress echoes ("warm-seed: copying // install volume…", "+ ") onto the instance console so the // operator can watch the "pulling from existing install" progress. d.sendEnv(&panelv1.AgentEnvelope{ SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{ InstanceId: newInstanceID, Stream: "install", At: timestamppb.Now(), Line: line, }}, }) }) }() exitCode, err := d.runtime.Wait(ctx, contID) cancelLogs() <-logDone if err != nil { return false, fmt.Errorf("warm-seed: wait sidecar: %w", err) } if exitCode != 0 { return false, fmt.Errorf("warm-seed: sidecar exited %d", exitCode) } d.log.Info("warm-seed: success", "src", srcID, "dst", newInstanceID) return true, nil } // volumeNameForPath returns the named-volume name whose container path // matches `path` exactly, or "" if no match. Bind mounts are skipped — // we only seed named volumes (host path is operator-managed). func volumeNameForPath(volumes []runtime.VolumeSpec, path string) string { for _, v := range volumes { if v.Type == "volume" && v.VolumeName != "" && v.ContainerPath == path { return v.VolumeName } } return "" } // findSavesVolume returns the (volumeName, containerPath) of the first // named-volume whose container path looks like a saves/data dir, ignoring // the install volume. Used to bootstrap the new instance's saves with a // rendered config from the sibling. func findSavesVolume(volumes []runtime.VolumeSpec, installPath string) (string, string) { for _, v := range volumes { if v.Type != "volume" || v.VolumeName == "" { continue } if v.ContainerPath == installPath { continue } lower := strings.ToLower(v.ContainerPath) if strings.Contains(lower, "save") || strings.Contains(lower, "data") { return v.VolumeName, v.ContainerPath } } return "", "" } // normalizeBranch collapses a provider `beta` value into a stable warm-seed // matching key: the empty beta (the moving public/stable branch) becomes // "public"; any explicit beta (e.g. "v2.6", "latest_experimental") is returned // trimmed as-is. Two installs are warm-seed compatible iff their normalized // branches are equal. func normalizeBranch(beta string) string { b := strings.TrimSpace(beta) if b == "" { return "public" } return b } // branchOrDefault returns the normalized branch for a record's Branch value, // treating an empty value (instances created before branch-tagging existed) as // the module's DEFAULT branch — UpdateProviders[0].Beta normalized. For 7DTD // the default is v2.6, so the pre-existing 2.6 fleet remains a valid warm-seed // source for a new 2.6 server. Returns "public" when the manifest has no // providers. func branchOrDefault(branch string, m *modulepkg.Manifest) string { if strings.TrimSpace(branch) != "" { return normalizeBranch(branch) } if m != nil && len(m.UpdateProviders) > 0 { return normalizeBranch(m.UpdateProviders[0].Beta) } return "public" }