2092 lines
83 KiB
Go
2092 lines
83 KiB
Go
// Package dispatch ties together the Target-side components: it receives
|
||
// ControllerEnvelope messages off the gRPC stream, routes them to the
|
||
// module registry and the active Runtime, and emits AgentEnvelope state
|
||
// updates back to the Controller.
|
||
//
|
||
// Survivability:
|
||
//
|
||
// - On successful Create, dispatch writes a sidecar JSON in metaDir so
|
||
// the agent can re-attach to the running container after a restart.
|
||
// - On Stop, the sidecar is deleted.
|
||
// - On agent startup, Rehydrate() scans sidecars, verifies each
|
||
// container still exists, and rebuilds the in-memory instance map.
|
||
// - On every (re)connect, SetSender + Announce re-plumb the upward
|
||
// pipe and re-send current state so the controller's DB stays honest.
|
||
package dispatch
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"log/slog"
|
||
"net"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"google.golang.org/protobuf/types/known/timestamppb"
|
||
|
||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||
"github.com/dbledeez/panel/agent/internal/rcon"
|
||
"github.com/dbledeez/panel/agent/internal/state"
|
||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||
)
|
||
|
||
// Sender is the subset of the gRPC stream we use to send events upstream.
|
||
type Sender interface {
|
||
Send(*panelv1.AgentEnvelope) error
|
||
}
|
||
|
||
// Dispatcher wires the gRPC stream to the runtime and module registry.
|
||
type Dispatcher struct {
|
||
log *slog.Logger
|
||
modules *modulepkg.Registry
|
||
runtime runtime.Runtime
|
||
dataRoot string
|
||
metaDir string
|
||
backupDir string
|
||
|
||
sendMu sync.Mutex
|
||
send Sender
|
||
// sendCh decouples envelope producers (handler goroutines, trackers,
|
||
// log streams) from the actual gRPC Send. A single drainer goroutine
|
||
// owns the stream's Send so concurrent producers never race on it
|
||
// (grpc ClientStream.Send is NOT safe for concurrent use) and never
|
||
// block the dispatch path when the controller is slow to drain. High-
|
||
// frequency telemetry (logs/stats/app-state/player/heartbeat) is
|
||
// DROPPED when the buffer is full rather than blocking — losing a log
|
||
// line is fine (the controller's durable state comes from Announce +
|
||
// InstanceState, not replayed history); freezing the agent is not.
|
||
// This is what stopped a chatty game (VEIN) from wedging the whole
|
||
// agent fleet-wide via a backpressured synchronous Send.
|
||
sendCh chan *panelv1.AgentEnvelope
|
||
sendClosed chan struct{}
|
||
|
||
mu sync.Mutex
|
||
instances map[string]*instanceRecord
|
||
// fsHelpers holds one long-lived alpine sidecar per stopped instance
|
||
// whose Files tab has been opened. Lets operators browse/edit files
|
||
// on a stopped container. Reaped after ~5 min idle; torn down when
|
||
// the main instance starts (frees up volumes for the game).
|
||
fsHelpers map[string]*fsHelper
|
||
// updaters tracks the active updater goroutine per instance so a
|
||
// stop / delete can cancel an in-flight steamcmd download. Without
|
||
// this the sidecar runs to completion (or its 30-min timeout)
|
||
// holding the volume mount, blocking the main delete from purging
|
||
// — which the operator sees as "stuck downloading" with the parent
|
||
// already gone. Each entry holds a unique pointer so the
|
||
// unregister path can compare against the slot it installed.
|
||
updaters map[string]*updaterSlot
|
||
|
||
// uploads holds in-progress chunked uploads keyed by upload_id.
|
||
// Each entry owns a temp file the chunk handler appends to; on the
|
||
// final chunk the file is shipped to the container, closed and
|
||
// unlinked. See files_chunked.go.
|
||
uploads map[string]*uploadSession
|
||
}
|
||
|
||
// updaterSlot pairs the cancel func with a unique pointer identity so
|
||
// concurrent unregisters from racing updater goroutines don't clobber
|
||
// each other. Comparing pointers is cheap and correct.
|
||
type updaterSlot struct {
|
||
cancel context.CancelFunc
|
||
}
|
||
|
||
// fsHelper tracks one file-ops sidecar: alpine mounting the instance's
|
||
// volumes read/write, running `sleep infinity`.
|
||
type fsHelper struct {
|
||
containerID string
|
||
lastAccess time.Time
|
||
}
|
||
|
||
type instanceRecord struct {
|
||
InstanceID string
|
||
ContainerID string
|
||
ModuleID string
|
||
// Branch is the normalized Steam branch this instance was installed from
|
||
// (e.g. "v2.6", "latest_experimental", "public"), derived at Create from
|
||
// the chosen update_provider. Used by warm-seed so a new instance only
|
||
// reuses an install from a sibling on the SAME branch — copying a 3.0
|
||
// (experimental) install into a server meant to run 2.6 would corrupt it.
|
||
// Empty on instances created before this feature (treated as the module
|
||
// default branch by branchOrDefault).
|
||
Branch string
|
||
DataPath string
|
||
// BrowseableRoot is the default (first) file-manager root —
|
||
// container-absolute. BrowseableRoots is the full list; when the
|
||
// module exposes more than one view (7DTD: saves + game files),
|
||
// safeJoinContainer accepts any of them. Always contains at least
|
||
// BrowseableRoot.
|
||
BrowseableRoot string
|
||
BrowseableRoots []modulepkg.BrowseableRoot
|
||
LogCancel context.CancelFunc
|
||
|
||
// ExitWatchCancel single-flights the watchExit goroutine per container
|
||
// lifetime. Before this guard, every activate() call — handleStart,
|
||
// Announce on EVERY controller reconnect, the post-warm-seed autostart —
|
||
// spawned a fresh watchExit, each parked in a ctx-unbounded
|
||
// ContainerWait: leaked goroutines + leaked docker connections (the same
|
||
// failure shape as the ExecCapture leak) + duplicate crashed/exited
|
||
// emissions when the container finally died. Guarded by d.mu like the
|
||
// other cancel fields.
|
||
ExitWatchCancel context.CancelFunc
|
||
|
||
// RCON wire-up, resolved at Create time, consumed at Start time.
|
||
RCONAddr string
|
||
RCONPassword string
|
||
|
||
// State tracker and its cancel, populated on Start.
|
||
Tracker *state.Tracker
|
||
TrackerCancel context.CancelFunc
|
||
TrackerDone chan struct{}
|
||
|
||
// stopping suppresses watchExit's CRASHED emission during graceful stop.
|
||
stopping atomic.Bool
|
||
|
||
// memGuardWarned latches true the first time the 45 GB warn path
|
||
// fires on this container. Both paths (warn + hard) check this to
|
||
// avoid stacking SIGKILL on top of an in-flight graceful save.
|
||
memGuardWarned atomic.Bool
|
||
|
||
// lastStatus tracks the most recent InstanceStatus the agent sent
|
||
// upstream for this instance. Read by EmitAppState to promote a stale
|
||
// CRASHED status back to RUNNING when state-source polls succeed —
|
||
// the docker-restart-policy auto-recovery flip in watchExit doesn't
|
||
// fire when a server segfaults during world-gen but stabilizes on
|
||
// retry, leaving the panel stuck on CRASHED until manual stop+start.
|
||
lastStatus atomic.Int32
|
||
|
||
// hangGuard is the per-instance state for the ark-sa hang-detection
|
||
// guardrail. Embedded by value so its zero value is usable. Methods
|
||
// on Dispatcher (arkHangGuardOn{LogLine,RconResult}, arkHangGuardRestart)
|
||
// own the field; nothing outside the guard touches it. See hangguard.go.
|
||
hangGuard hangGuardState
|
||
|
||
// provisionGuard single-flights the 7dtd cluster auto-provision watcher
|
||
// (one per instance at a time). See decoremap_provision.go.
|
||
provisionGuard atomic.Bool
|
||
}
|
||
|
||
// New constructs a dispatcher. The caller sets the sender via SetSender
|
||
// after the initial handshake (and on each reconnect).
|
||
func New(log *slog.Logger, modules *modulepkg.Registry, rt runtime.Runtime, dataRoot, metaDir, backupDir string) *Dispatcher {
|
||
d := &Dispatcher{
|
||
log: log,
|
||
modules: modules,
|
||
runtime: rt,
|
||
dataRoot: dataRoot,
|
||
metaDir: metaDir,
|
||
backupDir: backupDir,
|
||
instances: map[string]*instanceRecord{},
|
||
fsHelpers: map[string]*fsHelper{},
|
||
updaters: map[string]*updaterSlot{},
|
||
uploads: map[string]*uploadSession{},
|
||
// Buffer sized to absorb a burst of telemetry from a chatty game
|
||
// while the controller catches up. Beyond this, droppable envelopes
|
||
// are discarded (see sendEnv). 4096 is generous for ~dozens of
|
||
// instances; a full buffer means the controller is badly behind and
|
||
// dropping stale logs/stats is the right call.
|
||
sendCh: make(chan *panelv1.AgentEnvelope, 4096),
|
||
}
|
||
go d.fsHelperReaper()
|
||
go d.uploadReaper()
|
||
go d.sendLoop()
|
||
return d
|
||
}
|
||
|
||
// sendLoop is the SOLE owner of stream.Send. It drains sendCh and writes each
|
||
// envelope to the current sender. Running on one goroutine guarantees Sends
|
||
// are serialized (grpc ClientStream.Send is not safe for concurrent use) and
|
||
// that a slow/backpressured controller can never block a producer past the
|
||
// buffer. Envelopes enqueued while disconnected (send == nil) are dropped —
|
||
// the controller rebuilds durable state from Announce on reconnect.
|
||
func (d *Dispatcher) sendLoop() {
|
||
for env := range d.sendCh {
|
||
d.sendMu.Lock()
|
||
s := d.send
|
||
d.sendMu.Unlock()
|
||
if s == nil {
|
||
continue
|
||
}
|
||
_ = s.Send(env)
|
||
}
|
||
}
|
||
|
||
// Modules exposes the registry so main() can summarize it into the
|
||
// AgentHello envelope. Read-only from the caller's perspective.
|
||
func (d *Dispatcher) Modules() *modulepkg.Registry { return d.modules }
|
||
|
||
// SetSender installs (or replaces) the upstream sender after a
|
||
// (re)connect. Safe to call from any goroutine.
|
||
func (d *Dispatcher) SetSender(s Sender) {
|
||
d.sendMu.Lock()
|
||
d.send = s
|
||
d.sendMu.Unlock()
|
||
}
|
||
|
||
// sendEnv enqueues an envelope for the sendLoop drainer instead of calling
|
||
// stream.Send inline. This keeps producers (handler goroutines, trackers, log
|
||
// streams) off the gRPC send path entirely: they never race on Send and never
|
||
// block when the controller is slow to drain.
|
||
//
|
||
// Backpressure policy: high-frequency telemetry (logs / stats / app-state /
|
||
// player / heartbeat) is DROPPED if the buffer is full — losing a log line is
|
||
// harmless, wedging the agent is not. Low-frequency CRITICAL envelopes
|
||
// (command replies, instance-state, FS/backup/etc. results) are never dropped:
|
||
// if the buffer is somehow full they block briefly until space frees. Because
|
||
// critical envelopes are rare, this can't cause the flood-driven wedge that a
|
||
// chatty game's telemetry would.
|
||
//
|
||
// Messages emitted while disconnected (send == nil) are still effectively
|
||
// dropped by sendLoop — intentional, the controller rebuilds durable state
|
||
// from Announce() on reconnect, not replayed history.
|
||
func (d *Dispatcher) sendEnv(env *panelv1.AgentEnvelope) {
|
||
if isDroppableEnvelope(env) {
|
||
select {
|
||
case d.sendCh <- env:
|
||
default:
|
||
// Buffer full — drop this telemetry frame rather than block.
|
||
}
|
||
return
|
||
}
|
||
// Critical envelope: enqueue, blocking only if the buffer is full (rare).
|
||
d.sendCh <- env
|
||
}
|
||
|
||
// SendHeartbeat enqueues a heartbeat envelope on the droppable send queue.
|
||
// Routing heartbeats through sendLoop keeps it the SOLE owner of stream.Send:
|
||
// previously main.go's session ticker called stream.Send directly, racing the
|
||
// drainer goroutine (grpc ClientStream.Send is NOT safe for concurrent use).
|
||
// Heartbeats are droppable telemetry — a skipped beat under backpressure is
|
||
// harmless. Note the tradeoff: a broken stream is now detected by the recv
|
||
// loop (stream.Recv errors when the transport dies), not by a failed
|
||
// heartbeat send.
|
||
func (d *Dispatcher) SendHeartbeat(t time.Time) {
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
SentAt: timestamppb.New(t),
|
||
Payload: &panelv1.AgentEnvelope_Heartbeat{
|
||
Heartbeat: &panelv1.Heartbeat{At: timestamppb.New(t)},
|
||
},
|
||
})
|
||
}
|
||
|
||
// isDroppableEnvelope reports whether an envelope is high-frequency telemetry
|
||
// that's safe to discard under backpressure. Everything not listed here is
|
||
// treated as critical (command replies, instance state, FS/backup results,
|
||
// hello/announce) and is never dropped.
|
||
func isDroppableEnvelope(env *panelv1.AgentEnvelope) bool {
|
||
switch env.GetPayload().(type) {
|
||
case *panelv1.AgentEnvelope_Log,
|
||
*panelv1.AgentEnvelope_InstanceStats,
|
||
*panelv1.AgentEnvelope_AppState,
|
||
*panelv1.AgentEnvelope_Player,
|
||
*panelv1.AgentEnvelope_Heartbeat:
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// Handle routes a single inbound envelope.
|
||
func (d *Dispatcher) Handle(ctx context.Context, env *panelv1.ControllerEnvelope) {
|
||
switch p := env.Payload.(type) {
|
||
case *panelv1.ControllerEnvelope_Ping:
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
CorrelationId: env.CorrelationId,
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_Pong{Pong: &panelv1.Pong{Nonce: p.Ping.Nonce}},
|
||
})
|
||
case *panelv1.ControllerEnvelope_InstanceCreate:
|
||
d.handleCreate(ctx, env.CorrelationId, p.InstanceCreate)
|
||
case *panelv1.ControllerEnvelope_InstanceStart:
|
||
d.handleStart(ctx, env.CorrelationId, p.InstanceStart)
|
||
case *panelv1.ControllerEnvelope_InstanceStop:
|
||
d.handleStop(ctx, env.CorrelationId, p.InstanceStop)
|
||
case *panelv1.ControllerEnvelope_InstanceDelete:
|
||
d.handleDelete(ctx, env.CorrelationId, p.InstanceDelete)
|
||
case *panelv1.ControllerEnvelope_Backup:
|
||
d.handleBackup(ctx, env.CorrelationId, p.Backup)
|
||
case *panelv1.ControllerEnvelope_Restore:
|
||
d.handleRestore(ctx, env.CorrelationId, p.Restore)
|
||
case *panelv1.ControllerEnvelope_BackupListEntries:
|
||
d.handleBackupListEntries(env.CorrelationId, p.BackupListEntries)
|
||
case *panelv1.ControllerEnvelope_BackupReadFile:
|
||
d.handleBackupReadFile(env.CorrelationId, p.BackupReadFile)
|
||
case *panelv1.ControllerEnvelope_Exec:
|
||
// No exec implementation yet — tell the caller instead of logging
|
||
// and silently succeeding (the controller's Await would otherwise
|
||
// hang until timeout with no signal why).
|
||
d.log.Warn("exec requested but not implemented", "instance_id", p.Exec.InstanceId)
|
||
d.replyError(env.CorrelationId, p.Exec.InstanceId, "exec_not_implemented", "exec is not implemented on this agent")
|
||
case *panelv1.ControllerEnvelope_Rcon:
|
||
d.handleRCON(ctx, env.CorrelationId, p.Rcon)
|
||
case *panelv1.ControllerEnvelope_FsList:
|
||
d.handleFsList(env.CorrelationId, p.FsList)
|
||
case *panelv1.ControllerEnvelope_FsRead:
|
||
d.handleFsRead(env.CorrelationId, p.FsRead)
|
||
case *panelv1.ControllerEnvelope_FsWrite:
|
||
d.handleFsWrite(env.CorrelationId, p.FsWrite)
|
||
case *panelv1.ControllerEnvelope_FsDelete:
|
||
d.handleFsDelete(env.CorrelationId, p.FsDelete)
|
||
case *panelv1.ControllerEnvelope_FsSymlink:
|
||
d.handleFsSymlink(env.CorrelationId, p.FsSymlink)
|
||
case *panelv1.ControllerEnvelope_FsExtract:
|
||
d.handleFsExtract(env.CorrelationId, p.FsExtract)
|
||
case *panelv1.ControllerEnvelope_FsCompress:
|
||
d.handleFsCompress(env.CorrelationId, p.FsCompress)
|
||
case *panelv1.ControllerEnvelope_FsRename:
|
||
d.handleFsRename(env.CorrelationId, p.FsRename)
|
||
case *panelv1.ControllerEnvelope_FsWriteChunk:
|
||
d.handleFsWriteChunk(env.CorrelationId, p.FsWriteChunk)
|
||
case *panelv1.ControllerEnvelope_ArkSaveRestore:
|
||
d.handleArkSaveRestore(env.CorrelationId, p.ArkSaveRestore)
|
||
case *panelv1.ControllerEnvelope_DayzModInstall:
|
||
d.handleDayzModInstall(env.CorrelationId, p.DayzModInstall)
|
||
case *panelv1.ControllerEnvelope_DayzModUninstall:
|
||
d.handleDayzModUninstall(env.CorrelationId, p.DayzModUninstall)
|
||
case *panelv1.ControllerEnvelope_Update:
|
||
d.handleUpdate(env.CorrelationId, p.Update)
|
||
case *panelv1.ControllerEnvelope_EmpyrionScenarioInstall:
|
||
d.handleEmpyrionScenarioInstall(env.CorrelationId, p.EmpyrionScenarioInstall)
|
||
case *panelv1.ControllerEnvelope_EmpyrionDiscoveries:
|
||
d.handleEmpyrionDiscoveries(env.CorrelationId, p.EmpyrionDiscoveries)
|
||
case *panelv1.ControllerEnvelope_EmpyrionPlayerSummary:
|
||
d.handleEmpyrionPlayerSummary(env.CorrelationId, p.EmpyrionPlayerSummary)
|
||
case *panelv1.ControllerEnvelope_RegionScan:
|
||
d.handleRegionScan(ctx, env.CorrelationId, p.RegionScan)
|
||
case *panelv1.ControllerEnvelope_RegionHeal:
|
||
d.handleRegionHeal(ctx, env.CorrelationId, p.RegionHeal)
|
||
case *panelv1.ControllerEnvelope_InstanceRenderConfig:
|
||
d.handleRenderConfig(ctx, env.CorrelationId, p.InstanceRenderConfig)
|
||
case *panelv1.ControllerEnvelope_SeedMods:
|
||
d.handleSeedMods(ctx, env.CorrelationId, p.SeedMods)
|
||
case *panelv1.ControllerEnvelope_UpdateCheck:
|
||
d.handleUpdateCheck(env.CorrelationId, p.UpdateCheck)
|
||
default:
|
||
d.log.Debug("unhandled controller message", "type", fmt.Sprintf("%T", env.Payload))
|
||
}
|
||
}
|
||
|
||
// Shutdown winds down agent-internal goroutines on SIGINT/SIGTERM but
|
||
// LEAVES the game containers running. A `systemctl restart panel-agent`
|
||
// shouldn't stop every ARK / 7DTD / V Rising the operator has up — the
|
||
// agent is a control-plane process; the game containers are workloads
|
||
// that should survive control-plane restarts the same way they survive
|
||
// gRPC blips. Earlier behaviour (Stop every instance) caused half a
|
||
// dozen game containers to drop out from under players any time the
|
||
// agent binary was redeployed.
|
||
//
|
||
// What we still tear down: state trackers + log-stream pumps, since
|
||
// those are agent-side goroutines holding open RCON dials and Docker
|
||
// log subscriptions that the next agent process will re-establish on
|
||
// rehydrate. The Docker container itself stays untouched.
|
||
func (d *Dispatcher) Shutdown(_ context.Context) {
|
||
// Snapshot the cancel funcs under d.mu (they're mutated under the same
|
||
// mutex by activate/handleStop), then invoke them outside the lock.
|
||
d.mu.Lock()
|
||
cancels := make([]context.CancelFunc, 0, len(d.instances)*3)
|
||
n := 0
|
||
for _, r := range d.instances {
|
||
n++
|
||
for _, c := range []context.CancelFunc{r.TrackerCancel, r.LogCancel, r.ExitWatchCancel} {
|
||
if c != nil {
|
||
cancels = append(cancels, c)
|
||
}
|
||
}
|
||
r.TrackerCancel = nil
|
||
r.LogCancel = nil
|
||
r.ExitWatchCancel = nil
|
||
}
|
||
d.mu.Unlock()
|
||
for _, c := range cancels {
|
||
c()
|
||
}
|
||
d.log.Info("agent shutdown — game containers left running for next agent rehydrate", "instance_count", n)
|
||
}
|
||
|
||
// Rehydrate loads each sidecar in metaDir, verifies the underlying
|
||
// container still exists, and rebuilds the in-memory instance record.
|
||
// Stale sidecars (container gone) are deleted. Intended for agent cold
|
||
// start. Activation (log stream + tracker) is deferred to Announce so
|
||
// the sender is in place before we start emitting events.
|
||
func (d *Dispatcher) Rehydrate(ctx context.Context) error {
|
||
metas, err := d.loadAllMeta()
|
||
if err != nil {
|
||
return fmt.Errorf("load meta: %w", err)
|
||
}
|
||
haveMeta := make(map[string]bool, len(metas))
|
||
for _, m := range metas {
|
||
haveMeta[m.InstanceID] = true
|
||
}
|
||
for _, m := range metas {
|
||
state, err := d.runtime.InspectByName(ctx, "panel-"+m.InstanceID)
|
||
if err != nil {
|
||
d.log.Warn("rehydrate: container gone, removing sidecar", "instance_id", m.InstanceID, "err", err)
|
||
d.deleteMeta(m.InstanceID)
|
||
continue
|
||
}
|
||
// If the sidecar predates this feature, BrowseableRoot is empty.
|
||
// Fill it from the current manifest so rehydrated instances gain
|
||
// container-path file ops without needing recreate.
|
||
browseable := m.BrowseableRoot
|
||
var roots []modulepkg.BrowseableRoot
|
||
if mf, ok := d.modules.Get(m.ModuleID); ok {
|
||
// Always re-read roots from current manifest — lets operators
|
||
// add new browseable_roots entries and have existing instances
|
||
// pick them up on next agent restart without a container rebuild.
|
||
roots = resolveBrowseableRoots(mf)
|
||
if browseable == "" {
|
||
browseable = resolveBrowseableRoot(mf)
|
||
}
|
||
}
|
||
d.mu.Lock()
|
||
d.instances[m.InstanceID] = &instanceRecord{
|
||
InstanceID: m.InstanceID,
|
||
ContainerID: state.ContainerID,
|
||
ModuleID: m.ModuleID,
|
||
Branch: m.Branch,
|
||
DataPath: m.DataPath,
|
||
BrowseableRoot: browseable,
|
||
BrowseableRoots: roots,
|
||
RCONAddr: m.RCONAddr,
|
||
RCONPassword: m.RCONPassword,
|
||
}
|
||
d.mu.Unlock()
|
||
d.log.Info("rehydrated instance",
|
||
"instance_id", m.InstanceID,
|
||
"module_id", m.ModuleID,
|
||
"container_status", state.Status,
|
||
)
|
||
// Reap orphan sidecar containers from a previous agent crash.
|
||
// steamcmd/backup/wipe sidecars are usually torn down by their
|
||
// goroutine's defer — but if the agent dies mid-run, the defer
|
||
// never fires and the sidecar lingers. It then holds the
|
||
// instance's volumes "in use," blocking future delete-with-purge.
|
||
//
|
||
// We only sweep EXITED containers here; running ones might be
|
||
// legitimately in use (e.g. an fs-helper sidecar an operator
|
||
// kept open). Force-removed exited orphans are safe — they
|
||
// already finished whatever they were doing.
|
||
if dr, ok := d.runtime.(*runtime.DockerRuntime); ok {
|
||
holders, err := dr.ListContainersHoldingInstanceVolumes(ctx, m.InstanceID)
|
||
if err == nil {
|
||
for _, hid := range holders {
|
||
if hid == state.ContainerID {
|
||
continue // the main instance container itself
|
||
}
|
||
// InspectByName accepts an ID too — Docker's ContainerInspect
|
||
// resolves either form. Status tells us if this is an
|
||
// exited orphan vs a still-running fs-helper.
|
||
hstate, ierr := dr.InspectByName(ctx, hid)
|
||
if ierr != nil {
|
||
continue
|
||
}
|
||
if hstate.Status == "exited" || hstate.Status == "dead" {
|
||
if err := dr.ForceRemoveContainer(ctx, hid); err == nil {
|
||
d.log.Info("rehydrate: reaped orphan sidecar", "instance_id", m.InstanceID, "container_id", short(hid))
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Orphan detection: a main game-server container (panel.role=instance)
|
||
// that exists on the runtime but has NO sidecar metadata. This is the
|
||
// fingerprint of an interrupted delete — e.g. a delete racing with
|
||
// agent shutdown, where the sidecar got removed but the container
|
||
// remove failed because the Docker connection died mid-call. Such an
|
||
// instance is fully invisible to the agent (every file/RCON/stop op
|
||
// returns "not on this target") even though the controller still
|
||
// routes to it and the world keeps running. Surface it loudly so it
|
||
// can be recovered instead of silently lost.
|
||
if dr, ok := d.runtime.(*runtime.DockerRuntime); ok {
|
||
names, lerr := dr.ListMainInstanceNames(ctx)
|
||
if lerr != nil {
|
||
d.log.Warn("rehydrate: orphan scan failed", "err", lerr)
|
||
} else {
|
||
for _, id := range names {
|
||
if haveMeta[id] {
|
||
continue
|
||
}
|
||
d.mu.Lock()
|
||
_, tracked := d.instances[id]
|
||
d.mu.Unlock()
|
||
if tracked {
|
||
continue
|
||
}
|
||
d.log.Error("rehydrate: ORPHAN container has no sidecar metadata — instance is invisible to the agent (likely an interrupted delete). Recover by recreating its sidecar or re-issuing create/delete from the controller.",
|
||
"instance_id", id, "container", "panel-"+id)
|
||
}
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// Announce re-sends the current state of every tracked instance to the
|
||
// controller. For instances whose containers are currently running, it
|
||
// also (re)activates log streaming + the state tracker. Called after
|
||
// each (re)connect.
|
||
func (d *Dispatcher) Announce(ctx context.Context) {
|
||
d.mu.Lock()
|
||
recs := make([]*instanceRecord, 0, len(d.instances))
|
||
for _, r := range d.instances {
|
||
recs = append(recs, r)
|
||
}
|
||
d.mu.Unlock()
|
||
|
||
for _, rec := range recs {
|
||
rtState, err := d.runtime.InspectByName(ctx, "panel-"+rec.InstanceID)
|
||
if err != nil {
|
||
d.log.Warn("announce: inspect failed", "instance_id", rec.InstanceID, "err", err)
|
||
continue
|
||
}
|
||
status := rtStatusToProto(rtState.Status)
|
||
d.sendInstanceState(rec.InstanceID, status, rtState.ExitCode, "announced:"+rtState.Status)
|
||
if rtState.Status == "running" {
|
||
// On every reconnect, tear down and rebuild the log + stats
|
||
// streams. The reason: the controller's in-memory log ring buffer
|
||
// is wiped on its restart, so when the user opens the Console tab
|
||
// after a controller restart, history is blank. Docker's
|
||
// ContainerLogs API replays Tail=400 lines on every fresh attach,
|
||
// so rebuilding the streams here repopulates the controller's
|
||
// ring buffer with recent history (including the per-module
|
||
// "ready" log line, which lets the dashboard's auto-scan clear
|
||
// the "starting" overlay without needing the operator to open
|
||
// the console). Old streams keep flowing in the background until
|
||
// their goroutines notice the cancelled context and exit.
|
||
d.mu.Lock()
|
||
prevLogCancel := rec.LogCancel
|
||
rec.LogCancel = nil
|
||
d.mu.Unlock()
|
||
if prevLogCancel != nil {
|
||
prevLogCancel()
|
||
}
|
||
d.activate(rec)
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---- Emitter (used by state.Tracker) ----
|
||
|
||
// EmitAppState forwards an AppStateUpdate upstream.
|
||
//
|
||
// Side effect: a successful state-source poll (the only thing that
|
||
// produces AppStateUpdate) implies the server is responsive. If the
|
||
// last status we sent was CRASHED, promote to RUNNING — handles the
|
||
// "segfault on first world-gen, healthy on retry" pattern where docker
|
||
// restart-policy auto-recovery in watchExit doesn't fire because the
|
||
// container's restart was driven by a different cause.
|
||
func (d *Dispatcher) EmitAppState(a *panelv1.AppStateUpdate) {
|
||
if a != nil && a.InstanceId != "" {
|
||
d.mu.Lock()
|
||
rec := d.instances[a.InstanceId]
|
||
d.mu.Unlock()
|
||
if rec != nil {
|
||
last := panelv1.InstanceStatus(rec.lastStatus.Load())
|
||
if last == panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED {
|
||
d.sendInstanceState(a.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING, 0,
|
||
"running (recovered — state source healthy)")
|
||
}
|
||
}
|
||
}
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_AppState{AppState: a},
|
||
})
|
||
}
|
||
|
||
// EmitPlayerEvent forwards a PlayerEvent upstream.
|
||
func (d *Dispatcher) EmitPlayerEvent(p *panelv1.PlayerEvent) {
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_Player{Player: p},
|
||
})
|
||
}
|
||
|
||
// ---- Lifecycle handlers ----
|
||
|
||
// handleCreate resolves the manifest, creates the container, and records it.
|
||
// The container is NOT started here — that's InstanceStart's job.
|
||
func (d *Dispatcher) handleCreate(ctx context.Context, corrID string, req *panelv1.InstanceCreate) {
|
||
log := d.log.With("instance_id", req.InstanceId, "module_id", req.ModuleId, "correlation_id", corrID)
|
||
|
||
manifest, ok := d.modules.Get(req.ModuleId)
|
||
if !ok {
|
||
d.replyError(corrID, req.InstanceId, "module_not_found", fmt.Sprintf("module %q not in registry", req.ModuleId))
|
||
log.Warn("module not found")
|
||
return
|
||
}
|
||
|
||
if !manifest.HasMode("docker") {
|
||
d.replyError(corrID, req.InstanceId, "mode_unsupported", "module does not support docker mode")
|
||
return
|
||
}
|
||
|
||
if req.DataPath == "" {
|
||
req.DataPath = filepath.Join(d.dataRoot, req.InstanceId)
|
||
}
|
||
if err := os.MkdirAll(req.DataPath, 0o755); err != nil {
|
||
d.replyError(corrID, req.InstanceId, "data_path_mkdir", err.Error())
|
||
log.Error("mkdir data_path", "err", err)
|
||
return
|
||
}
|
||
|
||
// Expand templated paths in any mount_overrides the controller sent,
|
||
// then mkdir the resolved host path so Docker's bind-mount won't
|
||
// silently create an empty root-owned dir elsewhere.
|
||
//
|
||
// Templates understood (both are absolute after expansion):
|
||
// $AGENT_DATA_ROOT → agent's --data-root flag (where per-instance
|
||
// data lives; default ./data/instances)
|
||
// $PANEL_DATA_ROOT → parent of AGENT_DATA_ROOT — the panel's data
|
||
// root (./data on the default setup). This is
|
||
// where cross-instance shared dirs live (like
|
||
// ARK cluster transfer dirs, alongside
|
||
// instances/ and backups/).
|
||
if len(req.MountOverrides) > 0 {
|
||
panelRoot := filepath.Dir(d.dataRoot)
|
||
for k, v := range req.MountOverrides {
|
||
exp := v
|
||
exp = strings.ReplaceAll(exp, "$PANEL_DATA_ROOT", panelRoot)
|
||
exp = strings.ReplaceAll(exp, "$AGENT_DATA_ROOT", d.dataRoot)
|
||
req.MountOverrides[k] = exp
|
||
if filepath.IsAbs(exp) {
|
||
if err := os.MkdirAll(exp, 0o755); err != nil {
|
||
d.replyError(corrID, req.InstanceId, "mount_mkdir", err.Error())
|
||
log.Error("mkdir mount override", "path", exp, "err", err)
|
||
return
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
values, err := mergeValuesAndSecrets(manifest, req.ConfigValues)
|
||
if err != nil {
|
||
d.replyError(corrID, req.InstanceId, "secrets", err.Error())
|
||
log.Error("generate secrets", "err", err)
|
||
return
|
||
}
|
||
if err := modulepkg.RenderForBranch(manifest, req.DataPath, values, resolveCreateBranch(manifest, values)); err != nil {
|
||
d.replyError(corrID, req.InstanceId, "render", err.Error())
|
||
log.Error("render config files", "err", err)
|
||
return
|
||
}
|
||
req.ConfigValues = values
|
||
|
||
// _PANEL_SUPPRESS_AUTOSTART sentinel — controller-set marker for
|
||
// recreate dances (env-config / change-ports / cluster / mods) that
|
||
// preserve the prior running state. When true, skip the agent's
|
||
// post-create auto-start path so a stopped instance stays stopped
|
||
// after the recreate. The controller will issue an explicit Start
|
||
// itself when the prior state was "running".
|
||
suppressAutoStart := false
|
||
if v := req.ConfigValues["_PANEL_SUPPRESS_AUTOSTART"]; v == "true" {
|
||
suppressAutoStart = true
|
||
delete(req.ConfigValues, "_PANEL_SUPPRESS_AUTOSTART")
|
||
}
|
||
|
||
// _PANEL_SKIP_REINSTALL sentinel — controller-set marker for recreate
|
||
// dances that PRESERVE the install volume (env-config / change-ports /
|
||
// cluster / mods). On a recreate the game files + the operator's custom
|
||
// Mods are already on the volume, so the create-time warm-seed +
|
||
// first-update must NOT run: warm-seed does `find /game -delete` and
|
||
// re-copies from an ARBITRARY sibling, which silently drops a 7DTD
|
||
// server's custom Mods (the whole point of preserving the volume is to
|
||
// keep them). A first-ever create never carries this flag, so brand-new
|
||
// installs still warm-seed / download normally. Invariant for callers:
|
||
// any path that sets this must also set _PANEL_SUPPRESS_AUTOSTART when the
|
||
// prior state was stopped AND issue its own Start when it was running —
|
||
// the agent does neither for a skip-reinstall recreate.
|
||
skipReinstall := false
|
||
if v := req.ConfigValues["_PANEL_SKIP_REINSTALL"]; v == "true" {
|
||
skipReinstall = true
|
||
delete(req.ConfigValues, "_PANEL_SKIP_REINSTALL")
|
||
}
|
||
|
||
// _PANEL_SEED_MODS_FROM sentinel — controller-set marker for a cluster
|
||
// join that should clone the cluster master's Mods set onto this server.
|
||
// Value is the master's instance id. Handled in the skip-reinstall branch
|
||
// below (a cluster join is always a volume-preserving recreate): we run a
|
||
// Mods-only sidecar copy before the controller starts the server.
|
||
seedModsFrom := ""
|
||
if v := req.ConfigValues["_PANEL_SEED_MODS_FROM"]; v != "" {
|
||
seedModsFrom = v
|
||
delete(req.ConfigValues, "_PANEL_SEED_MODS_FROM")
|
||
}
|
||
|
||
// _PANEL_WIPE_PATHS sentinel — comma-separated list of paths
|
||
// (relative to BrowseableRoot) the controller wants nuked before
|
||
// the new container starts. Used by the ARK map-switch flow when
|
||
// the operator checks "wipe existing world data" — clean slate
|
||
// for the new map without keeping the old map's saves on disk.
|
||
//
|
||
// Strip the key so it doesn't pollute the container env. Wipe runs
|
||
// AFTER the previous container was deleted (volumes preserved) and
|
||
// BEFORE we create the new one, in an alpine sidecar that mounts
|
||
// the same volumes we're about to mount.
|
||
if wipeCSV := req.ConfigValues["_PANEL_WIPE_PATHS"]; wipeCSV != "" {
|
||
delete(req.ConfigValues, "_PANEL_WIPE_PATHS")
|
||
paths := []string{}
|
||
for _, p := range strings.Split(wipeCSV, ",") {
|
||
p = strings.TrimSpace(p)
|
||
if p != "" {
|
||
paths = append(paths, p)
|
||
}
|
||
}
|
||
if len(paths) > 0 {
|
||
if err := d.runWipeSidecar(ctx, manifest, req.InstanceId, req.DataPath, paths); err != nil {
|
||
d.replyError(corrID, req.InstanceId, "wipe", err.Error())
|
||
log.Error("wipe sidecar", "err", err)
|
||
return
|
||
}
|
||
log.Info("wiped paths before create", "instance_id", req.InstanceId, "paths", paths)
|
||
}
|
||
}
|
||
|
||
// Auto-bump host ports that collide with already-bound ports on the
|
||
// host. Without this, two ARK SA instances both try to bind 7777/UDP
|
||
// + 27020/TCP — the second container's port mapping silently drops
|
||
// (Docker creates the container without published ports), the agent's
|
||
// RCON tracker still dials the manifest default (27020), and ends up
|
||
// talking to whichever container won the binding. Result: both
|
||
// "running" servers share state from one container's RCON.
|
||
//
|
||
// We mutate req.Ports in place so both ResolveDocker (below) and
|
||
// resolveRCONAddress (after Create) see the bumped ports — keeping
|
||
// the RCON dial address aligned with the actual host binding.
|
||
if err := assignNonCollidingHostPorts(manifest, req); err != nil {
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, -1, "port_alloc_failed: "+err.Error())
|
||
d.replyError(corrID, req.InstanceId, "port_alloc", err.Error())
|
||
log.Error("port allocation", "err", err)
|
||
return
|
||
}
|
||
|
||
// Mirror the assigned ports into ConfigValues for any port that
|
||
// declares an `env:` mapping. This is what makes a non-default
|
||
// host port reach the game binary — without it, two ARK instances
|
||
// pick distinct host ports but both ARKs internally bind 7777/27015
|
||
// because their ASA_PORT/QUERY_PORT envs are still the manifest
|
||
// defaults. Steam server browser would then list only one of them.
|
||
//
|
||
// Also force ContainerPort = HostPort for ALL ports. The game's
|
||
// env var tells it which port to bind INSIDE the container; that
|
||
// port has to match what Docker forwards from the outside, otherwise
|
||
// the agent's RCON dial (or a player's join attempt) hits an empty
|
||
// container port. Was previously a 1:1 mapping for non-internal
|
||
// ports only — but RCON's env var (e.g. RCON_PORT) also had to
|
||
// follow the host port to avoid the same mismatch internally.
|
||
if req.ConfigValues == nil {
|
||
req.ConfigValues = map[string]string{}
|
||
}
|
||
for i, p := range req.Ports {
|
||
if p == nil {
|
||
continue
|
||
}
|
||
decl := manifest.Port(p.Name)
|
||
if decl == nil {
|
||
continue
|
||
}
|
||
if decl.Env != "" {
|
||
req.ConfigValues[decl.Env] = fmt.Sprintf("%d", p.HostPort)
|
||
}
|
||
req.Ports[i].ContainerPort = p.HostPort
|
||
}
|
||
|
||
spec, err := agentmodule.ResolveDocker(manifest, req)
|
||
if err != nil {
|
||
d.replyError(corrID, req.InstanceId, "resolve", err.Error())
|
||
log.Error("resolve docker spec", "err", err)
|
||
return
|
||
}
|
||
|
||
// Forward image-acquisition progress (auto-build of `panel-*` images
|
||
// on first use, or registry pull progress) to the controller as log
|
||
// lines on the regular stdout stream — operator sees them in the
|
||
// Console tab while waiting on the create. Without this, first-time
|
||
// builds run silently for 30s–3min and the UI just sits on
|
||
// "installing".
|
||
spec.LogSink = func(line string) {
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||
InstanceId: req.InstanceId,
|
||
Stream: "stdout",
|
||
At: timestamppb.Now(),
|
||
Line: line,
|
||
}},
|
||
})
|
||
}
|
||
|
||
log.Info("pulling image and creating container", "image", spec.Image)
|
||
containerID, err := d.runtime.Create(ctx, spec)
|
||
if err != nil {
|
||
// Emit a terminal state so the controller's DB row moves out of
|
||
// "creating" and the operator can see what went wrong.
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, -1, "create_failed: "+err.Error())
|
||
d.replyError(corrID, req.InstanceId, "create", err.Error())
|
||
log.Error("runtime create", "err", err)
|
||
return
|
||
}
|
||
|
||
rconAddr, rconPassword := resolveRCONAddress(manifest, req)
|
||
browseable := resolveBrowseableRoot(manifest)
|
||
roots := resolveBrowseableRoots(manifest)
|
||
|
||
rec := &instanceRecord{
|
||
InstanceID: req.InstanceId,
|
||
ContainerID: containerID,
|
||
ModuleID: req.ModuleId,
|
||
Branch: resolveCreateBranch(manifest, req.ConfigValues),
|
||
DataPath: req.DataPath,
|
||
BrowseableRoot: browseable,
|
||
BrowseableRoots: roots,
|
||
RCONAddr: rconAddr,
|
||
RCONPassword: rconPassword,
|
||
}
|
||
|
||
d.mu.Lock()
|
||
d.instances[req.InstanceId] = rec
|
||
d.mu.Unlock()
|
||
|
||
if err := d.writeMeta(rec); err != nil {
|
||
log.Warn("write sidecar meta failed (instance will not survive agent restart)", "err", err)
|
||
}
|
||
|
||
log.Info("instance created", "container_id", short(containerID), "rcon_addr", rconAddr)
|
||
// Detail string is parsed by the controller's UI to drive banners; for
|
||
// modules whose first update_provider needs Steam creds we tag the
|
||
// state so the dashboard can show a "Sign in + Update to install"
|
||
// banner directly on the card. Otherwise stay with the generic
|
||
// "created" detail.
|
||
createdDetail := "created"
|
||
if len(manifest.UpdateProviders) > 0 && manifest.UpdateProviders[0].RequiresSteamLogin {
|
||
createdDetail = "needs_steam_login"
|
||
}
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, createdDetail)
|
||
d.replyOK(corrID)
|
||
|
||
// Shared auto-start helper — used from both the update-success path and
|
||
// the "skip update but still auto-start" path so the Create flow lands
|
||
// on a running server regardless of which install model the module uses.
|
||
autoStart := func() {
|
||
startCorr := fmt.Sprintf("autostart-%d", time.Now().UnixNano())
|
||
d.log.Info("auto-starting after create", "instance_id", req.InstanceId, "correlation_id", startCorr)
|
||
startCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||
defer cancel()
|
||
d.handleStart(startCtx, startCorr, &panelv1.InstanceStart{InstanceId: req.InstanceId})
|
||
}
|
||
|
||
// Auto-fire the primary update provider so the operator doesn't have to
|
||
// click Update as a separate step. Opt-out via `auto_update_on_create:
|
||
// false` in module.yaml for modules whose image handles its own
|
||
// downloads (e.g. ark-sa's community image).
|
||
//
|
||
// Recreate-dance creates carry _PANEL_SUPPRESS_AUTOSTART=true; for those
|
||
// we skip BOTH the update kick AND the auto-start so a previously
|
||
// stopped instance stays stopped after an env-config edit. Volumes
|
||
// already hold the game files (recreate preserves them), so re-running
|
||
// the updater would just waste minutes and then auto-start at the end
|
||
// via onUpdateComplete. Controller will issue an explicit Start when
|
||
// the prior state was running.
|
||
if suppressAutoStart {
|
||
log.Info("create: auto-start + auto-update suppressed (recreate dance, prior state was stopped)", "instance_id", req.InstanceId)
|
||
} else if skipReinstall {
|
||
// Recreate that PRESERVES a populated install volume (env-config /
|
||
// cluster / ports / mods). Skip the create-time warm-seed +
|
||
// first-update — they wipe /game and re-copy from a sibling, which
|
||
// silently drops a 7DTD server's custom Mods. The controller drives
|
||
// the post-recreate start itself (explicit Start when the prior state
|
||
// was running; stays stopped otherwise via _PANEL_SUPPRESS_AUTOSTART),
|
||
// so the agent does nothing further here — EXCEPT an optional mod-seed.
|
||
if seedModsFrom != "" {
|
||
// Cluster join with "seed mods from master": clone the master's
|
||
// Mods set onto this server before the controller starts it.
|
||
// Synchronous so the seeded mods are in place by the time the
|
||
// create reply returns and the controller issues Start (~5-10s for
|
||
// a typical ~170 MB 7DTD mod set — well inside the 90s create
|
||
// budget). Best-effort: a failure leaves the existing mods intact.
|
||
seedCtx, cancel := context.WithTimeout(context.Background(), 75*time.Second)
|
||
if _, serr := d.seedModsFromInstance(seedCtx, seedModsFrom, req.InstanceId, req.DataPath, manifest); serr != nil {
|
||
log.Warn("mod-seed failed — server keeps its existing mods", "instance_id", req.InstanceId, "src", seedModsFrom, "err", serr)
|
||
}
|
||
cancel()
|
||
}
|
||
log.Info("create: install preserved (recreate dance) — skipping warm-seed + update", "instance_id", req.InstanceId)
|
||
} else if shouldAutoUpdate(manifest) {
|
||
// Default to the first declared provider, but allow a config_value
|
||
// to override it at create time. This is what lets Conan Exiles
|
||
// pick `enhanced` (UE5) vs `legacy` (UE4) from the create wizard's
|
||
// EDITION dropdown — the matching update_provider id fires on
|
||
// first install. Generic by design: any module can expose a
|
||
// `provider_id` config_value and have it auto-honored.
|
||
providerID := manifest.UpdateProviders[0].ID
|
||
if pref := pickProviderFromConfigValues(req.ConfigValues, manifest); pref != "" {
|
||
providerID = pref
|
||
}
|
||
log.Info("auto-triggering first update", "provider_id", providerID,
|
||
"chosen_via_config_values", providerID != manifest.UpdateProviders[0].ID)
|
||
autoCorr := fmt.Sprintf("autoupdate-%d", time.Now().UnixNano())
|
||
instanceID := req.InstanceId
|
||
// Surface the install as a server-authoritative state so any operator
|
||
// viewing the dashboard (not just the one who clicked Create) sees
|
||
// the "updating" pulse. The client was previously only flipping
|
||
// is-installing locally on its own Create button press — refreshing
|
||
// or joining a session mid-install showed "stopped" incorrectly.
|
||
// handleUpdateWithHook emits UPDATING again immediately below, but
|
||
// firing it here too closes the UI race between create-completion
|
||
// and the goroutine starting.
|
||
d.sendInstanceState(instanceID, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, "installing")
|
||
|
||
// When the first-time download finishes, auto-start the server. That
|
||
// way Create → "installing" → "starting" → "running" happens as one
|
||
// operator gesture instead of requiring a separate Start click. Mirrors
|
||
// AMP-style "create and run" behaviour. Guarded by the same flag as
|
||
// auto-update itself — a module that opts out of auto-update stays
|
||
// stopped after create as before.
|
||
onUpdateComplete := func(err error) {
|
||
if err != nil {
|
||
// Clear the "installing" pulse so the UI stops pretending the
|
||
// download is in flight. Detail carries the error for anyone
|
||
// inspecting the DB / logs.
|
||
d.sendInstanceState(instanceID, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, -1, "install_failed: "+err.Error())
|
||
return
|
||
}
|
||
// Cluster join "seed mods from master" requested at CREATE time: on a
|
||
// FRESH create the install ran through this warm-seed/update path (not
|
||
// the skip-reinstall branch that also honors seedModsFrom), so clone
|
||
// the master's Mods set HERE — after the base install, before the
|
||
// server is startable. Best-effort: a failure keeps the base mods.
|
||
if seedModsFrom != "" {
|
||
d.sendInstanceState(instanceID, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, fmt.Sprintf("seeding mods from cluster master (%s)", seedModsFrom))
|
||
seedCtx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||
if _, serr := d.seedModsFromInstance(seedCtx, seedModsFrom, instanceID, req.DataPath, manifest); serr != nil {
|
||
log.Warn("create mod-seed failed — server keeps its base mods", "instance_id", instanceID, "src", seedModsFrom, "err", serr)
|
||
}
|
||
cancel()
|
||
}
|
||
// Drop the installing marker before kicking Start — handleStart
|
||
// sends its own STARTING event which will supersede, but emitting
|
||
// the transition explicitly avoids an awkward "installing" flash
|
||
// if Start races the next poll tick.
|
||
d.sendInstanceState(instanceID, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "installed")
|
||
// Auto-start ONLY if the module opts in. 7DTD sets
|
||
// auto_start_on_create=false so the operator finishes world/RWG +
|
||
// cluster setup and starts manually (and to avoid the warm-seed →
|
||
// immediate-start memory race that locked up the agent host).
|
||
if shouldAutoStart(manifest) {
|
||
autoStart()
|
||
} else {
|
||
log.Info("create: install complete — auto-start suppressed (auto_start_on_create=false); operator starts manually", "instance_id", instanceID)
|
||
}
|
||
}
|
||
// Warm-seed shortcut: if a sibling instance of the same module
|
||
// already has the install volume populated, copy it instead of
|
||
// running the updater. Saves the operator a 30-90 min SteamCMD
|
||
// round-trip on the second/third/Nth instance of a game. If no
|
||
// sibling or the seed fails, fall through to the normal updater.
|
||
// The seed itself runs in a debian sidecar and takes ~2-5 min
|
||
// for ~16 GB on local SSD.
|
||
dataPath := req.DataPath
|
||
go func() {
|
||
seedCtx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||
defer cancel()
|
||
seeded, err := d.tryWarmSeedFromSibling(seedCtx, instanceID, manifest, dataPath)
|
||
if err != nil {
|
||
log.Warn("warm-seed failed, falling through to updater", "instance_id", instanceID, "err", err)
|
||
} else if seeded {
|
||
log.Info("warm-seed succeeded — skipping initial updater run", "instance_id", instanceID, "module_id", manifest.ID)
|
||
onUpdateComplete(nil)
|
||
return
|
||
}
|
||
d.handleUpdateWithHook(autoCorr, &panelv1.UpdateRequest{
|
||
InstanceId: instanceID,
|
||
ProviderId: providerID,
|
||
}, onUpdateComplete)
|
||
}()
|
||
} else if shouldAutoStart(manifest) {
|
||
// Image-managed install path (ark-sa et al): no external updater but
|
||
// we still want Create → running in one gesture. The container's
|
||
// entrypoint handles first-boot SteamCMD itself, so all we do here
|
||
// is kick Start. UI sees STARTING (yellow pulse) until the game's
|
||
// ready-regex fires. Slight delay gives the agent's replyOK envelope
|
||
// time to reach the controller before the next state update.
|
||
log.Info("auto-starting (module handles install via container entrypoint)")
|
||
go func() {
|
||
time.Sleep(250 * time.Millisecond)
|
||
autoStart()
|
||
}()
|
||
}
|
||
}
|
||
|
||
// shouldAutoStart decides whether a just-created instance should boot
|
||
// automatically after the create (and first-time update, if any) lands.
|
||
// Default: yes. Modules that legitimately want "create then wait for the
|
||
// operator to configure before starting" opt out with
|
||
// `auto_start_on_create: false`. Independent of shouldAutoUpdate so that
|
||
// image-managed modules (ark-sa: no external updater, in-container
|
||
// SteamCMD) still end the Create flow on a running server.
|
||
//
|
||
// EXCEPTION — Steam-login-required modules: when the first update_provider
|
||
// has requires_steam_login, the create-time auto-update is skipped (no
|
||
// creds to use), so the game files were never downloaded. Auto-starting
|
||
// in that state would just bounce the container with an "executable not
|
||
// found" error and leave the instance crashed. We hold off until the
|
||
// operator has signed in to Steam + clicked Update; the update path's
|
||
// onUpdateComplete hook fires the start at the right moment.
|
||
func shouldAutoStart(m *modulepkg.Manifest) bool {
|
||
if m == nil {
|
||
return false
|
||
}
|
||
if m.AutoStartOnCreate != nil {
|
||
return *m.AutoStartOnCreate
|
||
}
|
||
// Don't auto-start before the Steam-gated install has run.
|
||
if len(m.UpdateProviders) > 0 && m.UpdateProviders[0].RequiresSteamLogin {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// shouldAutoUpdate decides whether a just-created instance should kick off
|
||
// its first update provider automatically. Default: yes, if there's any
|
||
// provider declared. Modules whose container image handles its own
|
||
// downloads (ark-sa via UPDATE_SERVER=TRUE) should set
|
||
// `auto_update_on_create: false` in their manifest.
|
||
func shouldAutoUpdate(m *modulepkg.Manifest) bool {
|
||
if m == nil || len(m.UpdateProviders) == 0 {
|
||
return false
|
||
}
|
||
if m.AutoUpdateOnCreate != nil {
|
||
return *m.AutoUpdateOnCreate
|
||
}
|
||
// Modules whose first provider requires a Steam login can't auto-update:
|
||
// the agent doesn't have credentials on the auto-create path (the
|
||
// controller-side Steam-login gate doesn't intercept create events).
|
||
// The operator clicks Update manually once creds are in place; that
|
||
// path flows through the controller and picks up the cached login.
|
||
if m.UpdateProviders[0].RequiresSteamLogin {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// pickProviderFromConfigValues lets a module expose a config_value that
|
||
// overrides which update_provider auto-fires at create time. Two keys are
|
||
// honored, in order of preference:
|
||
//
|
||
// 1. config_values["provider_id"] — explicit, generic. Matches an
|
||
// update_provider's `id` directly.
|
||
// 2. config_values["EDITION"] — module-specific convention used by
|
||
// Conan Exiles (enhanced/legacy → matching provider id).
|
||
//
|
||
// If neither is set, or the named provider doesn't exist, returns "" so
|
||
// the caller falls back to UpdateProviders[0]. Case-insensitive match
|
||
// against the provider id; trim whitespace defensively because
|
||
// `runtime.docker.env` defaults can sneak through with stray spaces.
|
||
func pickProviderFromConfigValues(cv map[string]string, m *modulepkg.Manifest) string {
|
||
if cv == nil || m == nil || len(m.UpdateProviders) == 0 {
|
||
return ""
|
||
}
|
||
candidates := []string{cv["provider_id"], cv["EDITION"]}
|
||
for _, want := range candidates {
|
||
want = strings.TrimSpace(strings.ToLower(want))
|
||
if want == "" {
|
||
continue
|
||
}
|
||
for _, p := range m.UpdateProviders {
|
||
if strings.ToLower(p.ID) == want {
|
||
return p.ID
|
||
}
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// resolveCreateBranch returns the normalized Steam branch an instance is being
|
||
// installed from, derived from the update_provider it will use (the same
|
||
// selection logic as the create-time auto-update: config_values.provider_id
|
||
// via pickProviderFromConfigValues, else UpdateProviders[0]). The value tags
|
||
// the instance record so warm-seed only reuses an install from a sibling on
|
||
// the SAME branch. See normalizeBranch for the public-equivalent collapsing.
|
||
func resolveCreateBranch(m *modulepkg.Manifest, cv map[string]string) string {
|
||
if m == nil || len(m.UpdateProviders) == 0 {
|
||
return ""
|
||
}
|
||
id := m.UpdateProviders[0].ID
|
||
if pref := pickProviderFromConfigValues(cv, m); pref != "" {
|
||
id = pref
|
||
}
|
||
for i := range m.UpdateProviders {
|
||
if m.UpdateProviders[i].ID == id {
|
||
return normalizeBranch(m.UpdateProviders[i].Beta)
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// handleStart starts a previously-created instance and begins log streaming.
|
||
func (d *Dispatcher) handleStart(ctx context.Context, corrID string, req *panelv1.InstanceStart) {
|
||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||
|
||
d.mu.Lock()
|
||
rec, ok := d.instances[req.InstanceId]
|
||
d.mu.Unlock()
|
||
if !ok {
|
||
d.replyError(corrID, req.InstanceId, "not_found", "instance not created on this target")
|
||
return
|
||
}
|
||
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STARTING, 0, "starting")
|
||
|
||
// A Start racing an in-flight graceful stop is a trap: docker start on a
|
||
// container that's still shutting down is a no-op "success", and when the
|
||
// stop's SIGTERM grace finally lands it kills the instance the operator
|
||
// just asked to start (observed live: a UI restart left the server
|
||
// stranded stopped). Wait for the stop to settle before proceeding —
|
||
// bounded so a wedged stop can't park Start forever.
|
||
if rec.stopping.Load() {
|
||
log.Info("start racing an in-flight stop — waiting for it to settle")
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STARTING, 0, "waiting for in-flight stop to finish…")
|
||
deadline := time.Now().Add(90 * time.Second)
|
||
for rec.stopping.Load() && time.Now().Before(deadline) {
|
||
select {
|
||
case <-ctx.Done():
|
||
d.replyError(corrID, req.InstanceId, "cancelled", ctx.Err().Error())
|
||
return
|
||
case <-time.After(time.Second):
|
||
}
|
||
}
|
||
}
|
||
|
||
// Region Medic — for 7DTD, validate + heal the active world's regions while
|
||
// the container is still STOPPED, snapshot a fresh rolling backup, and ping
|
||
// Discord on heal. The CLI reads its per-server config (enable/keep/channel)
|
||
// from region-medic.json in the saves volume. Best-effort: never blocks start.
|
||
if rec.ModuleID == "7dtd" {
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STARTING, 0, "region medic: validating world…")
|
||
d.runRegionMedicOnStart(ctx, rec)
|
||
// Snapshot the shared cluster Player/ dir (character .ttp saves) to a
|
||
// rolling slot before boot. No-op for standalone servers. The panel
|
||
// tar backup can't capture this — the active Player dir is a symlink
|
||
// out to the /cluster bind — so this is the only restore path for
|
||
// per-character data on a clustered server.
|
||
d.snapshotClusterPlayerOnStart(ctx, rec)
|
||
}
|
||
|
||
// Retry start on transient port-bind races. When a container exits the
|
||
// kernel can hold its UDP bindings for a few seconds; the next Start
|
||
// sees "ports are not available" until the OS releases them. A short
|
||
// backoff is much friendlier than making the user click Start twice.
|
||
const maxAttempts = 4
|
||
var err error
|
||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||
err = d.runtime.Start(ctx, rec.ContainerID)
|
||
if err == nil {
|
||
break
|
||
}
|
||
if !isTransientStartError(err) || attempt == maxAttempts {
|
||
break
|
||
}
|
||
wait := time.Duration(attempt*2) * time.Second
|
||
log.Warn("transient start error, retrying", "err", err, "attempt", attempt, "wait", wait)
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STARTING, 0,
|
||
fmt.Sprintf("retrying (port still held): attempt %d/%d", attempt+1, maxAttempts))
|
||
select {
|
||
case <-ctx.Done():
|
||
d.replyError(corrID, req.InstanceId, "cancelled", ctx.Err().Error())
|
||
return
|
||
case <-time.After(wait):
|
||
}
|
||
}
|
||
if err != nil {
|
||
d.replyError(corrID, req.InstanceId, "start", err.Error())
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, -1, err.Error())
|
||
log.Error("runtime start", "err", err)
|
||
return
|
||
}
|
||
d.activate(rec)
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING, 0, "running")
|
||
d.replyOK(corrID)
|
||
log.Info("instance started")
|
||
|
||
// Auto-provision a freshly-generated clustered 7DTD world: watch (best-effort,
|
||
// off-thread) for the gen boot to bake native-banded decorations and bounce the
|
||
// container ONCE so the entrypoint aligns them to the cluster canonical. Strict
|
||
// no-op for already-canon worlds (season10/insane/pvp/creative) and non-cluster
|
||
// servers; single-flight. See decoremap_provision.go.
|
||
if rec.ModuleID == "7dtd" {
|
||
d.autoProvisionClusterWorldOnStart(rec)
|
||
}
|
||
|
||
// post_start lifecycle hooks — best-effort shell commands that run
|
||
// INSIDE the container after it's up. Used to patch image-generated
|
||
// config files that need a panel-side fixup (e.g. ark-sa's RCON port
|
||
// needs to be duplicated into [SessionSettings] of GameUserSettings.ini
|
||
// because ASA ignores AUTH responses when it's only under
|
||
// [ServerSettings]). Failures are logged but don't fail the Start —
|
||
// a broken hook shouldn't block an otherwise-healthy server.
|
||
if manifest, ok := d.modules.Get(rec.ModuleID); ok && manifest.Lifecycle != nil && len(manifest.Lifecycle.PostStart) > 0 {
|
||
go d.runPostStartHooks(rec, manifest.Lifecycle.PostStart)
|
||
}
|
||
}
|
||
|
||
// runPostStartHooks executes each post_start command inside the running
|
||
// container, via docker exec. Runs async so a slow or blocking hook (e.g.
|
||
// one that waits for a config file to be written by the entrypoint) never
|
||
// holds up the Start reply or the dashboard state transition to RUNNING.
|
||
func (d *Dispatcher) runPostStartHooks(rec *instanceRecord, commands []string) {
|
||
log := d.log.With("instance_id", rec.InstanceID, "phase", "post_start")
|
||
// Bound the whole hook suite at 10 minutes — ASA's ini fixup waits
|
||
// for the image's entrypoint to write the file on first boot, which
|
||
// on a cold SteamCMD install can take several minutes.
|
||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||
defer cancel()
|
||
for i, cmd := range commands {
|
||
if cmd == "" {
|
||
continue
|
||
}
|
||
log.Info("running hook", "index", i, "cmd_preview", previewCmd(cmd))
|
||
stdout, stderr, code, err := d.runtime.ExecCapture(ctx, rec.ContainerID, []string{"sh", "-c", cmd})
|
||
if err != nil {
|
||
log.Warn("hook exec failed", "index", i, "err", err)
|
||
continue
|
||
}
|
||
if code != 0 {
|
||
log.Warn("hook non-zero exit", "index", i, "exit_code", code,
|
||
"stdout", trimForLog(stdout), "stderr", trimForLog(stderr))
|
||
continue
|
||
}
|
||
log.Info("hook ok", "index", i, "exit_code", code, "stdout", trimForLog(stdout))
|
||
}
|
||
}
|
||
|
||
func previewCmd(s string) string {
|
||
s = strings.TrimSpace(strings.ReplaceAll(s, "\n", " "))
|
||
if len(s) > 80 {
|
||
return s[:80] + "…"
|
||
}
|
||
return s
|
||
}
|
||
|
||
func trimForLog(b []byte) string {
|
||
s := strings.TrimSpace(string(b))
|
||
if len(s) > 400 {
|
||
s = s[:400] + "…"
|
||
}
|
||
return s
|
||
}
|
||
|
||
// extractINIValue pulls a single key from a specific section of an INI
|
||
// file. If section is empty the first matching key anywhere in the file
|
||
// wins. Lines with `;` or `#` as the first non-space character are
|
||
// comments. Whitespace around the value is trimmed.
|
||
//
|
||
// Used to read live RCON passwords out of game config files (ARK's
|
||
// [ServerSettings]ServerAdminPassword) so operator edits through the
|
||
// Config tab take effect without panel manual intervention.
|
||
func extractINIValue(content, section, key string) (string, error) {
|
||
inSection := section == ""
|
||
scanner := strings.Split(content, "\n")
|
||
for _, raw := range scanner {
|
||
line := strings.TrimSpace(raw)
|
||
if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
|
||
sec := line[1 : len(line)-1]
|
||
if section == "" {
|
||
// No section filter — keep scanning across sections.
|
||
continue
|
||
}
|
||
inSection = sec == section
|
||
continue
|
||
}
|
||
if !inSection {
|
||
continue
|
||
}
|
||
eq := strings.IndexByte(line, '=')
|
||
if eq <= 0 {
|
||
continue
|
||
}
|
||
k := strings.TrimSpace(line[:eq])
|
||
if k == key {
|
||
return strings.TrimSpace(line[eq+1:]), nil
|
||
}
|
||
}
|
||
return "", fmt.Errorf("ini key [%s]%s not found", section, key)
|
||
}
|
||
|
||
// isTransientStartError detects Docker errors that typically resolve within
|
||
// a few seconds (port bind still held by kernel after previous container
|
||
// exit, daemon momentarily busy). We treat these as retryable so the user
|
||
// doesn't see a stale "ports unavailable" the first time they click Start.
|
||
func isTransientStartError(err error) bool {
|
||
if err == nil {
|
||
return false
|
||
}
|
||
s := err.Error()
|
||
return strings.Contains(s, "ports are not available") ||
|
||
strings.Contains(s, "port is already allocated") ||
|
||
strings.Contains(s, "address already in use") ||
|
||
strings.Contains(s, "bind: Only one usage")
|
||
}
|
||
|
||
// activate starts log streaming + state tracker for a running instance.
|
||
// Idempotent: if LogCancel is already set, this is a no-op.
|
||
func (d *Dispatcher) activate(rec *instanceRecord) {
|
||
d.mu.Lock()
|
||
alreadyActive := rec.LogCancel != nil
|
||
d.mu.Unlock()
|
||
if alreadyActive {
|
||
return
|
||
}
|
||
// Free the shared volumes for the game by tearing down the fs helper.
|
||
// Both containers on the same volume is technically allowed, but we
|
||
// prefer the game own it exclusively while it's running.
|
||
d.teardownFsHelper(rec.InstanceID)
|
||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||
if !ok {
|
||
d.log.Warn("activate: module vanished", "instance_id", rec.InstanceID, "module_id", rec.ModuleID)
|
||
return
|
||
}
|
||
|
||
// Tear down any tracker still attached to this rec from a previous
|
||
// activation BEFORE building a new one. activate() is called from
|
||
// handleStart (which doesn't pre-cancel), Announce on every controller
|
||
// reconnect (definitely doesn't pre-cancel), and the post-warm-seed
|
||
// autostart in handleCreate. Without this guard each call leaves the
|
||
// previous goroutine alive with its frozen cfg.RCONAddr — visible as
|
||
// "rcon dial failed ... 127.0.0.1:<old_port>" forever after a recreate
|
||
// (the old tracker still references the old container's port allocation;
|
||
// only an agent restart cleared it). Wait briefly for the goroutine to
|
||
// exit so the fresh dial doesn't race the stale one.
|
||
d.mu.Lock()
|
||
prevCancel := rec.TrackerCancel
|
||
prevDone := rec.TrackerDone
|
||
rec.TrackerCancel = nil
|
||
rec.Tracker = nil
|
||
rec.TrackerDone = nil
|
||
d.mu.Unlock()
|
||
if prevCancel != nil {
|
||
prevCancel()
|
||
if prevDone != nil {
|
||
select {
|
||
case <-prevDone:
|
||
case <-time.After(5 * time.Second):
|
||
d.log.Warn("activate: prior tracker did not exit within 5s",
|
||
"instance_id", rec.InstanceID)
|
||
}
|
||
}
|
||
}
|
||
|
||
// On rehydrate, sidecar may have an empty RCONPassword (older agent
|
||
// versions didn't persist secrets, or the password lookup hadn't
|
||
// been wired for this module yet). Re-resolve from the live manifest
|
||
// so a manifest update (e.g. adding password_secret to ark-sa) takes
|
||
// effect on the next start without forcing a delete + recreate.
|
||
if rec.RCONPassword == "" && manifest.RCON != nil {
|
||
_, pw := resolveRCONAddress(manifest, &panelv1.InstanceCreate{InstanceId: rec.InstanceID})
|
||
if pw != "" {
|
||
rec.RCONPassword = pw
|
||
_ = d.writeMeta(rec) // best-effort persist
|
||
}
|
||
}
|
||
|
||
// If the module declares a way to source the RCON password at runtime
|
||
// (from a file or from a specific INI key), wire a PasswordFunc that
|
||
// re-reads on each dial attempt. Lets modules whose entrypoint
|
||
// generates secrets at first boot (7DTD's TelnetPassword via
|
||
// /game-saves/.panel-telnet-password) or whose config file is the
|
||
// source of truth (ARK: SA's [ServerSettings]ServerAdminPassword)
|
||
// work without manual wiring.
|
||
var passwordFunc func(context.Context) (string, error)
|
||
if manifest.RCON != nil {
|
||
if iniSpec := manifest.RCON.PasswordFromINI; iniSpec != nil && iniSpec.File != "" && iniSpec.Key != "" {
|
||
path := iniSpec.File
|
||
section := iniSpec.Section
|
||
key := iniSpec.Key
|
||
containerID := rec.ContainerID
|
||
rt := d.runtime
|
||
passwordFunc = func(ctx context.Context) (string, error) {
|
||
data, err := rt.CopyFileFromContainer(ctx, containerID, path)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return extractINIValue(string(data), section, key)
|
||
}
|
||
} else if manifest.RCON.PasswordFromFile != "" {
|
||
path := manifest.RCON.PasswordFromFile
|
||
containerID := rec.ContainerID
|
||
rt := d.runtime
|
||
passwordFunc = func(ctx context.Context) (string, error) {
|
||
data, err := rt.CopyFileFromContainer(ctx, containerID, path)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return strings.TrimSpace(string(data)), nil
|
||
}
|
||
}
|
||
}
|
||
|
||
tracker, err := state.New(d.log, state.Config{
|
||
InstanceID: rec.InstanceID,
|
||
Manifest: manifest,
|
||
RCONAddr: rec.RCONAddr,
|
||
RCONPassword: rec.RCONPassword,
|
||
PasswordFunc: passwordFunc,
|
||
Emitter: d,
|
||
// stdio adapter needs a handle to the runtime so it can attach to
|
||
// the container's stdin. Irrelevant for tcp adapters — tracker
|
||
// ignores these fields unless manifest.RCON.Adapter == "stdio".
|
||
ContainerID: "panel-" + rec.InstanceID,
|
||
Stdio: d.runtime.(rcon.StdioBackend),
|
||
// arkHangGuard hook — counts consecutive RCON failures per
|
||
// instance and triggers an auto-restart at threshold. No-op for
|
||
// non-ark-sa modules (the guard short-circuits inside).
|
||
OnRconResult: func(success bool) { d.arkHangGuardOnRconResult(rec, success) },
|
||
})
|
||
if err != nil {
|
||
d.log.Warn("state tracker init failed (continuing without it)", "instance_id", rec.InstanceID, "err", err)
|
||
} else {
|
||
trackerCtx, trackerCancel := context.WithCancel(context.Background())
|
||
done := make(chan struct{})
|
||
d.mu.Lock()
|
||
rec.Tracker = tracker
|
||
rec.TrackerCancel = trackerCancel
|
||
rec.TrackerDone = done
|
||
d.mu.Unlock()
|
||
go func() {
|
||
defer close(done)
|
||
tracker.Run(trackerCtx)
|
||
}()
|
||
}
|
||
|
||
// Same teardown discipline for the log + stats streams: every caller of
|
||
// activate (Announce, handleStart, post-warm-seed autostart) reaches
|
||
// here, and most pre-cancel LogCancel — but not all. Cancel anything
|
||
// lingering before installing fresh streams so we don't end up with
|
||
// two streamLogs goroutines duplicating every log line into the
|
||
// emitter.
|
||
d.mu.Lock()
|
||
prevLogCancel := rec.LogCancel
|
||
rec.LogCancel = nil
|
||
d.mu.Unlock()
|
||
if prevLogCancel != nil {
|
||
prevLogCancel()
|
||
}
|
||
|
||
logCtx, logCancel := context.WithCancel(context.Background())
|
||
d.mu.Lock()
|
||
rec.LogCancel = logCancel
|
||
d.mu.Unlock()
|
||
go d.streamLogs(logCtx, rec)
|
||
d.startExitWatch(rec)
|
||
// Stats poller shares the log-lifetime context: cancelled on stop,
|
||
// auto-closed if the container exits.
|
||
go d.streamStats(logCtx, rec)
|
||
}
|
||
|
||
// startExitWatch (re)arms the exit watcher for a record, single-flight:
|
||
// any watcher from a previous activation is cancelled first, so re-Announce
|
||
// on controller reconnects can't stack N watchExit goroutines each holding
|
||
// its own ContainerWait docker connection.
|
||
func (d *Dispatcher) startExitWatch(rec *instanceRecord) {
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
d.mu.Lock()
|
||
prev := rec.ExitWatchCancel
|
||
rec.ExitWatchCancel = cancel
|
||
d.mu.Unlock()
|
||
if prev != nil {
|
||
prev()
|
||
}
|
||
go d.watchExit(ctx, rec)
|
||
}
|
||
|
||
// handleStop stops the container gracefully and leaves everything else
|
||
// intact — the in-memory record, the sidecar JSON, the Docker container
|
||
// (in Exited state), the volumes, and the DB row all persist so the
|
||
// instance can be re-started, backed up, or restored. Full teardown is
|
||
// InstanceDelete's job.
|
||
func (d *Dispatcher) handleStop(ctx context.Context, corrID string, req *panelv1.InstanceStop) {
|
||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||
|
||
d.mu.Lock()
|
||
rec, ok := d.instances[req.InstanceId]
|
||
d.mu.Unlock()
|
||
if !ok {
|
||
d.replyError(corrID, req.InstanceId, "not_found", "instance not on this target")
|
||
return
|
||
}
|
||
|
||
rec.stopping.Store(true)
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPING, 0, "stopping")
|
||
grace := time.Duration(req.GraceSeconds) * time.Second
|
||
if grace == 0 {
|
||
grace = 30 * time.Second
|
||
}
|
||
// Cancel any in-flight updater (steamcmd download, github fetch).
|
||
// Otherwise Stop kills the main container but the sidecar keeps
|
||
// running, which the operator perceives as "stop didn't work — it's
|
||
// still downloading."
|
||
d.cancelUpdater(req.InstanceId)
|
||
|
||
// Graceful in-game save BEFORE signalling the container. A bare docker
|
||
// stop (SIGTERM→SIGKILL after grace) can tear a region file mid-write —
|
||
// the root cause of 7DTD .7rg "Wrong chunk header!" corruption that
|
||
// Region Medic exists to repair. If the module declares an RCON `save`
|
||
// command and the tracker is live, flush the world first. Best-effort:
|
||
// short timeout, and fall through to the normal stop on any error.
|
||
d.mu.Lock()
|
||
tracker := rec.Tracker
|
||
d.mu.Unlock()
|
||
if manifest, ok := d.modules.Get(rec.ModuleID); !req.Force && tracker != nil && ok && manifest.RCON != nil {
|
||
if saveCmd := manifest.RCON.Commands["save"]; saveCmd != "" {
|
||
saveCtx, cancelSave := context.WithTimeout(ctx, 15*time.Second)
|
||
if out, err := tracker.Exec(saveCtx, saveCmd); err != nil {
|
||
log.Warn("graceful save before stop failed (continuing to stop)", "cmd", saveCmd, "err", err)
|
||
} else {
|
||
log.Info("graceful save before stop", "cmd", saveCmd, "out", strings.TrimSpace(out))
|
||
time.Sleep(2 * time.Second) // let the engine flush region files to disk
|
||
}
|
||
cancelSave()
|
||
}
|
||
}
|
||
|
||
// Tear down the tracker, log/stats streams, and exit watcher. Mutations
|
||
// go under d.mu — activate/Announce/Shutdown touch the same fields from
|
||
// other goroutines; the cancel calls themselves happen outside the lock.
|
||
d.mu.Lock()
|
||
trackerCancel := rec.TrackerCancel
|
||
logCancel := rec.LogCancel
|
||
exitWatchCancel := rec.ExitWatchCancel
|
||
rec.TrackerCancel = nil
|
||
rec.Tracker = nil
|
||
rec.LogCancel = nil
|
||
rec.ExitWatchCancel = nil
|
||
d.mu.Unlock()
|
||
for _, c := range []context.CancelFunc{trackerCancel, logCancel, exitWatchCancel} {
|
||
if c != nil {
|
||
c()
|
||
}
|
||
}
|
||
if err := d.runtime.Stop(ctx, rec.ContainerID, grace); err != nil {
|
||
// Clear the flag on failure too — the container is still running, so
|
||
// a future exit is NOT part of a graceful stop, and handleStart's
|
||
// race guard must not wait on a stop that already gave up.
|
||
rec.stopping.Store(false)
|
||
d.replyError(corrID, req.InstanceId, "stop", err.Error())
|
||
log.Error("runtime stop", "err", err)
|
||
return
|
||
}
|
||
|
||
// Reset stopping so the exit watcher (if it fires again on a future
|
||
// restart) treats future exits normally.
|
||
rec.stopping.Store(false)
|
||
|
||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "stopped")
|
||
d.replyOK(corrID)
|
||
log.Info("instance stopped (container retained for restart/backup)")
|
||
}
|
||
|
||
// handleRCON runs one operator-supplied RCON command against the instance's
|
||
// tracker client and returns the result via an RCONResult envelope carrying
|
||
// the same correlation_id so the controller can match it to the original
|
||
// ExecRCON request.
|
||
func (d *Dispatcher) handleRCON(ctx context.Context, corrID string, req *panelv1.RCONCommand) {
|
||
d.mu.Lock()
|
||
rec, ok := d.instances[req.InstanceId]
|
||
var tracker *state.Tracker
|
||
if ok {
|
||
tracker = rec.Tracker
|
||
}
|
||
d.mu.Unlock()
|
||
if !ok {
|
||
d.sendRCONResult(corrID, "", "not_found", "instance not on this target")
|
||
return
|
||
}
|
||
if tracker == nil {
|
||
d.sendRCONResult(corrID, "", "no_tracker", "instance has no RCON tracker (module may not declare rcon)")
|
||
return
|
||
}
|
||
// Use tracker.Exec so we transparently redial on dead connections —
|
||
// Palworld's RCON closes sockets after ~60s idle, so the first operator
|
||
// command after a quiet period would otherwise always fail with EOF.
|
||
out, err := tracker.Exec(ctx, req.Command)
|
||
if err != nil {
|
||
d.sendRCONResult(corrID, out, "exec", err.Error())
|
||
return
|
||
}
|
||
d.sendRCONResult(corrID, out, "", "")
|
||
}
|
||
|
||
func (d *Dispatcher) sendRCONResult(corrID, output, _code, errMsg string) {
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
CorrelationId: corrID,
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_RconResult{
|
||
RconResult: &panelv1.RCONResult{Output: output, Error: errMsg},
|
||
},
|
||
})
|
||
}
|
||
|
||
func (d *Dispatcher) streamLogs(ctx context.Context, rec *instanceRecord) {
|
||
// Coalesce progress-style bursts (SteamCMD download ticks, preallocation,
|
||
// LGSM step banners) into at most one update per 200ms per "key". Non-
|
||
// progress lines go through immediately so errors and unique content are
|
||
// never delayed. The Tracker still sees every line for event matching.
|
||
//
|
||
// The whole thing runs in a restart loop. Reason: docker's
|
||
// ContainerLogs(--follow) can return cleanly (EOF / pipe closed) for
|
||
// reasons that have nothing to do with the container exiting — daemon
|
||
// hiccups, log driver buffer flushes, occasional plain weirdness. Before
|
||
// this loop, when StreamLogs returned without ctx.Err(), the goroutine
|
||
// just exited and the instance went silent until the next agent restart.
|
||
// Symptom in the wild: a server's pill stuck on "starting" because no
|
||
// log lines were reaching the controller's ring buffer, even though the
|
||
// container itself was fine. Live-debugged on V Rising 2026-04-29.
|
||
for {
|
||
if ctx.Err() != nil {
|
||
return
|
||
}
|
||
// Skip if the container is no longer running — watchExit owns the
|
||
// terminal "stopped" / "crashed" transition, no point chasing the
|
||
// stream of a dead container.
|
||
if rec.ContainerID == "" {
|
||
return
|
||
}
|
||
|
||
co := newLogCoalescer(200*time.Millisecond, func(stream runtime.LogStream, line string, at time.Time) {
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||
InstanceId: rec.InstanceID,
|
||
Stream: string(stream),
|
||
At: timestamppb.New(at),
|
||
Line: line,
|
||
}},
|
||
})
|
||
})
|
||
err := d.runtime.StreamLogs(ctx, rec.ContainerID, func(stream runtime.LogStream, line string, at time.Time) {
|
||
co.offer(stream, line, at)
|
||
// Snapshot the tracker under d.mu — handleStop/activate swap it
|
||
// concurrently. Uncontended mutex cost is noise next to the
|
||
// regex matching OnLogLine does.
|
||
d.mu.Lock()
|
||
tracker := rec.Tracker
|
||
d.mu.Unlock()
|
||
if tracker != nil {
|
||
tracker.OnLogLine(line)
|
||
}
|
||
// ark-sa hang-detection guardrail: fires on the engine's
|
||
// "!!!HANG DETECTED!!!" self-detection token. No-op for
|
||
// other modules. See hangguard.go.
|
||
d.arkHangGuardOnLogLine(rec, line)
|
||
})
|
||
co.stop()
|
||
|
||
if ctx.Err() != nil {
|
||
return
|
||
}
|
||
if err != nil {
|
||
d.log.Warn("log stream ended with error, will restart",
|
||
"instance_id", rec.InstanceID, "err", err)
|
||
} else {
|
||
d.log.Info("log stream returned cleanly, restarting",
|
||
"instance_id", rec.InstanceID)
|
||
}
|
||
// Brief backoff so a wedged docker daemon doesn't peg CPU on tight
|
||
// retry. 2s is plenty quick for "the operator is watching the
|
||
// pill," far short of human-perceptible silence.
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-time.After(2 * time.Second):
|
||
}
|
||
}
|
||
}
|
||
|
||
// watchExit blocks on ContainerWait and reports the container's exit (or
|
||
// crash/auto-restart cycle) upstream. ctx-bound: cancelled by startExitWatch
|
||
// on re-activation, by handleStop, and by Shutdown, so a stale watcher can't
|
||
// hold a docker connection past its container lifetime.
|
||
func (d *Dispatcher) watchExit(ctx context.Context, rec *instanceRecord) {
|
||
for {
|
||
code, err := d.runtime.Wait(ctx, rec.ContainerID)
|
||
if ctx.Err() != nil {
|
||
return
|
||
}
|
||
if err != nil {
|
||
if strings.Contains(err.Error(), "context") {
|
||
return
|
||
}
|
||
d.log.Warn("wait error", "instance_id", rec.InstanceID, "err", err)
|
||
}
|
||
if rec.stopping.Load() {
|
||
return
|
||
}
|
||
|
||
// Docker's restart policy (unless-stopped / on-failure / always) may
|
||
// relaunch the container process after the previous one dies. If the
|
||
// container is back in "running" state by the time Wait returns, this
|
||
// is a crash-then-auto-restart cycle, not a terminal exit — report the
|
||
// crash + recovery to the UI and re-arm our streams against the new
|
||
// process. Otherwise the Console goes dead and the status pill gets
|
||
// stuck on "stopped" even though the server is live again.
|
||
state, ierr := d.runtime.InspectByName(ctx, "panel-"+rec.InstanceID)
|
||
if ierr == nil && state.Status == "running" {
|
||
d.log.Warn("container auto-restarted by docker restart policy",
|
||
"instance_id", rec.InstanceID, "prior_exit_code", code)
|
||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, int32(code),
|
||
fmt.Sprintf("crashed (exit %d) — docker auto-restarted", code))
|
||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING, 0,
|
||
"running (recovered after crash)")
|
||
d.rearmStreamsAfterAutoRestart(rec)
|
||
continue
|
||
}
|
||
|
||
// Terminal exit — no auto-restart.
|
||
status := panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED
|
||
if code != 0 {
|
||
status = panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED
|
||
}
|
||
d.sendInstanceState(rec.InstanceID, status, int32(code), "exited")
|
||
return
|
||
}
|
||
}
|
||
|
||
// rearmStreamsAfterAutoRestart cancels the log + stats streams left pointing
|
||
// at the dead process and spawns fresh ones against the newly-restarted
|
||
// container. Called from watchExit on a crash-recovery cycle so the Console
|
||
// tab and stats graph don't go dark between the docker `die` and the next
|
||
// `start`. The RCON tracker (if any) is left to reconnect on its next poll —
|
||
// no explicit rearm needed.
|
||
func (d *Dispatcher) rearmStreamsAfterAutoRestart(rec *instanceRecord) {
|
||
d.mu.Lock()
|
||
if rec.LogCancel != nil {
|
||
rec.LogCancel()
|
||
}
|
||
logCtx, logCancel := context.WithCancel(context.Background())
|
||
rec.LogCancel = logCancel
|
||
d.mu.Unlock()
|
||
go d.streamLogs(logCtx, rec)
|
||
go d.streamStats(logCtx, rec)
|
||
}
|
||
|
||
func (d *Dispatcher) sendInstanceState(instanceID string, status panelv1.InstanceStatus, exitCode int32, detail string) {
|
||
// Record the last-sent status on the instance record (when one exists)
|
||
// so EmitAppState can detect a stale CRASHED that needs promotion.
|
||
d.mu.Lock()
|
||
rec := d.instances[instanceID]
|
||
d.mu.Unlock()
|
||
if rec != nil {
|
||
rec.lastStatus.Store(int32(status))
|
||
}
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_InstanceState{
|
||
InstanceState: &panelv1.InstanceStateUpdate{
|
||
InstanceId: instanceID,
|
||
Status: status,
|
||
ExitCode: exitCode,
|
||
At: timestamppb.Now(),
|
||
Detail: detail,
|
||
},
|
||
},
|
||
})
|
||
}
|
||
|
||
func (d *Dispatcher) replyOK(corrID string) {
|
||
if corrID == "" {
|
||
return
|
||
}
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
CorrelationId: corrID,
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_Result{
|
||
Result: &panelv1.CommandResult{ExitCode: 0},
|
||
},
|
||
})
|
||
}
|
||
|
||
func (d *Dispatcher) replyError(corrID, instanceID, code, msg string) {
|
||
if corrID == "" && instanceID == "" {
|
||
return
|
||
}
|
||
d.sendEnv(&panelv1.AgentEnvelope{
|
||
CorrelationId: corrID,
|
||
SentAt: timestamppb.Now(),
|
||
Payload: &panelv1.AgentEnvelope_Result{
|
||
Result: &panelv1.CommandResult{
|
||
ExitCode: 1,
|
||
Error: &panelv1.Error{
|
||
Code: code,
|
||
Message: msg,
|
||
At: timestamppb.Now(),
|
||
},
|
||
},
|
||
},
|
||
})
|
||
}
|
||
|
||
// assignNonCollidingHostPorts walks the manifest's ports list and, for
|
||
// each declared port, picks an actually-available host port — auto-
|
||
// bumping if the manifest default (or operator-supplied override) is
|
||
// already in use by another process or container. Mutates req.Ports
|
||
// in place: every declared manifest port ends up with an explicit
|
||
// HostPort entry, so downstream code (ResolveDocker, resolveRCONAddress)
|
||
// sees the same chosen port.
|
||
//
|
||
// Probe strategy: bind a real socket on 0.0.0.0:port (or 127.0.0.1:port
|
||
// for internal ports) for ~50ms; if the bind succeeds, the port is
|
||
// available — Docker's published-port machinery will succeed there too.
|
||
// Avoids guessing from /proc/net or shelling to ss/lsof.
|
||
//
|
||
// Bump direction: increment by 1 starting from the requested port up
|
||
// to +200. We intentionally don't pick from a far-flung high range —
|
||
// keeping ports clustered makes operator firewall configs easier and
|
||
// the typical "default + N siblings" workload fits in <50.
|
||
func assignNonCollidingHostPorts(manifest *modulepkg.Manifest, req *panelv1.InstanceCreate) error {
|
||
if manifest == nil || manifest.Runtime.Docker == nil {
|
||
return nil
|
||
}
|
||
// Build by-name lookup of any existing operator overrides so we can
|
||
// preserve them and only mutate when they collide.
|
||
overrideByName := map[string]*panelv1.PortMap{}
|
||
for i, ov := range req.Ports {
|
||
if ov != nil && ov.Name != "" {
|
||
overrideByName[ov.Name] = req.Ports[i]
|
||
}
|
||
}
|
||
// Track ports we've claimed during THIS create — sibling ports in the
|
||
// same module (e.g. ark-sa game/raw/rcon) all bind on this same call,
|
||
// so the probe needs to know about ports we're about to take.
|
||
claimed := map[string]bool{} // key: "tcp:7777" / "udp:7777"
|
||
// Reserve list for predictable rollback if something goes wrong mid-call.
|
||
for _, p := range manifest.Ports {
|
||
if p.Name == "" || p.Default == 0 {
|
||
continue
|
||
}
|
||
proto := strings.ToLower(p.Proto)
|
||
if proto != "tcp" && proto != "udp" {
|
||
proto = "tcp"
|
||
}
|
||
// Operator-supplied override wins as the starting point.
|
||
startHost := uint16(p.Default)
|
||
if ov, ok := overrideByName[p.Name]; ok && ov.HostPort != 0 {
|
||
startHost = uint16(ov.HostPort)
|
||
}
|
||
chosen, err := findFreePort(startHost, proto, p.Internal, claimed)
|
||
if err != nil {
|
||
return fmt.Errorf("port %q: %w", p.Name, err)
|
||
}
|
||
claimed[fmt.Sprintf("%s:%d", proto, chosen)] = true
|
||
// Update or insert the PortMap entry. ContainerPort stays at the
|
||
// manifest default (or operator override) — only HostPort is
|
||
// adjusted to dodge the collision.
|
||
containerPort := uint32(p.Default)
|
||
if ov, ok := overrideByName[p.Name]; ok && ov.ContainerPort != 0 {
|
||
containerPort = ov.ContainerPort
|
||
}
|
||
entry := &panelv1.PortMap{
|
||
Name: p.Name,
|
||
Proto: proto,
|
||
ContainerPort: containerPort,
|
||
HostPort: uint32(chosen),
|
||
Internal: p.Internal,
|
||
}
|
||
if existing, ok := overrideByName[p.Name]; ok {
|
||
*existing = *entry
|
||
} else {
|
||
req.Ports = append(req.Ports, entry)
|
||
overrideByName[p.Name] = req.Ports[len(req.Ports)-1]
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// findFreePort scans upward from start, returning the first port that
|
||
// can be bound on the appropriate interface. claimed tracks ports we've
|
||
// already chosen earlier in the same call. Caps at +200 — keeps panel-
|
||
// assigned ports in a tight cluster for operator firewall sanity.
|
||
func findFreePort(start uint16, proto string, internal bool, claimed map[string]bool) (uint16, error) {
|
||
host := "0.0.0.0"
|
||
if internal {
|
||
host = "127.0.0.1"
|
||
}
|
||
for offset := 0; offset < 200; offset++ {
|
||
port := uint32(start) + uint32(offset)
|
||
if port > 65535 {
|
||
break
|
||
}
|
||
key := fmt.Sprintf("%s:%d", proto, port)
|
||
if claimed[key] {
|
||
continue
|
||
}
|
||
addr := fmt.Sprintf("%s:%d", host, port)
|
||
if proto == "udp" {
|
||
l, err := net.ListenPacket("udp", addr)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
_ = l.Close()
|
||
return uint16(port), nil
|
||
}
|
||
l, err := net.Listen("tcp", addr)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
_ = l.Close()
|
||
return uint16(port), nil
|
||
}
|
||
return 0, fmt.Errorf("no free %s port in range %d-%d", proto, start, uint32(start)+200)
|
||
}
|
||
|
||
// resolveRCONAddress determines the 127.0.0.1:port the Agent will use to
|
||
// reach the instance's RCON and pulls the password from config_values.
|
||
// Returns ("", "") if the manifest declares no RCON.
|
||
//
|
||
// Password source precedence:
|
||
// 1. config_values["rcon_password"] — explicit override
|
||
// 2. config_values[manifest.RCON.PasswordSecret] — module-declared mapping
|
||
// (operator-set value via the Config tab)
|
||
// 3. manifest.Runtime.Docker.Env[manifest.RCON.PasswordSecret] —
|
||
// module manifest's own default for the secret-named env var (this
|
||
// is the line that fixes ARK SA: SERVER_ADMIN_PASSWORD lives in
|
||
// env defaults, never gets seeded into ConfigValues, and the
|
||
// acekorneya image doesn't write it back to GameUserSettings.ini)
|
||
// 4. manifest.RCON.PasswordLiteral — explicit literal in manifest
|
||
// 5. "" (tracker will still attempt connect; auth may fail)
|
||
func resolveRCONAddress(manifest *modulepkg.Manifest, req *panelv1.InstanceCreate) (addr, password string) {
|
||
if manifest.RCON == nil || manifest.RCON.HostPort == "" {
|
||
return "", ""
|
||
}
|
||
portDecl := manifest.Port(manifest.RCON.HostPort)
|
||
if portDecl == nil {
|
||
return "", ""
|
||
}
|
||
hostPort := uint16(portDecl.Default)
|
||
for _, ov := range req.Ports {
|
||
if ov != nil && ov.Name == manifest.RCON.HostPort && ov.HostPort != 0 {
|
||
hostPort = uint16(ov.HostPort)
|
||
break
|
||
}
|
||
}
|
||
if req.ConfigValues != nil {
|
||
if p := req.ConfigValues["rcon_password"]; p != "" {
|
||
password = p
|
||
} else if key := manifest.RCON.PasswordSecret; key != "" {
|
||
password = req.ConfigValues[key]
|
||
}
|
||
}
|
||
// Fall back to the manifest's env-defaults map when the secret
|
||
// hasn't been overridden via ConfigValues. The agent merges env
|
||
// defaults into the container's env at create time, but it does
|
||
// NOT seed ConfigValues with them — so without this, RCON sees
|
||
// an empty password and dials get RST'd at auth.
|
||
if password == "" && manifest.RCON.PasswordSecret != "" && manifest.Runtime.Docker != nil {
|
||
if v, ok := manifest.Runtime.Docker.Env[manifest.RCON.PasswordSecret]; ok {
|
||
password = v
|
||
}
|
||
}
|
||
// Last resort: explicit literal in the manifest.
|
||
if password == "" {
|
||
password = manifest.RCON.PasswordLiteral
|
||
}
|
||
return fmt.Sprintf("127.0.0.1:%d", hostPort), password
|
||
}
|
||
|
||
// resolveBrowseableRoot returns the container-absolute path the file
|
||
// manager should treat as the DEFAULT root (first view). Precedence:
|
||
// browseable_roots[0], explicit browseable_root, first declared volume's
|
||
// container path, or "" to fall back to bind mount.
|
||
func resolveBrowseableRoot(manifest *modulepkg.Manifest) string {
|
||
if manifest.Runtime.Docker == nil {
|
||
return ""
|
||
}
|
||
d := manifest.Runtime.Docker
|
||
if len(d.BrowseableRoots) > 0 && d.BrowseableRoots[0].Path != "" {
|
||
return d.BrowseableRoots[0].Path
|
||
}
|
||
if d.BrowseableRoot != "" {
|
||
return d.BrowseableRoot
|
||
}
|
||
if len(d.Volumes) > 0 {
|
||
return d.Volumes[0].Container
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// resolveBrowseableRoots returns every root the operator is allowed to
|
||
// browse. Guarantees at least one entry (synthesized from browseable_root
|
||
// if the module only declares the singular form).
|
||
func resolveBrowseableRoots(manifest *modulepkg.Manifest) []modulepkg.BrowseableRoot {
|
||
if manifest.Runtime.Docker == nil {
|
||
return nil
|
||
}
|
||
d := manifest.Runtime.Docker
|
||
if len(d.BrowseableRoots) > 0 {
|
||
return d.BrowseableRoots
|
||
}
|
||
if d.BrowseableRoot != "" {
|
||
return []modulepkg.BrowseableRoot{{Name: "default", Path: d.BrowseableRoot}}
|
||
}
|
||
if len(d.Volumes) > 0 {
|
||
return []modulepkg.BrowseableRoot{{Name: "default", Path: d.Volumes[0].Container}}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// mergeValuesAndSecrets generates any module-declared secrets that are
|
||
// flagged Generated, then layers user-supplied config_values on top
|
||
// (user wins). Returns a fresh map suitable for template rendering.
|
||
func mergeValuesAndSecrets(manifest *modulepkg.Manifest, userValues map[string]string) (map[string]string, error) {
|
||
secrets, err := modulepkg.GenerateSecrets(manifest)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make(map[string]string, len(secrets)+len(userValues))
|
||
for k, v := range secrets {
|
||
out[k] = v
|
||
}
|
||
for k, v := range userValues {
|
||
if v != "" {
|
||
out[k] = v
|
||
}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// rtStatusToProto converts a Docker container State.Status to our
|
||
// InstanceStatus enum. Unknown states map to UNSPECIFIED.
|
||
func rtStatusToProto(s string) panelv1.InstanceStatus {
|
||
switch s {
|
||
case "running":
|
||
return panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING
|
||
case "restarting":
|
||
return panelv1.InstanceStatus_INSTANCE_STATUS_STARTING
|
||
case "created":
|
||
return panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED
|
||
case "paused":
|
||
return panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING
|
||
case "exited":
|
||
return panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED
|
||
case "dead":
|
||
return panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED
|
||
default:
|
||
return panelv1.InstanceStatus_INSTANCE_STATUS_UNSPECIFIED
|
||
}
|
||
}
|
||
|
||
func short(id string) string {
|
||
if len(id) > 12 {
|
||
return id[:12]
|
||
}
|
||
return id
|
||
}
|