package dispatch import ( "context" "fmt" "strings" "sync" "sync/atomic" "time" "google.golang.org/protobuf/types/known/timestamppb" panelv1 "github.com/dbledeez/panel/proto/panel/v1" ) // hangGuard implements a belt-and-suspenders hang-detection guardrail for // ARK Survival Ascended (ark-sa) instances running under Wine 9. // The Wine 9 RCON listener can wedge under sustained traffic; the engine // also sometimes emits a "!!!HANG DETECTED!!!" line on its own. // This guard auto-restarts the docker container in either case, with a // cooldown to prevent flapping. // The root-cause fix (docker_exec_rcon adapter) shipped 2026-05-03; this // guard handles the rare residual cases. // It replaces the bash watchdog that previously ran on princess via systemd timer. const ( hangGuardWindow = 10 * time.Minute hangGuardFailThreshold = 5 hangGuardCooldown = 10 * time.Minute hangGuardEngineMarker = "!!!HANG DETECTED!!!" ) // hangGuardState holds per-instance hang-detection state. // Embed by value in instanceRecord; zero value is ready to use. type hangGuardState struct { mu sync.Mutex consecutiveFails int windowStart time.Time lastRestart time.Time restarting atomic.Bool } // arkHangGuardOnLogLine checks for engine-side hang detection tokens // in console log lines. Only applies to ark-sa instances. func (d *Dispatcher) arkHangGuardOnLogLine(rec *instanceRecord, line string) { if rec.ModuleID != "ark-sa" { return } if !strings.Contains(line, hangGuardEngineMarker) { return } // Engine self-detected a hang; trigger restart asynchronously. go d.arkHangGuardRestart(rec, fmt.Sprintf("engine emitted '%s'", hangGuardEngineMarker)) } // arkHangGuardOnRconResult tracks RCON call successes/failures for // hang detection. A threshold of consecutive failures triggers a restart. func (d *Dispatcher) arkHangGuardOnRconResult(rec *instanceRecord, success bool) { if rec.ModuleID != "ark-sa" { return } rec.hangGuard.mu.Lock() defer rec.hangGuard.mu.Unlock() if success { rec.hangGuard.consecutiveFails = 0 rec.hangGuard.windowStart = time.Time{} return } now := time.Now() if rec.hangGuard.windowStart.IsZero() || now.Sub(rec.hangGuard.windowStart) > hangGuardWindow { // Start a new window rec.hangGuard.windowStart = now rec.hangGuard.consecutiveFails = 1 } else { rec.hangGuard.consecutiveFails++ } if rec.hangGuard.consecutiveFails >= hangGuardFailThreshold { go d.arkHangGuardRestart(rec, fmt.Sprintf("%d RCON failures in last %v", rec.hangGuard.consecutiveFails, hangGuardWindow)) } } // arkHangGuardRestart performs the actual container restart, respecting a // cooldown and using a single-flight pattern to prevent concurrent restarts. func (d *Dispatcher) arkHangGuardRestart(rec *instanceRecord, reason string) { // Single-flight: only one restart at a time per instance. if !rec.hangGuard.restarting.CompareAndSwap(false, true) { return } defer rec.hangGuard.restarting.Store(false) // Lock only for cooldown check and counter reset. We do NOT set // lastRestart here — only after a successful runtime.Restart, so a // failed restart attempt doesn't lock out retries for the full // cooldown (would block the next legitimate hang signal). Resetting // consecutiveFails / windowStart up front is fine: the bounce is // either about to happen (success path will set lastRestart) or has // failed (caller logged + emitted; future signals will re-arm). rec.hangGuard.mu.Lock() if !rec.hangGuard.lastRestart.IsZero() && time.Since(rec.hangGuard.lastRestart) < hangGuardCooldown { remaining := hangGuardCooldown - time.Since(rec.hangGuard.lastRestart) d.log.Debug("hang-guard skipped (cooldown)", "instance_id", rec.InstanceID, "cooldown_remaining", remaining, ) rec.hangGuard.mu.Unlock() return } rec.hangGuard.consecutiveFails = 0 rec.hangGuard.windowStart = time.Time{} rec.hangGuard.mu.Unlock() d.log.Warn("hang-guard restarting container", "instance_id", rec.InstanceID, "container_id", rec.ContainerID, "reason", reason, ) // Emit a panel log line announcing the restart. d.sendEnv(&panelv1.AgentEnvelope{ SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{ InstanceId: rec.InstanceID, Stream: "stderr", At: timestamppb.Now(), Line: "[panel] HANG-GUARD: " + reason, }}, }) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() // 2026-05-25: bumped grace 5s → 30s. The hang-guard's bounce can // fire mid-SaveWorld; the 5s window made it a near-guarantee that the // SIGKILL would land inside the SQLite-backed save's flush — same // torn-save pattern that the memory guardrail was producing. 30s lets // most in-flight saves finish before the engine dies; if the engine is // truly wedged it'll get SIGKILL'd after the 30s anyway. See // stats.go's emergencyStopGracePeriod for the matching rationale. if err := d.runtime.Restart(ctx, rec.ContainerID, 30*time.Second); err != nil { d.log.Error("hang-guard restart failed", "instance_id", rec.InstanceID, "err", err, ) d.sendEnv(&panelv1.AgentEnvelope{ SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{ InstanceId: rec.InstanceID, Stream: "stderr", At: timestamppb.Now(), Line: "[panel] HANG-GUARD restart failed: " + err.Error(), }}, }) return } // Successful restart — start the cooldown clock so we don't bounce // the same instance again for hangGuardCooldown. A failed restart // (above branch returned early) intentionally leaves lastRestart // untouched so the next hang signal can retry. rec.hangGuard.mu.Lock() rec.hangGuard.lastRestart = time.Now() rec.hangGuard.mu.Unlock() d.log.Info("hang-guard restart issued", "instance_id", rec.InstanceID, ) d.sendEnv(&panelv1.AgentEnvelope{ SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{ InstanceId: rec.InstanceID, Stream: "stderr", At: timestamppb.Now(), Line: "[panel] HANG-GUARD restart issued", }}, }) }