panel — open-source game server manager (public release)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-14 19:19:43 -07:00
commit 5232609719
2160 changed files with 300415 additions and 0 deletions
@@ -0,0 +1,180 @@
+package dispatch
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// hangGuard implements a belt-and-suspenders hang-detection guardrail for
+// ARK Survival Ascended (ark-sa) instances running under Wine 9.
+// The Wine 9 RCON listener can wedge under sustained traffic; the engine
+// also sometimes emits a "!!!HANG DETECTED!!!" line on its own.
+// This guard auto-restarts the docker container in either case, with a
+// cooldown to prevent flapping.
+// The root-cause fix (docker_exec_rcon adapter) shipped 2026-05-03; this
+// guard handles the rare residual cases.
+// It replaces the bash watchdog that previously ran on princess via systemd timer.
+
+const (
+	hangGuardWindow        = 10 * time.Minute
+	hangGuardFailThreshold = 5
+	hangGuardCooldown      = 10 * time.Minute
+	hangGuardEngineMarker  = "!!!HANG DETECTED!!!"
+)
+
+// hangGuardState holds per-instance hang-detection state.
+// Embed by value in instanceRecord; zero value is ready to use.
+type hangGuardState struct {
+	mu              sync.Mutex
+	consecutiveFails int
+	windowStart     time.Time
+	lastRestart     time.Time
+	restarting      atomic.Bool
+}
+
+// arkHangGuardOnLogLine checks for engine-side hang detection tokens
+// in console log lines. Only applies to ark-sa instances.
+func (d *Dispatcher) arkHangGuardOnLogLine(rec *instanceRecord, line string) {
+	if rec.ModuleID != "ark-sa" {
+		return
+	}
+	if !strings.Contains(line, hangGuardEngineMarker) {
+		return
+	}
+	// Engine self-detected a hang; trigger restart asynchronously.
+	go d.arkHangGuardRestart(rec, fmt.Sprintf("engine emitted '%s'", hangGuardEngineMarker))
+}
+
+// arkHangGuardOnRconResult tracks RCON call successes/failures for
+// hang detection. A threshold of consecutive failures triggers a restart.
+func (d *Dispatcher) arkHangGuardOnRconResult(rec *instanceRecord, success bool) {
+	if rec.ModuleID != "ark-sa" {
+		return
+	}
+
+	rec.hangGuard.mu.Lock()
+	defer rec.hangGuard.mu.Unlock()
+
+	if success {
+		rec.hangGuard.consecutiveFails = 0
+		rec.hangGuard.windowStart = time.Time{}
+		return
+	}
+
+	now := time.Now()
+	if rec.hangGuard.windowStart.IsZero() || now.Sub(rec.hangGuard.windowStart) > hangGuardWindow {
+		// Start a new window
+		rec.hangGuard.windowStart = now
+		rec.hangGuard.consecutiveFails = 1
+	} else {
+		rec.hangGuard.consecutiveFails++
+	}
+
+	if rec.hangGuard.consecutiveFails >= hangGuardFailThreshold {
+		go d.arkHangGuardRestart(rec, fmt.Sprintf("%d RCON failures in last %v", rec.hangGuard.consecutiveFails, hangGuardWindow))
+	}
+}
+
+// arkHangGuardRestart performs the actual container restart, respecting a
+// cooldown and using a single-flight pattern to prevent concurrent restarts.
+func (d *Dispatcher) arkHangGuardRestart(rec *instanceRecord, reason string) {
+	// Single-flight: only one restart at a time per instance.
+	if !rec.hangGuard.restarting.CompareAndSwap(false, true) {
+		return
+	}
+	defer rec.hangGuard.restarting.Store(false)
+
+	// Lock only for cooldown check and counter reset. We do NOT set
+	// lastRestart here — only after a successful runtime.Restart, so a
+	// failed restart attempt doesn't lock out retries for the full
+	// cooldown (would block the next legitimate hang signal). Resetting
+	// consecutiveFails / windowStart up front is fine: the bounce is
+	// either about to happen (success path will set lastRestart) or has
+	// failed (caller logged + emitted; future signals will re-arm).
+	rec.hangGuard.mu.Lock()
+	if !rec.hangGuard.lastRestart.IsZero() && time.Since(rec.hangGuard.lastRestart) < hangGuardCooldown {
+		remaining := hangGuardCooldown - time.Since(rec.hangGuard.lastRestart)
+		d.log.Debug("hang-guard skipped (cooldown)",
+			"instance_id", rec.InstanceID,
+			"cooldown_remaining", remaining,
+		)
+		rec.hangGuard.mu.Unlock()
+		return
+	}
+	rec.hangGuard.consecutiveFails = 0
+	rec.hangGuard.windowStart = time.Time{}
+	rec.hangGuard.mu.Unlock()
+
+	d.log.Warn("hang-guard restarting container",
+		"instance_id", rec.InstanceID,
+		"container_id", rec.ContainerID,
+		"reason", reason,
+	)
+
+	// Emit a panel log line announcing the restart.
+	d.sendEnv(&panelv1.AgentEnvelope{
+		SentAt: timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+			InstanceId: rec.InstanceID,
+			Stream:     "stderr",
+			At:         timestamppb.Now(),
+			Line:       "[panel] HANG-GUARD: " + reason,
+		}},
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// 2026-05-25: bumped grace 5s → 30s. The hang-guard's bounce can
+	// fire mid-SaveWorld; the 5s window made it a near-guarantee that the
+	// SIGKILL would land inside the SQLite-backed save's flush — same
+	// torn-save pattern that the memory guardrail was producing. 30s lets
+	// most in-flight saves finish before the engine dies; if the engine is
+	// truly wedged it'll get SIGKILL'd after the 30s anyway. See
+	// stats.go's emergencyStopGracePeriod for the matching rationale.
+	if err := d.runtime.Restart(ctx, rec.ContainerID, 30*time.Second); err != nil {
+		d.log.Error("hang-guard restart failed",
+			"instance_id", rec.InstanceID,
+			"err", err,
+		)
+		d.sendEnv(&panelv1.AgentEnvelope{
+			SentAt: timestamppb.Now(),
+			Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+				InstanceId: rec.InstanceID,
+				Stream:     "stderr",
+				At:         timestamppb.Now(),
+				Line:       "[panel] HANG-GUARD restart failed: " + err.Error(),
+			}},
+		})
+		return
+	}
+
+	// Successful restart — start the cooldown clock so we don't bounce
+	// the same instance again for hangGuardCooldown. A failed restart
+	// (above branch returned early) intentionally leaves lastRestart
+	// untouched so the next hang signal can retry.
+	rec.hangGuard.mu.Lock()
+	rec.hangGuard.lastRestart = time.Now()
+	rec.hangGuard.mu.Unlock()
+
+	d.log.Info("hang-guard restart issued",
+		"instance_id", rec.InstanceID,
+	)
+	d.sendEnv(&panelv1.AgentEnvelope{
+		SentAt: timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+			InstanceId: rec.InstanceID,
+			Stream:     "stderr",
+			At:         timestamppb.Now(),
+			Line:       "[panel] HANG-GUARD restart issued",
+		}},
+	})
+}