panel/agent/internal/dispatch/hangguard.go

package dispatch

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"google.golang.org/protobuf/types/known/timestamppb"

	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
)

// hangGuard implements a belt-and-suspenders hang-detection guardrail for
// ARK Survival Ascended (ark-sa) instances running under Wine 9.
// The Wine 9 RCON listener can wedge under sustained traffic; the engine
// also sometimes emits a "!!!HANG DETECTED!!!" line on its own.
// This guard auto-restarts the docker container in either case, with a
// cooldown to prevent flapping.
// The root-cause fix (docker_exec_rcon adapter) shipped 2026-05-03; this
// guard handles the rare residual cases.
// It replaces the bash watchdog that previously ran on princess via systemd timer.

const (
	hangGuardWindow        = 10 * time.Minute
	hangGuardFailThreshold = 5
	hangGuardCooldown      = 10 * time.Minute
	hangGuardEngineMarker  = "!!!HANG DETECTED!!!"
)

// hangGuardState holds per-instance hang-detection state.
// Embed by value in instanceRecord; zero value is ready to use.
type hangGuardState struct {
	mu              sync.Mutex
	consecutiveFails int
	windowStart     time.Time
	lastRestart     time.Time
	restarting      atomic.Bool
}

// arkHangGuardOnLogLine checks for engine-side hang detection tokens
// in console log lines. Only applies to ark-sa instances.
func (d *Dispatcher) arkHangGuardOnLogLine(rec *instanceRecord, line string) {
	if rec.ModuleID != "ark-sa" {
		return
	}
	if !strings.Contains(line, hangGuardEngineMarker) {
		return
	}
	// Engine self-detected a hang; trigger restart asynchronously.
	go d.arkHangGuardRestart(rec, fmt.Sprintf("engine emitted '%s'", hangGuardEngineMarker))
}

// arkHangGuardOnRconResult tracks RCON call successes/failures for
// hang detection. A threshold of consecutive failures triggers a restart.
func (d *Dispatcher) arkHangGuardOnRconResult(rec *instanceRecord, success bool) {
	if rec.ModuleID != "ark-sa" {
		return
	}

	rec.hangGuard.mu.Lock()
	defer rec.hangGuard.mu.Unlock()

	if success {
		rec.hangGuard.consecutiveFails = 0
		rec.hangGuard.windowStart = time.Time{}
		return
	}

	now := time.Now()
	if rec.hangGuard.windowStart.IsZero() || now.Sub(rec.hangGuard.windowStart) > hangGuardWindow {
		// Start a new window
		rec.hangGuard.windowStart = now
		rec.hangGuard.consecutiveFails = 1
	} else {
		rec.hangGuard.consecutiveFails++
	}

	if rec.hangGuard.consecutiveFails >= hangGuardFailThreshold {
		go d.arkHangGuardRestart(rec, fmt.Sprintf("%d RCON failures in last %v", rec.hangGuard.consecutiveFails, hangGuardWindow))
	}
}

// arkHangGuardRestart performs the actual container restart, respecting a
// cooldown and using a single-flight pattern to prevent concurrent restarts.
func (d *Dispatcher) arkHangGuardRestart(rec *instanceRecord, reason string) {
	// Single-flight: only one restart at a time per instance.
	if !rec.hangGuard.restarting.CompareAndSwap(false, true) {
		return
	}
	defer rec.hangGuard.restarting.Store(false)

	// Lock only for cooldown check and counter reset. We do NOT set
	// lastRestart here — only after a successful runtime.Restart, so a
	// failed restart attempt doesn't lock out retries for the full
	// cooldown (would block the next legitimate hang signal). Resetting
	// consecutiveFails / windowStart up front is fine: the bounce is
	// either about to happen (success path will set lastRestart) or has
	// failed (caller logged + emitted; future signals will re-arm).
	rec.hangGuard.mu.Lock()
	if !rec.hangGuard.lastRestart.IsZero() && time.Since(rec.hangGuard.lastRestart) < hangGuardCooldown {
		remaining := hangGuardCooldown - time.Since(rec.hangGuard.lastRestart)
		d.log.Debug("hang-guard skipped (cooldown)",
			"instance_id", rec.InstanceID,
			"cooldown_remaining", remaining,
		)
		rec.hangGuard.mu.Unlock()
		return
	}
	rec.hangGuard.consecutiveFails = 0
	rec.hangGuard.windowStart = time.Time{}
	rec.hangGuard.mu.Unlock()

	d.log.Warn("hang-guard restarting container",
		"instance_id", rec.InstanceID,
		"container_id", rec.ContainerID,
		"reason", reason,
	)

	// Emit a panel log line announcing the restart.
	d.sendEnv(&panelv1.AgentEnvelope{
		SentAt: timestamppb.Now(),
		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
			InstanceId: rec.InstanceID,
			Stream:     "stderr",
			At:         timestamppb.Now(),
			Line:       "[panel] HANG-GUARD: " + reason,
		}},
	})

	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()

	// 2026-05-25: bumped grace 5s → 30s. The hang-guard's bounce can
	// fire mid-SaveWorld; the 5s window made it a near-guarantee that the
	// SIGKILL would land inside the SQLite-backed save's flush — same
	// torn-save pattern that the memory guardrail was producing. 30s lets
	// most in-flight saves finish before the engine dies; if the engine is
	// truly wedged it'll get SIGKILL'd after the 30s anyway. See
	// stats.go's emergencyStopGracePeriod for the matching rationale.
	if err := d.runtime.Restart(ctx, rec.ContainerID, 30*time.Second); err != nil {
		d.log.Error("hang-guard restart failed",
			"instance_id", rec.InstanceID,
			"err", err,
		)
		d.sendEnv(&panelv1.AgentEnvelope{
			SentAt: timestamppb.Now(),
			Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
				InstanceId: rec.InstanceID,
				Stream:     "stderr",
				At:         timestamppb.Now(),
				Line:       "[panel] HANG-GUARD restart failed: " + err.Error(),
			}},
		})
		return
	}

	// Successful restart — start the cooldown clock so we don't bounce
	// the same instance again for hangGuardCooldown. A failed restart
	// (above branch returned early) intentionally leaves lastRestart
	// untouched so the next hang signal can retry.
	rec.hangGuard.mu.Lock()
	rec.hangGuard.lastRestart = time.Now()
	rec.hangGuard.mu.Unlock()

	d.log.Info("hang-guard restart issued",
		"instance_id", rec.InstanceID,
	)
	d.sendEnv(&panelv1.AgentEnvelope{
		SentAt: timestamppb.Now(),
		Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
			InstanceId: rec.InstanceID,
			Stream:     "stderr",
			At:         timestamppb.Now(),
			Line:       "[panel] HANG-GUARD restart issued",
		}},
	})
}