panel v0.9.1 — open-source game server manager
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,535 @@
|
||||
// Package state runs the per-instance state-tracking pipeline:
|
||||
// - RCON poll loops that execute manifest-declared commands and parse the
|
||||
// output into AppStateUpdate messages (players online, etc.)
|
||||
// - log-line event matching that runs manifest-declared regexes against
|
||||
// each log line and emits PlayerEvent messages (join/leave/chat/death)
|
||||
//
|
||||
// A Tracker is owned by the dispatcher for the lifetime of a running
|
||||
// instance. It is stopped by cancelling the context passed to Run.
|
||||
package state
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/rcon"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Emitter is the upward path for tracker output. Implementations forward
|
||||
// messages onto the gRPC stream back to the Controller.
|
||||
type Emitter interface {
|
||||
EmitAppState(*panelv1.AppStateUpdate)
|
||||
EmitPlayerEvent(*panelv1.PlayerEvent)
|
||||
}
|
||||
|
||||
// Config wires the tracker to an instance + its RCON endpoint.
|
||||
type Config struct {
|
||||
InstanceID string
|
||||
Manifest *modulepkg.Manifest
|
||||
RCONAddr string // "host:port" — required for tcp adapters (telnet, source_rcon)
|
||||
RCONPassword string
|
||||
// PasswordFunc, if set, overrides RCONPassword on each dial attempt.
|
||||
// Used when the module's entrypoint generates an RCON secret at first
|
||||
// boot (e.g. 7DTD's TelnetPassword in /game-saves/.panel-telnet-password).
|
||||
// Called with the dial context; the returned string is used as the
|
||||
// password for that single attempt.
|
||||
PasswordFunc func(ctx context.Context) (string, error)
|
||||
Emitter Emitter
|
||||
// ContainerID is the Docker container name/ID for the stdio adapter.
|
||||
// Ignored by tcp adapters. The stdio adapter calls Stdio.AttachStdio
|
||||
// with this id to get a stdin/stdout handle to the running game.
|
||||
ContainerID string
|
||||
// Stdio is the backend that knows how to attach to a container's
|
||||
// stdio. Required for stdio adapter, nil otherwise.
|
||||
Stdio rcon.StdioBackend
|
||||
// OnRconResult is an optional callback invoked after every RCON
|
||||
// exec attempt (poll loop or operator-driven). success=true means
|
||||
// the engine answered; success=false means the exec failed (dial,
|
||||
// timeout, dead conn). Used by the dispatcher's arkHangGuard to
|
||||
// count consecutive failures and trigger an auto-restart when the
|
||||
// Wine 9 listener wedges. Nil-safe: callers should check before invoke.
|
||||
OnRconResult func(success bool)
|
||||
}
|
||||
|
||||
// Tracker is created at instance start time and run in a goroutine.
|
||||
type Tracker struct {
|
||||
log *slog.Logger
|
||||
cfg Config
|
||||
events []compiledEvent
|
||||
// stateRE holds the state-source parse regexes, compiled once at New so
|
||||
// a bad pattern surfaces as a construction error instead of silently
|
||||
// recompiling (and silently failing) on every poll tick. Keyed by the
|
||||
// pattern string — pollOnce receives the StateSource by value, so the
|
||||
// pattern is the stable identity.
|
||||
stateRE map[string]*regexp.Regexp
|
||||
|
||||
mu sync.Mutex
|
||||
client rcon.Client
|
||||
}
|
||||
|
||||
type compiledEvent struct {
|
||||
name string
|
||||
re *regexp.Regexp
|
||||
kind panelv1.PlayerEvent_Kind
|
||||
}
|
||||
|
||||
// New constructs a Tracker. Event patterns are compiled now so config errors
|
||||
// surface before Run is called.
|
||||
func New(log *slog.Logger, cfg Config) (*Tracker, error) {
|
||||
if cfg.Manifest == nil {
|
||||
return nil, errors.New("manifest is required")
|
||||
}
|
||||
if cfg.Emitter == nil {
|
||||
return nil, errors.New("emitter is required")
|
||||
}
|
||||
events, err := compileEvents(cfg.Manifest.Events)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Compile state-source parse patterns up front — a bad pattern is a
|
||||
// manifest bug and should fail loudly at init, not degrade into a
|
||||
// per-poll recompile that silently yields no state updates.
|
||||
stateRE := map[string]*regexp.Regexp{}
|
||||
for i := range cfg.Manifest.StateSources {
|
||||
p := cfg.Manifest.StateSources[i].Parse
|
||||
if p == nil || p.Kind != "regex" || p.Pattern == "" {
|
||||
continue
|
||||
}
|
||||
re, err := regexp.Compile(p.Pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("state source %d: compile pattern: %w", i, err)
|
||||
}
|
||||
stateRE[p.Pattern] = re
|
||||
}
|
||||
return &Tracker{
|
||||
log: log.With("instance_id", cfg.InstanceID, "component", "state"),
|
||||
cfg: cfg,
|
||||
events: events,
|
||||
stateRE: stateRE,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Client returns the currently-connected RCON client, or nil if the tracker
|
||||
// hasn't finished dialing (or the manifest has no RCON at all). Used by the
|
||||
// dispatcher to handle ad-hoc RCON commands from operators.
|
||||
func (t *Tracker) Client() rcon.Client {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.client
|
||||
}
|
||||
|
||||
// Exec runs one operator command via the tracker's cached client. On a dead
|
||||
// connection (EOF, broken pipe — Palworld closes RCON after 60s idle) we
|
||||
// redial once and retry, so operators don't see the 60s-idle EOF in the UI.
|
||||
// Returns ("", "not_connected", ...) if the tracker hasn't finished dialing.
|
||||
func (t *Tracker) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
t.mu.Lock()
|
||||
client := t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return "", errors.New("RCON not yet connected; try again in a moment")
|
||||
}
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := client.Exec(execCtx, cmd)
|
||||
if err == nil || !isDeadConnErr(err) {
|
||||
return out, err
|
||||
}
|
||||
t.redial(ctx)
|
||||
t.mu.Lock()
|
||||
client = t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return out, err
|
||||
}
|
||||
retryCtx, retryCancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer retryCancel()
|
||||
return client.Exec(retryCtx, cmd)
|
||||
}
|
||||
|
||||
// OnLogLine runs each log line through compiled event patterns and emits a
|
||||
// PlayerEvent on the first match. Called from the dispatcher's log pump; must
|
||||
// be fast and non-blocking.
|
||||
func (t *Tracker) OnLogLine(line string) {
|
||||
for _, ev := range t.events {
|
||||
m := ev.re.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
name := firstGroup(ev.re, m, "name", "player_name")
|
||||
id := firstGroup(ev.re, m, "platform_id", "owner", "cross_id", "id", "player_id")
|
||||
detail := firstGroup(ev.re, m, "msg", "detail", "reason")
|
||||
t.cfg.Emitter.EmitPlayerEvent(&panelv1.PlayerEvent{
|
||||
InstanceId: t.cfg.InstanceID,
|
||||
Kind: ev.kind,
|
||||
PlayerName: name,
|
||||
PlayerId: id,
|
||||
Detail: detail,
|
||||
At: timestamppb.Now(),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Run connects to RCON (with retries) and starts one goroutine per declared
|
||||
// RCON state source. Returns when ctx is cancelled.
|
||||
//
|
||||
// If the manifest has no RCON config at all, Run is a no-op that blocks on
|
||||
// ctx.Done. If it has RCON but no state sources, we still dial once so
|
||||
// operator ad-hoc commands from the UI work — state-source poll loops just
|
||||
// don't spawn.
|
||||
func (t *Tracker) Run(ctx context.Context) {
|
||||
if t.cfg.Manifest.RCON == nil {
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
|
||||
client, err := t.dialWithRetries(ctx)
|
||||
if err != nil {
|
||||
// Context cancelled before we could connect; nothing to do.
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.client = client
|
||||
t.mu.Unlock()
|
||||
defer func() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.client != nil {
|
||||
_ = t.client.Close()
|
||||
t.client = nil
|
||||
}
|
||||
}()
|
||||
|
||||
t.log.Info("rcon connected", "addr", t.cfg.RCONAddr, "adapter", t.cfg.Manifest.RCON.Adapter)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range t.cfg.Manifest.StateSources {
|
||||
ss := t.cfg.Manifest.StateSources[i]
|
||||
if ss.Type != "rcon" || ss.Every.Std() <= 0 {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
t.runRCONPoller(ctx, ss)
|
||||
}()
|
||||
}
|
||||
// Block until the tracker is cancelled so the defer (which closes the
|
||||
// rcon client) fires on teardown, not immediately. With no rcon state
|
||||
// sources wg.Wait() returns instantly — without this extra wait the
|
||||
// client would vanish right after Run dialed, making ad-hoc commands
|
||||
// from the UI fail with "RCON not yet connected" forever.
|
||||
<-ctx.Done()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (t *Tracker) hasRCONStateSource() bool {
|
||||
for _, ss := range t.cfg.Manifest.StateSources {
|
||||
if ss.Type == "rcon" && ss.Every.Std() > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *Tracker) dialWithRetries(ctx context.Context) (rcon.Client, error) {
|
||||
rc := t.cfg.Manifest.RCON
|
||||
dialer, err := rcon.DialerFor(rc.Adapter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// stdio has no host/port — it attaches to the running container's stdin.
|
||||
// We still retry-loop because the container may not be up yet when the
|
||||
// tracker starts (race between Start + activate).
|
||||
if rc.Adapter == "stdio" {
|
||||
backoff := 2 * time.Second
|
||||
for {
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
client, err := dialer.Dial(dialCtx, rcon.DialOptions{
|
||||
ContainerID: t.cfg.ContainerID,
|
||||
Stdio: t.cfg.Stdio,
|
||||
})
|
||||
cancel()
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
t.log.Warn("stdio attach failed, retrying", "err", err, "backoff", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 30*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
host, portStr, err := net.SplitHostPort(t.cfg.RCONAddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rcon addr %q: %w", t.cfg.RCONAddr, err)
|
||||
}
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rcon port %q: %w", portStr, err)
|
||||
}
|
||||
|
||||
// Dial-retry backoff capped at 8s (not 30s) — during the post-Start
|
||||
// warm-up window the server's RCON flips between responsive and slow
|
||||
// every few seconds, and a 32s ceiling meant one unlucky timeout
|
||||
// stranded the panel for half a minute before the next retry. Cap
|
||||
// at 8s so ragnarok/etc. get back onto RCON within a few seconds of
|
||||
// ASA actually being ready, without hammering the port on a truly
|
||||
// dead server (8s × a few retries is ~30s before the tracker gives
|
||||
// up per-cycle, which the agent's outer loop still reschedules).
|
||||
backoff := 2 * time.Second
|
||||
for {
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
// Resolve password per-attempt when PasswordFunc is set — the
|
||||
// module's entrypoint may be mid-generation on first boot, so re-
|
||||
// reading on each retry lets us pick up the final value as soon
|
||||
// as it's written.
|
||||
pw := t.cfg.RCONPassword
|
||||
if t.cfg.PasswordFunc != nil {
|
||||
if p, err := t.cfg.PasswordFunc(dialCtx); err == nil && p != "" {
|
||||
pw = p
|
||||
}
|
||||
}
|
||||
client, err := dialer.Dial(dialCtx, rcon.DialOptions{
|
||||
Host: host,
|
||||
Port: port,
|
||||
Password: pw,
|
||||
ConnectTimeout: 5 * time.Second,
|
||||
ContainerID: t.cfg.ContainerID,
|
||||
})
|
||||
cancel()
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
t.log.Warn("rcon dial failed, retrying", "err", err, "backoff", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 8*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) runRCONPoller(ctx context.Context, ss modulepkg.StateSource) {
|
||||
every := ss.Every.Std()
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
t.pollOnce(ctx, ss) // immediate first tick
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
t.pollOnce(ctx, ss)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) pollOnce(ctx context.Context, ss modulepkg.StateSource) {
|
||||
t.mu.Lock()
|
||||
client := t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return
|
||||
}
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := client.Exec(execCtx, ss.Command)
|
||||
if err != nil {
|
||||
t.log.Warn("rcon exec failed", "cmd", ss.Command, "err", err)
|
||||
if t.cfg.OnRconResult != nil {
|
||||
t.cfg.OnRconResult(false)
|
||||
}
|
||||
if isDeadConnErr(err) {
|
||||
t.redial(ctx)
|
||||
}
|
||||
return
|
||||
}
|
||||
if t.cfg.OnRconResult != nil {
|
||||
t.cfg.OnRconResult(true)
|
||||
}
|
||||
app := t.parseState(out, ss)
|
||||
if app == nil {
|
||||
// Parser yielded no structured fields (regex didn't match this game's
|
||||
// output, fields all empty, etc.) but the RCON exec succeeded — that
|
||||
// alone is proof the server is responsive. Emit a bare update keyed
|
||||
// by InstanceID so the dispatcher can use it to promote a stale
|
||||
// CRASHED status back to RUNNING.
|
||||
app = &panelv1.AppStateUpdate{InstanceId: t.cfg.InstanceID}
|
||||
}
|
||||
t.cfg.Emitter.EmitAppState(app)
|
||||
}
|
||||
|
||||
// isDeadConnErr returns true for errors that indicate the RCON socket is no
|
||||
// longer usable (EOF, connection reset, broken pipe). Palworld's RCON, for
|
||||
// instance, closes the socket after ~60s idle — any subsequent Exec fails
|
||||
// with EOF and stays broken until we redial.
|
||||
func isDeadConnErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
return strings.Contains(s, "EOF") ||
|
||||
strings.Contains(s, "broken pipe") ||
|
||||
strings.Contains(s, "connection reset") ||
|
||||
strings.Contains(s, "forcibly closed") ||
|
||||
strings.Contains(s, "use of closed network connection")
|
||||
}
|
||||
|
||||
// redial closes the current client and dials a fresh one. Runs inline with
|
||||
// the poll loop so the next tick gets a working client. On failure we leave
|
||||
// t.client = nil and the next poll will call us again.
|
||||
func (t *Tracker) redial(ctx context.Context) {
|
||||
t.mu.Lock()
|
||||
if t.client != nil {
|
||||
_ = t.client.Close()
|
||||
t.client = nil
|
||||
}
|
||||
t.mu.Unlock()
|
||||
t.log.Info("rcon redialing after dead connection")
|
||||
fresh, err := t.dialWithRetries(ctx)
|
||||
if err != nil {
|
||||
t.log.Warn("rcon redial failed", "err", err)
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.client = fresh
|
||||
t.mu.Unlock()
|
||||
t.log.Info("rcon reconnected", "addr", t.cfg.RCONAddr, "adapter", t.cfg.Manifest.RCON.Adapter)
|
||||
}
|
||||
|
||||
// parseState runs the configured parser against raw RCON output using the
|
||||
// regex precompiled at New, returning an AppStateUpdate or nil if nothing
|
||||
// matched.
|
||||
func (t *Tracker) parseState(out string, ss modulepkg.StateSource) *panelv1.AppStateUpdate {
|
||||
if ss.Parse == nil || ss.Parse.Kind != "regex" {
|
||||
return nil
|
||||
}
|
||||
re := t.stateRE[ss.Parse.Pattern]
|
||||
if re == nil {
|
||||
return nil
|
||||
}
|
||||
return parseRegexCompiled(t.cfg.InstanceID, out, re, ss.Parse)
|
||||
}
|
||||
|
||||
// parseStateOutput is the uncached variant (compiles the pattern per call).
|
||||
// Kept for tests and out-of-tracker callers; the tracker's poll loop uses
|
||||
// the precompiled parseState path.
|
||||
func parseStateOutput(instanceID, out string, ss modulepkg.StateSource) *panelv1.AppStateUpdate {
|
||||
if ss.Parse == nil {
|
||||
return nil
|
||||
}
|
||||
switch ss.Parse.Kind {
|
||||
case "regex":
|
||||
return parseRegex(instanceID, out, ss.Parse)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func parseRegex(instanceID, out string, p *modulepkg.ParseConfig) *panelv1.AppStateUpdate {
|
||||
re, err := regexp.Compile(p.Pattern)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return parseRegexCompiled(instanceID, out, re, p)
|
||||
}
|
||||
|
||||
func parseRegexCompiled(instanceID, out string, re *regexp.Regexp, p *modulepkg.ParseConfig) *panelv1.AppStateUpdate {
|
||||
m := re.FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
app := &panelv1.AppStateUpdate{
|
||||
InstanceId: instanceID,
|
||||
At: timestamppb.Now(),
|
||||
}
|
||||
for outName, groupName := range p.Fields {
|
||||
val := firstGroup(re, m, groupName)
|
||||
switch outName {
|
||||
case "players_online":
|
||||
if n, err := strconv.Atoi(val); err == nil {
|
||||
app.PlayersOnline = int32(n)
|
||||
}
|
||||
case "players_max":
|
||||
if n, err := strconv.Atoi(val); err == nil {
|
||||
app.PlayersMax = int32(n)
|
||||
}
|
||||
case "uptime_seconds":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
app.UptimeSeconds = n
|
||||
}
|
||||
default:
|
||||
if app.ModuleFields == nil {
|
||||
app.ModuleFields = map[string]string{}
|
||||
}
|
||||
app.ModuleFields[outName] = val
|
||||
}
|
||||
}
|
||||
return app
|
||||
}
|
||||
|
||||
func compileEvents(events map[string]modulepkg.Event) ([]compiledEvent, error) {
|
||||
out := make([]compiledEvent, 0, len(events))
|
||||
for name, ev := range events {
|
||||
re, err := regexp.Compile(ev.Pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("event %q: compile pattern: %w", name, err)
|
||||
}
|
||||
out = append(out, compiledEvent{
|
||||
name: name,
|
||||
re: re,
|
||||
kind: parseKind(ev.Kind),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseKind(s string) panelv1.PlayerEvent_Kind {
|
||||
switch s {
|
||||
case "join":
|
||||
return panelv1.PlayerEvent_KIND_JOIN
|
||||
case "leave":
|
||||
return panelv1.PlayerEvent_KIND_LEAVE
|
||||
case "chat":
|
||||
return panelv1.PlayerEvent_KIND_CHAT
|
||||
case "death":
|
||||
return panelv1.PlayerEvent_KIND_DEATH
|
||||
default:
|
||||
return panelv1.PlayerEvent_KIND_CUSTOM
|
||||
}
|
||||
}
|
||||
|
||||
// firstGroup returns the first non-empty named submatch from names. Missing
|
||||
// or empty groups are treated as absent so we try the next name.
|
||||
func firstGroup(re *regexp.Regexp, m []string, names ...string) string {
|
||||
for _, name := range names {
|
||||
i := re.SubexpIndex(name)
|
||||
if i >= 0 && i < len(m) && m[i] != "" {
|
||||
return m[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user