166 lines
7.6 KiB
Go
166 lines
7.6 KiB
Go
package dispatch
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"time"
|
|
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
|
|
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
|
)
|
|
|
|
// Auto-provision a freshly-generated clustered 7DTD world (zero-touch create),
|
|
// AND keep a visible "setting up" status until it is actually done so an operator
|
|
// who refreshes the dashboard sees it is not ready yet (don't restart it).
|
|
//
|
|
// A new cluster world bakes its decorations (decoration.7dt / multiblocks.7dt)
|
|
// with the engine's NATIVE block ids at generation. The entrypoint stamps the
|
|
// cluster CANONICAL .nim over the world's table every boot so shared inventory
|
|
// (.ttp) decodes uniformly — but native-baked decorations then resolve to a null
|
|
// Block under canonical -> DecoManager NRE -> the world loads but never reaches
|
|
// GameStartDone ("online but unjoinable"). The entrypoint fixes this by remapping
|
|
// the decorations native->canonical on the FIRST boot it sees them in the native
|
|
// band — which can only be a boot AFTER the gen boot (the running server writes
|
|
// decoration.7dt; the entrypoint can't see it until next start).
|
|
//
|
|
// This watcher (a) closes that 2-phase gap so create is zero-touch — it bounces
|
|
// the container ONCE so the entrypoint's remap runs — and (b) holds a persistent
|
|
// "Finishing cluster setup…" status detail (the dashboard renders it as a "setting
|
|
// up" pill instead of a green "running" one) until the world carries the
|
|
// .canon-provisioned marker, so a page refresh always shows it isn't done yet.
|
|
//
|
|
// Strict no-op for non-cluster servers (no /cluster bind -> the watcher exits) and
|
|
// already-canon worlds (deco in canon band -> "CANON" on the first probe -> the
|
|
// indicator clears immediately). Single-flight; the entrypoint is the backstop if
|
|
// the watcher ever misses (the world still provisions on its next restart).
|
|
// Mirrors the hang-guard single-flight container-restart pattern (hangguard.go).
|
|
|
|
const (
|
|
provisionPollInterval = 20 * time.Second
|
|
provisionPollTimeout = 15 * time.Minute
|
|
provisionExecTimeout = 20 * time.Second
|
|
provisionRestartGrace = 30 * time.Second
|
|
)
|
|
|
|
// provisionDetail is the persistent status detail shown while a clustered world is
|
|
// still aligning to the cluster canonical. The dashboard matches this text to paint
|
|
// a "setting up" pill, so a refresh shows the server isn't ready yet.
|
|
const provisionDetail = "Finishing cluster setup — aligning decorations to the cluster (it auto-restarts once); please wait, don't restart it."
|
|
|
|
// provisionProbeScript runs inside the container and classifies the active cluster
|
|
// world's provisioning state. Emits exactly one token on stdout:
|
|
//
|
|
// NOWORLD no recorded save dir (RWG world not yet picked in the Cluster tab)
|
|
// PROVISIONED .canon-provisioned marker present (already aligned)
|
|
// GENERATING no main.ttw yet (world still generating)
|
|
// CANON decorations already in the canon band (>=24000) -> nothing to do
|
|
// ATTEMPTED already bounced once and still not aligned -> stop (avoid a loop)
|
|
// NEEDS_ALIGN gen done; decorations native-banded OR not baked yet -> bounce once
|
|
//
|
|
// Gated on main.ttw (= gen finished) not on decoration.7dt existing, because some
|
|
// worlds only write decoration.7dt on the first save — the bounce's graceful save
|
|
// writes it (native), then the entrypoint remaps it on the align boot. First deco
|
|
// block-id = u16 @ offset 17 (hdr 5 + record id-offset 12), masked to 15 bits.
|
|
const provisionProbeScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
|
|
`[ -z "$SB" ] && { echo NOWORLD; exit 0; }; ` +
|
|
`[ -f "$SB/.canon-provisioned" ] && { echo PROVISIONED; exit 0; }; ` +
|
|
`[ -f "$SB/main.ttw" ] || { echo GENERATING; exit 0; }; ` +
|
|
`D="$SB/decoration.7dt"; ` +
|
|
`if [ -f "$D" ]; then raw="$(od -An -tu2 -j17 -N2 "$D" 2>/dev/null | tr -d ' ')"; id=$(( ${raw:-0} & 0x7FFF )); [ "$id" -ge 24000 ] && { echo CANON; exit 0; }; fi; ` +
|
|
`[ -f "$SB/.panel-provision-attempted" ] && { echo ATTEMPTED; exit 0; }; ` +
|
|
`echo NEEDS_ALIGN`
|
|
|
|
// provisionAttemptScript drops a durable one-bounce guard so a world that doesn't
|
|
// align after its single bounce is never bounced again (the entrypoint remains the
|
|
// backstop on ordinary restarts).
|
|
const provisionAttemptScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
|
|
`[ -n "$SB" ] && touch "$SB/.panel-provision-attempted" 2>/dev/null; true`
|
|
|
|
// autoProvisionClusterWorldOnStart starts (single-flight) the provision watcher for
|
|
// a just-started 7DTD instance. Returns immediately; the watch runs off a background
|
|
// goroutine bounded by provisionPollTimeout.
|
|
func (d *Dispatcher) autoProvisionClusterWorldOnStart(rec *instanceRecord) {
|
|
if rec.ModuleID != "7dtd" {
|
|
return
|
|
}
|
|
if !rec.provisionGuard.CompareAndSwap(false, true) {
|
|
return // a watcher is already running for this instance
|
|
}
|
|
go func() {
|
|
defer rec.provisionGuard.Store(false)
|
|
log := d.log.With("instance_id", rec.InstanceID, "auto_provision", true)
|
|
|
|
// Only clustered worlds provision (and show the indicator). Resolve the
|
|
// /cluster bind like clusterplayer.go; bail out for standalone servers.
|
|
mctx, mcancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
mounts, _ := d.runtime.ContainerMounts(mctx, "panel-"+rec.InstanceID)
|
|
mcancel()
|
|
clustered := false
|
|
for _, m := range mounts {
|
|
if m.Destination == "/cluster" && m.Type == "bind" && m.Source != "" {
|
|
clustered = true
|
|
break
|
|
}
|
|
}
|
|
if !clustered {
|
|
return
|
|
}
|
|
|
|
deadline := time.Now().Add(provisionPollTimeout)
|
|
delay := 5 * time.Second // first poll soon, to override a premature green "running"
|
|
var lastDetail string
|
|
bounced := false
|
|
emit := func(detail string) {
|
|
if detail == lastDetail {
|
|
return
|
|
}
|
|
lastDetail = detail
|
|
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING, 0, detail)
|
|
}
|
|
for {
|
|
time.Sleep(delay)
|
|
delay = provisionPollInterval
|
|
if time.Now().After(deadline) {
|
|
return // gave up; the entrypoint aligns on the next ordinary restart
|
|
}
|
|
ectx, cancel := context.WithTimeout(context.Background(), provisionExecTimeout)
|
|
stdout, _, code, err := d.runtime.ExecCapture(ectx, rec.ContainerID, []string{"sh", "-c", provisionProbeScript})
|
|
cancel()
|
|
if err != nil || code != 0 {
|
|
continue // mid-boot / exec hiccup — leave the agent's own status alone
|
|
}
|
|
switch strings.TrimSpace(string(stdout)) {
|
|
case "CANON", "PROVISIONED":
|
|
emit("running") // fully set up — clear the "setting up" indicator
|
|
return
|
|
case "NEEDS_ALIGN":
|
|
emit(provisionDetail)
|
|
if !bounced {
|
|
bounced = true
|
|
actx, acancel := context.WithTimeout(context.Background(), provisionExecTimeout)
|
|
_, _, _, _ = d.runtime.ExecCapture(actx, rec.ContainerID, []string{"sh", "-c", provisionAttemptScript})
|
|
acancel()
|
|
log.Info("auto-provision: aligning new cluster world to canonical (one-time bounce)")
|
|
d.sendEnv(&panelv1.AgentEnvelope{
|
|
SentAt: timestamppb.Now(),
|
|
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
|
InstanceId: rec.InstanceID, Stream: "stdout", At: timestamppb.Now(),
|
|
Line: "[panel] auto-provision: aligning the new world's decorations to the cluster (one-time restart)…",
|
|
}},
|
|
})
|
|
rctx, rcancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
if err := d.runtime.Restart(rctx, rec.ContainerID, provisionRestartGrace); err != nil {
|
|
log.Error("auto-provision restart failed (entrypoint aligns on the next ordinary restart)", "err", err)
|
|
}
|
|
rcancel()
|
|
}
|
|
continue // keep watching across the bounce until the align boot marks it provisioned
|
|
default: // GENERATING / NOWORLD / ATTEMPTED — still setting up
|
|
emit(provisionDetail)
|
|
continue
|
|
}
|
|
}
|
|
}()
|
|
}
|