panel v0.9.2 — open-source game server manager
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/dispatch"
|
||||
agentruntime "github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
"github.com/dbledeez/panel/pkg/version"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// loadClientTLS returns gRPC credentials configured for mTLS against
|
||||
// the controller's CA, or nil if cert files aren't present (caller then
|
||||
// decides whether to refuse or fall back to plaintext). The returned CN
|
||||
// is the agent identity the controller issued during pairing — callers
|
||||
// should use it as the agent_id so identity stays cryptographically bound
|
||||
// to the cert instead of drifting to whatever hostname the process sees.
|
||||
func loadClientTLS(certDir, controllerAddr string) (credentials.TransportCredentials, string, error) {
|
||||
caPath := filepath.Join(certDir, "ca.crt")
|
||||
certPath := filepath.Join(certDir, "agent.crt")
|
||||
keyPath := filepath.Join(certDir, "agent.key")
|
||||
|
||||
for _, p := range []string{caPath, certPath, keyPath} {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
return nil, "", fmt.Errorf("missing %s: %w", p, err)
|
||||
}
|
||||
}
|
||||
caPEM, err := os.ReadFile(caPath)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, "", fmt.Errorf("bad CA PEM")
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("load cert+key: %w", err)
|
||||
}
|
||||
var cn string
|
||||
if len(cert.Certificate) > 0 {
|
||||
if leaf, perr := x509.ParseCertificate(cert.Certificate[0]); perr == nil {
|
||||
cn = leaf.Subject.CommonName
|
||||
}
|
||||
}
|
||||
// ServerName defaults to the host part of controllerAddr. If it's
|
||||
// an IP we still set ServerName so Go's TLS stack uses SNI; the
|
||||
// controller cert includes 127.0.0.1 as an IP SAN so this works.
|
||||
serverName := controllerAddr
|
||||
if i := strings.LastIndex(serverName, ":"); i > 0 {
|
||||
serverName = serverName[:i]
|
||||
}
|
||||
tlsCfg := &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
RootCAs: pool,
|
||||
ServerName: serverName,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
return credentials.NewTLS(tlsCfg), cn, nil
|
||||
}
|
||||
|
||||
// agentVersion mirrors the shared build version (ldflags-injected via
|
||||
// github.com/dbledeez/panel/pkg/version.Version; "dev" in ad-hoc builds).
|
||||
var agentVersion = version.Version
|
||||
|
||||
func hostname() string {
|
||||
h, err := os.Hostname()
|
||||
if err != nil || h == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func main() {
|
||||
controllerAddr := flag.String("controller", "localhost:8443", "controller gRPC address host:port")
|
||||
agentID := flag.String("agent-id", "", "agent identifier (default: mTLS cert CN, or hostname when --insecure)")
|
||||
modulesDir := flag.String("modules-dir", "./modules", "path to modules directory")
|
||||
dataRoot := flag.String("data-root", "./data/instances", "root directory for instance data")
|
||||
metaDir := flag.String("meta-dir", "./data/instance-meta", "directory for per-instance sidecar metadata (survives agent restart)")
|
||||
backupDir := flag.String("backup-dir", "./data/backups", "directory where instance backup archives are written")
|
||||
certDir := flag.String("cert-dir", "./data/certs", "directory holding agent.crt + agent.key + ca.crt after pairing")
|
||||
heartbeatInterval := flag.Duration("heartbeat", 10*time.Second, "heartbeat interval")
|
||||
insecureDial := flag.Bool("insecure", false, "force plaintext (no TLS). Use --insecure=true only for dev/local setups.")
|
||||
pairToken := flag.String("pair-token", "", "if set, agent runs in pair mode: fetches CA + signed cert from --pair-url and exits")
|
||||
pairURL := flag.String("pair-url", "http://localhost:8080", "controller HTTP URL for pair/CA/issue endpoints (used with --pair-token)")
|
||||
logLevel := flag.String("log-level", "info", "log level: debug, info, warn, error")
|
||||
flag.Parse()
|
||||
|
||||
var lvl slog.Level
|
||||
if err := lvl.UnmarshalText([]byte(*logLevel)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "invalid log level %q: %v\n", *logLevel, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
// Pair mode: fetch CA + sign a client cert, then exit. Pair runs before
|
||||
// we touch the local cert so the agent-id has to come from either --agent-id
|
||||
// or hostname — there's no cert CN to read yet.
|
||||
if *pairToken != "" {
|
||||
if *agentID == "" {
|
||||
*agentID = hostname()
|
||||
}
|
||||
pairLogger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})).
|
||||
With("agent_id", *agentID)
|
||||
if err := runPair(pairConfig{
|
||||
AgentID: *agentID,
|
||||
PairURL: *pairURL,
|
||||
Token: *pairToken,
|
||||
CertDir: *certDir,
|
||||
Logger: pairLogger,
|
||||
}); err != nil {
|
||||
pairLogger.Error("pair failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// TLS creds: try loading cert files; fall back if --insecure.
|
||||
// The cert's CN is the canonical agent identity. If --agent-id wasn't
|
||||
// set explicitly, adopt the CN; if it was set to something different,
|
||||
// refuse — a mismatched identity just silently orphans instances in the
|
||||
// DB (the exact bug that motivated this change).
|
||||
var tlsCreds credentials.TransportCredentials
|
||||
var certCN string
|
||||
creds, cn, tlsErr := loadClientTLS(*certDir, *controllerAddr)
|
||||
switch {
|
||||
case tlsErr == nil:
|
||||
tlsCreds = creds
|
||||
certCN = cn
|
||||
case *insecureDial:
|
||||
// Plaintext path — cert CN unavailable, fall through to hostname.
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "no TLS certs found and --insecure not set. Run pairing first:\n"+
|
||||
" agent --pair-token <token> --pair-url http://controller:8080 --cert-dir %s\n"+
|
||||
"detail: %v\n", *certDir, tlsErr)
|
||||
os.Exit(2)
|
||||
}
|
||||
switch {
|
||||
case *agentID == "" && certCN != "":
|
||||
*agentID = certCN
|
||||
case *agentID == "":
|
||||
*agentID = hostname()
|
||||
case certCN != "" && *agentID != certCN:
|
||||
fmt.Fprintf(os.Stderr, "--agent-id=%q does not match mTLS cert CN=%q; refusing to start.\n"+
|
||||
"Re-pair the agent with the intended id or drop --agent-id to adopt the cert CN.\n",
|
||||
*agentID, certCN)
|
||||
os.Exit(2)
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})).
|
||||
With("agent_id", *agentID)
|
||||
if tlsCreds != nil {
|
||||
logger.Info("mTLS enabled", "cert_dir", *certDir, "cert_cn", certCN)
|
||||
} else {
|
||||
logger.Warn("running WITHOUT TLS (--insecure) — agent traffic is plaintext")
|
||||
}
|
||||
|
||||
absDataRoot, err := filepath.Abs(*dataRoot)
|
||||
if err != nil {
|
||||
logger.Error("resolve data-root", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.MkdirAll(absDataRoot, 0o755); err != nil {
|
||||
logger.Error("mkdir data-root", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
absMetaDir, err := filepath.Abs(*metaDir)
|
||||
if err != nil {
|
||||
logger.Error("resolve meta-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.MkdirAll(absMetaDir, 0o755); err != nil {
|
||||
logger.Error("mkdir meta-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
absBackupDir, err := filepath.Abs(*backupDir)
|
||||
if err != nil {
|
||||
logger.Error("resolve backup-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.MkdirAll(absBackupDir, 0o755); err != nil {
|
||||
logger.Error("mkdir backup-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
registry := modulepkg.NewRegistry()
|
||||
if err := registry.LoadDir(*modulesDir); err != nil {
|
||||
logger.Warn("module registry load failed", "dir", *modulesDir, "err", err)
|
||||
}
|
||||
logger.Info("modules loaded", "count", registry.Len(), "dir", *modulesDir)
|
||||
for _, m := range registry.List() {
|
||||
logger.Info("module registered", "id", m.ID, "name", m.Name, "version", m.Version)
|
||||
}
|
||||
|
||||
dockerRT, err := agentruntime.NewDocker()
|
||||
if err != nil {
|
||||
logger.Error("docker runtime init", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer dockerRT.Close()
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
if err := dockerRT.Ping(ctx); err != nil {
|
||||
logger.Warn("docker daemon not reachable; instance commands will fail", "err", err)
|
||||
} else {
|
||||
logger.Info("docker runtime ready")
|
||||
}
|
||||
|
||||
disp := dispatch.New(logger, registry, dockerRT, absDataRoot, absMetaDir, absBackupDir)
|
||||
|
||||
if err := disp.Rehydrate(ctx); err != nil {
|
||||
logger.Warn("rehydrate failed (agent will start fresh)", "err", err)
|
||||
}
|
||||
|
||||
// Reconnect loop: keep dialing the controller until ctx is cancelled.
|
||||
// A session ends either because ctx was cancelled (operator signal)
|
||||
// or because the stream died (controller blip, network, etc.).
|
||||
backoff := 2 * time.Second
|
||||
const maxBackoff = 30 * time.Second
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
err := runSession(ctx, logger, *controllerAddr, *agentID, *heartbeatInterval, disp, tlsCreds)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
logger.Warn("session ended, will reconnect", "err", err, "in", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(backoff + jitter(backoff)):
|
||||
}
|
||||
if backoff < maxBackoff {
|
||||
backoff *= 2
|
||||
if backoff > maxBackoff {
|
||||
backoff = maxBackoff
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("operator signal — gracefully stopping running instances")
|
||||
disp.Shutdown(context.Background())
|
||||
}
|
||||
|
||||
// runSession opens one gRPC connection, handshakes, (re)announces current
|
||||
// instance state, then runs the recv/send loops until the stream ends.
|
||||
// Returns nil if the operator signalled shutdown, or an error if the
|
||||
// connection broke.
|
||||
func runSession(ctx context.Context, logger *slog.Logger, addr, agentID string, hb time.Duration, disp *dispatch.Dispatcher, tlsCreds credentials.TransportCredentials) error {
|
||||
var dialOpt grpc.DialOption
|
||||
if tlsCreds != nil {
|
||||
dialOpt = grpc.WithTransportCredentials(tlsCreds)
|
||||
} else {
|
||||
dialOpt = grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
}
|
||||
// Lift the default 4 MiB per-message cap so big file uploads
|
||||
// (FsWrite carrying mod zips, scenario archives, save backups)
|
||||
// flow through end-to-end. Matches the controller-side cap.
|
||||
const grpcMaxMsgSize = 5 * 1024 * 1024 * 1024
|
||||
conn, err := grpc.NewClient(addr, dialOpt,
|
||||
grpc.WithDefaultCallOptions(
|
||||
grpc.MaxCallRecvMsgSize(grpcMaxMsgSize),
|
||||
grpc.MaxCallSendMsgSize(grpcMaxMsgSize),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("grpc client: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := panelv1.NewAgentClient(conn)
|
||||
|
||||
logger.Info("connecting to controller", "addr", addr)
|
||||
stream, err := client.Connect(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open stream: %w", err)
|
||||
}
|
||||
|
||||
hello := &panelv1.AgentEnvelope{
|
||||
CorrelationId: "hello",
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Hello{
|
||||
Hello: &panelv1.AgentHello{
|
||||
AgentId: agentID,
|
||||
AgentVersion: agentVersion,
|
||||
HostOs: runtime.GOOS,
|
||||
HostArch: runtime.GOARCH,
|
||||
Hostname: hostname(),
|
||||
SupportedRuntimes: []string{"docker"},
|
||||
Modules: summarizeModules(disp),
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := stream.Send(hello); err != nil {
|
||||
return fmt.Errorf("send hello: %w", err)
|
||||
}
|
||||
|
||||
resp, err := stream.Recv()
|
||||
if err != nil {
|
||||
return fmt.Errorf("recv hello: %w", err)
|
||||
}
|
||||
ch := resp.GetHello()
|
||||
if ch == nil {
|
||||
return fmt.Errorf("unexpected first message: %T", resp.Payload)
|
||||
}
|
||||
logger.Info("handshake complete", "controller_version", ch.ControllerVersion)
|
||||
|
||||
// Install the current stream as the upward sender and re-announce.
|
||||
disp.SetSender(stream)
|
||||
disp.Announce(ctx)
|
||||
|
||||
// Drop the sender before we return so no one sends into a dead stream.
|
||||
defer disp.SetSender(nil)
|
||||
|
||||
recvDone := make(chan struct{})
|
||||
recvErr := make(chan error, 1)
|
||||
go func() {
|
||||
defer close(recvDone)
|
||||
for {
|
||||
env, err := stream.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
logger.Info("controller closed stream")
|
||||
recvErr <- nil
|
||||
} else if ctx.Err() == nil {
|
||||
recvErr <- err
|
||||
} else {
|
||||
recvErr <- nil
|
||||
}
|
||||
return
|
||||
}
|
||||
// Dispatch each envelope in its OWN goroutine so a slow/blocked
|
||||
// handler can't freeze the recv loop (and thus the whole agent).
|
||||
// This was the Valheim+stdio hang: an RCON write to a wedged stdin
|
||||
// pipe blocked here forever, so the subsequent stop never got read.
|
||||
// Safe to parallelize: handlers guard shared state with d.mu, and the
|
||||
// controller serializes dependent ops (create→start) via its own
|
||||
// correlation-id awaits, so it never sends a dependent op before the
|
||||
// prior reply arrives.
|
||||
go disp.Handle(ctx, env)
|
||||
}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(hb)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = stream.CloseSend()
|
||||
<-recvDone
|
||||
return nil
|
||||
case <-recvDone:
|
||||
return <-recvErr
|
||||
case t := <-ticker.C:
|
||||
// Heartbeats go through the dispatcher's send queue (droppable)
|
||||
// so sendLoop stays the sole owner of stream.Send — sending here
|
||||
// directly raced the drainer goroutine on the same gRPC stream.
|
||||
// A dead stream is detected by the recv goroutine (stream.Recv
|
||||
// errors → recvDone), not by a failed heartbeat send.
|
||||
disp.SendHeartbeat(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// jitter returns up to ±25% of d as a random offset, so N agents don't
|
||||
// synchronize their reconnects against a restarting controller.
|
||||
func jitter(d time.Duration) time.Duration {
|
||||
if d <= 0 {
|
||||
return 0
|
||||
}
|
||||
half := int64(d / 4)
|
||||
return time.Duration(rand.Int64N(2*half+1) - half)
|
||||
}
|
||||
|
||||
// summarizeModules reduces the agent's loaded manifests to the small
|
||||
// ModuleSummary record the controller caches for UI picking.
|
||||
func summarizeModules(disp *dispatch.Dispatcher) []*panelv1.ModuleSummary {
|
||||
mods := disp.Modules().List()
|
||||
out := make([]*panelv1.ModuleSummary, 0, len(mods))
|
||||
for _, m := range mods {
|
||||
providers := make([]*panelv1.ModuleUpdateProviderSummary, 0, len(m.UpdateProviders))
|
||||
for _, p := range m.UpdateProviders {
|
||||
providers = append(providers, &panelv1.ModuleUpdateProviderSummary{
|
||||
Id: p.ID,
|
||||
Kind: p.Kind,
|
||||
RequiresSteamLogin: p.RequiresSteamLogin,
|
||||
})
|
||||
}
|
||||
out = append(out, &panelv1.ModuleSummary{
|
||||
Id: m.ID,
|
||||
Name: m.Name,
|
||||
Version: m.Version,
|
||||
SupportedModes: m.SupportedModes,
|
||||
UpdateProviders: providers,
|
||||
HasRcon: m.RCON != nil,
|
||||
Authors: m.Authors,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pairing flow (agent side):
|
||||
//
|
||||
// 1. Fetch the controller's CA cert over HTTPS (with InsecureSkipVerify —
|
||||
// this is the only moment we're exposed; the one-time token gates
|
||||
// the actual cert issuance below).
|
||||
// 2. Generate an RSA-2048 keypair locally. The private key never
|
||||
// leaves this host.
|
||||
// 3. Build a CSR with CN = agent-id.
|
||||
// 4. POST {token, agent_id, csr_pem} to /api/pair/issue, now verifying
|
||||
// the server cert against the CA we just fetched.
|
||||
// 5. Persist ca.crt + agent.crt + agent.key to --cert-dir, mode 0600 on the key.
|
||||
//
|
||||
// After pairing, run the agent normally; it auto-detects the cert files
|
||||
// and connects to the controller with mTLS.
|
||||
|
||||
type pairConfig struct {
|
||||
AgentID string
|
||||
PairURL string // e.g. https://controller:8080 — the HTTP dashboard port
|
||||
Token string
|
||||
CertDir string
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
func runPair(cfg pairConfig) error {
|
||||
cfg.Logger.Info("pair: starting", "pair_url", cfg.PairURL, "agent_id", cfg.AgentID, "cert_dir", cfg.CertDir)
|
||||
|
||||
if err := os.MkdirAll(cfg.CertDir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir cert-dir: %w", err)
|
||||
}
|
||||
|
||||
// Step 1 — fetch CA with InsecureSkipVerify, then verify with it.
|
||||
caPEM, err := fetchCA(cfg.PairURL, cfg.Logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch CA: %w", err)
|
||||
}
|
||||
caPath := filepath.Join(cfg.CertDir, "ca.crt")
|
||||
if err := os.WriteFile(caPath, caPEM, 0o644); err != nil {
|
||||
return fmt.Errorf("write ca.crt: %w", err)
|
||||
}
|
||||
cfg.Logger.Info("pair: CA saved", "path", caPath, "bytes", len(caPEM))
|
||||
|
||||
// Step 2 — keypair + CSR
|
||||
priv, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gen key: %w", err)
|
||||
}
|
||||
tpl := &x509.CertificateRequest{
|
||||
Subject: pkix.Name{CommonName: cfg.AgentID, Organization: []string{"panel-agent"}},
|
||||
DNSNames: []string{cfg.AgentID},
|
||||
}
|
||||
csrDER, err := x509.CreateCertificateRequest(rand.Reader, tpl, priv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create CSR: %w", err)
|
||||
}
|
||||
csrPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrDER})
|
||||
|
||||
// Step 3 — POST to /api/pair/issue, verifying CA.
|
||||
certPEM, err := issueCert(cfg.PairURL, caPEM, cfg.Token, cfg.AgentID, csrPEM, cfg.Logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("issue cert: %w", err)
|
||||
}
|
||||
|
||||
// Step 4 — persist.
|
||||
if err := os.WriteFile(filepath.Join(cfg.CertDir, "agent.crt"), certPEM, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
|
||||
if err := os.WriteFile(filepath.Join(cfg.CertDir, "agent.key"), keyPEM, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Logger.Info("pair: cert issued", "cert_dir", cfg.CertDir, "agent_id", cfg.AgentID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchCA(baseURL string, log *slog.Logger) ([]byte, error) {
|
||||
trimmed := strings.TrimSuffix(baseURL, "/")
|
||||
url := trimmed + "/api/pair/ca"
|
||||
log.Info("pair: fetching CA", "url", url)
|
||||
|
||||
// Until we have the CA we can't verify the server. InsecureSkipVerify
|
||||
// is narrowly scoped to this one request — the cert-issuance POST
|
||||
// below verifies against the CA we just got.
|
||||
cli := &http.Client{
|
||||
Timeout: 15 * time.Second,
|
||||
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
|
||||
}
|
||||
resp, err := cli.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return nil, fmt.Errorf("http %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
block, _ := pem.Decode(data)
|
||||
if block == nil || block.Type != "CERTIFICATE" {
|
||||
return nil, fmt.Errorf("response isn't a CERTIFICATE PEM")
|
||||
}
|
||||
if _, err := x509.ParseCertificate(block.Bytes); err != nil {
|
||||
return nil, fmt.Errorf("parse CA cert: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func issueCert(baseURL string, caPEM []byte, token, agentID string, csrPEM []byte, log *slog.Logger) ([]byte, error) {
|
||||
trimmed := strings.TrimSuffix(baseURL, "/")
|
||||
url := trimmed + "/api/pair/issue"
|
||||
log.Info("pair: requesting cert", "url", url, "agent_id", agentID)
|
||||
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("append CA to pool failed")
|
||||
}
|
||||
cli := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}},
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"token": token,
|
||||
"agent_id": agentID,
|
||||
"csr_pem": string(csrPEM),
|
||||
})
|
||||
req, _ := http.NewRequest("POST", url, bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
return nil, fmt.Errorf("http %d: %s", resp.StatusCode, string(errBody))
|
||||
}
|
||||
var out struct {
|
||||
CertPEM string `json:"cert_pem"`
|
||||
CAPEM string `json:"ca_pem"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out.CertPEM == "" {
|
||||
return nil, fmt.Errorf("response missing cert_pem")
|
||||
}
|
||||
return []byte(out.CertPEM), nil
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
)
|
||||
|
||||
// region-medic autoheal — the auto-on-reboot flow, designed to run (in a sidecar)
|
||||
// while the 7DTD server is STOPPED, just before the agent starts it:
|
||||
//
|
||||
// 1. Validate EVERY .7rg in the active world's Region dir (catches whole-file
|
||||
// corruption like a bad magic header, which has no error_backups and so is
|
||||
// invisible to the error_backup-cluster scan).
|
||||
// 2. Heal each corrupt region from the newest source that has a validated-clean
|
||||
// copy: the rolling folder snapshots first (fast, local), then the panel's
|
||||
// tar backups as a fallback. Corrupt originals are quarantined first.
|
||||
// 3. Snapshot the now-clean Region dir to a fresh rolling slot (real folder
|
||||
// copy — instant restore, no unzip) and rotate to keep the newest -keep.
|
||||
//
|
||||
// Emits a JSON report the agent relays to the controller for the Discord ping.
|
||||
//
|
||||
// region-medic autoheal -region-dir D -rolling-dir R [-backups-dir B] [-keep 2] [-apply]
|
||||
//
|
||||
// Without -apply it only validates + reports (dry run: no heal, no snapshot).
|
||||
|
||||
type ahHeal struct {
|
||||
Region string `json:"region"` // r.X.Z
|
||||
Source string `json:"source"` // "rolling:<slot>" | "backup:<id>"
|
||||
SourceTime string `json:"source_time"` // RFC3339 of the snapshot/backup
|
||||
}
|
||||
|
||||
type ahReport struct {
|
||||
RegionDir string `json:"region_dir"`
|
||||
TotalRegions int `json:"total_regions"`
|
||||
CorruptFound int `json:"corrupt_found"`
|
||||
Healed []ahHeal `json:"healed"`
|
||||
Unhealable []string `json:"unhealable"`
|
||||
SnapshotSlot string `json:"snapshot_slot"`
|
||||
RollingKept int `json:"rolling_kept"`
|
||||
Applied bool `json:"applied"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
}
|
||||
|
||||
func cmdAutoheal(args []string) error {
|
||||
var regionDir, rollingDir, backupsDir, savesRoot, discordChannel, discordTokenFile, label string
|
||||
keep := 2
|
||||
apply := false
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "-region-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
regionDir = args[i]
|
||||
}
|
||||
case "-saves-root":
|
||||
i++
|
||||
if i < len(args) {
|
||||
savesRoot = args[i]
|
||||
}
|
||||
case "-rolling-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
rollingDir = args[i]
|
||||
}
|
||||
case "-backups-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
backupsDir = args[i]
|
||||
}
|
||||
case "-keep":
|
||||
i++
|
||||
if i < len(args) {
|
||||
keep, _ = strconv.Atoi(args[i])
|
||||
}
|
||||
case "-discord-channel":
|
||||
i++
|
||||
if i < len(args) {
|
||||
discordChannel = args[i]
|
||||
}
|
||||
case "-discord-token-file":
|
||||
i++
|
||||
if i < len(args) {
|
||||
discordTokenFile = args[i]
|
||||
}
|
||||
case "-label":
|
||||
i++
|
||||
if i < len(args) {
|
||||
label = args[i]
|
||||
}
|
||||
case "-apply", "--apply":
|
||||
apply = true
|
||||
default:
|
||||
return fmt.Errorf("unknown flag %q", args[i])
|
||||
}
|
||||
}
|
||||
// -saves-root auto-resolves the active world's Region dir (so the agent
|
||||
// doesn't have to know GameWorld/GameName, and RWG seed-named worlds work).
|
||||
// Per-server config lives in the saves volume (panel writes it via FsWrite,
|
||||
// no recreate): world pin + keep + Discord channel; enabled:false skips.
|
||||
var cfgWorld string
|
||||
if savesRoot != "" {
|
||||
if c, ok := readSavesMedicConfig(savesRoot); ok {
|
||||
if c.Enabled != nil && !*c.Enabled {
|
||||
fmt.Fprintln(os.Stderr, "autoheal: disabled via region-medic.json, skipping")
|
||||
return nil
|
||||
}
|
||||
if c.Keep > 0 {
|
||||
keep = c.Keep
|
||||
}
|
||||
if c.DiscordChannel != "" {
|
||||
discordChannel = c.DiscordChannel
|
||||
}
|
||||
cfgWorld = strings.TrimSpace(c.World)
|
||||
}
|
||||
}
|
||||
// Resolve the Region dir. An explicit world pin wins over auto-detect — some
|
||||
// servers carry multiple saves, so the operator picks which one to heal. If
|
||||
// the pinned world has no Region dir we SKIP rather than silently auto-pick a
|
||||
// different world the operator didn't choose.
|
||||
if regionDir == "" && savesRoot != "" {
|
||||
if cfgWorld != "" {
|
||||
wd := filepath.Join(savesRoot, ".local/share/7DaysToDie/Saves", cfgWorld, "Region")
|
||||
if st, err := os.Stat(wd); err != nil || !st.IsDir() {
|
||||
fmt.Fprintf(os.Stderr, "autoheal: pinned world %q has no Region dir at %s — skipping\n", cfgWorld, wd)
|
||||
return nil
|
||||
}
|
||||
regionDir = wd
|
||||
} else {
|
||||
rd, rerr := resolveRegionDir(savesRoot)
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
regionDir = rd
|
||||
}
|
||||
}
|
||||
if regionDir == "" || rollingDir == "" {
|
||||
return fmt.Errorf("required: -region-dir (or -saves-root) and -rolling-dir (and -backups-dir for tar fallback)")
|
||||
}
|
||||
if keep < 1 {
|
||||
keep = 1
|
||||
}
|
||||
start := time.Now()
|
||||
rep := ahReport{RegionDir: regionDir, RollingKept: keep, Applied: apply, Healed: []ahHeal{}, Unhealable: []string{}}
|
||||
|
||||
// 1. Validate every .7rg.
|
||||
ents, err := os.ReadDir(regionDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read region dir: %w", err)
|
||||
}
|
||||
var corrupt []regionmedic.RegionCoord
|
||||
for _, e := range ents {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
rc, ok := regionmedic.ParseRegionFileName(e.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rep.TotalRegions++
|
||||
if !regionmedic.ValidateRegionFile(filepath.Join(regionDir, e.Name()), false).Healthy() {
|
||||
corrupt = append(corrupt, rc)
|
||||
}
|
||||
}
|
||||
rep.CorruptFound = len(corrupt)
|
||||
|
||||
// 2. Heal (only with -apply).
|
||||
quarantineDir := filepath.Join(rollingDir, "_quarantine")
|
||||
if apply && len(corrupt) > 0 {
|
||||
slots := listRollingSlots(rollingDir) // newest first
|
||||
var panelBackups []regionmedic.BackupRef
|
||||
if backupsDir != "" {
|
||||
panelBackups, _ = regionmedic.ListPanelBackups(backupsDir)
|
||||
}
|
||||
stamp := start.UTC().Format("20060102-150405")
|
||||
for _, rc := range corrupt {
|
||||
fn := rc.FileName()
|
||||
done := false
|
||||
// 2a. newest rolling snapshot with a clean copy.
|
||||
for _, s := range slots {
|
||||
cand := filepath.Join(s.path, fn)
|
||||
if fi, e := os.Stat(cand); e == nil && !fi.IsDir() &&
|
||||
regionmedic.ValidateRegionFile(cand, false).Healthy() {
|
||||
if e := healFromFile(regionDir, fn, cand, quarantineDir, stamp); e == nil {
|
||||
rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "rolling:" + filepath.Base(s.path), SourceTime: s.t.UTC().Format(time.RFC3339)})
|
||||
done = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2b. panel tar backups fallback.
|
||||
if !done && len(panelBackups) > 0 {
|
||||
entry := regionmedic.SaveRelEntry(regionDir, fn)
|
||||
best, _, _ := regionmedic.FindLastGoodRegion(panelBackups, entry, false, time.Time{})
|
||||
if best.Found && best.Clean() {
|
||||
if e := healFromBytes(regionDir, fn, best.Bytes, quarantineDir, stamp); e == nil {
|
||||
rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "backup:" + best.Backup.ID, SourceTime: best.Backup.CreatedAt.UTC().Format(time.RFC3339)})
|
||||
done = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !done {
|
||||
rep.Unhealable = append(rep.Unhealable, rc.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Snapshot the now-clean Region dir to a fresh rolling slot + rotate.
|
||||
if apply {
|
||||
slotName := fmt.Sprintf("slot-%d", start.Unix())
|
||||
if err := snapshotRegions(regionDir, filepath.Join(rollingDir, slotName)); err != nil {
|
||||
rep.SnapshotSlot = "ERROR: " + err.Error()
|
||||
} else {
|
||||
rep.SnapshotSlot = slotName
|
||||
rotateRollingSlots(rollingDir, keep)
|
||||
}
|
||||
}
|
||||
|
||||
rep.DurationMs = time.Since(start).Milliseconds()
|
||||
b, _ := json.MarshalIndent(rep, "", " ")
|
||||
fmt.Println(string(b))
|
||||
fmt.Fprintf(os.Stderr, "autoheal: %d regions, %d corrupt, %d healed, %d unhealable, snapshot=%s (%dms)\n",
|
||||
rep.TotalRegions, rep.CorruptFound, len(rep.Healed), len(rep.Unhealable), rep.SnapshotSlot, rep.DurationMs)
|
||||
// Discord alert — only when we actually healed something on this reboot.
|
||||
if apply {
|
||||
if derr := postDiscordHealReport(discordTokenFile, discordChannel, label, rep); derr != nil {
|
||||
fmt.Fprintf(os.Stderr, "discord post failed: %v\n", derr)
|
||||
} else if discordChannel != "" && len(rep.Healed) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "discord: posted heal embed to channel %s\n", discordChannel)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type rollingSlot struct {
|
||||
path string
|
||||
t time.Time
|
||||
}
|
||||
|
||||
// listRollingSlots returns slot dirs named "slot-<unix>" newest first.
|
||||
func listRollingSlots(dir string) []rollingSlot {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []rollingSlot
|
||||
for _, e := range ents {
|
||||
if !e.IsDir() || !strings.HasPrefix(e.Name(), "slot-") {
|
||||
continue
|
||||
}
|
||||
secs, err := strconv.ParseInt(strings.TrimPrefix(e.Name(), "slot-"), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, rollingSlot{path: filepath.Join(dir, e.Name()), t: time.Unix(secs, 0)})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].t.After(out[j].t) })
|
||||
return out
|
||||
}
|
||||
|
||||
// rotateRollingSlots keeps the newest `keep` slot dirs, removing older ones.
|
||||
func rotateRollingSlots(dir string, keep int) {
|
||||
for i, s := range listRollingSlots(dir) {
|
||||
if i >= keep {
|
||||
_ = os.RemoveAll(s.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// snapshotRegions copies every *.7rg from regionDir into slotPath.
|
||||
func snapshotRegions(regionDir, slotPath string) error {
|
||||
if err := os.MkdirAll(slotPath, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
ents, err := os.ReadDir(regionDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range ents {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".7rg") {
|
||||
continue
|
||||
}
|
||||
if err := copyFile(filepath.Join(regionDir, e.Name()), filepath.Join(slotPath, e.Name())); err != nil {
|
||||
return fmt.Errorf("copy %s: %w", e.Name(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// healFromFile quarantines the corrupt original then overwrites the live region
|
||||
// file in place with a validated-clean source file.
|
||||
func healFromFile(regionDir, fn, srcPath, quarantineDir, stamp string) error {
|
||||
_ = os.MkdirAll(quarantineDir, 0o755)
|
||||
live := filepath.Join(regionDir, fn)
|
||||
_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
|
||||
return copyFileInPlace(srcPath, live)
|
||||
}
|
||||
|
||||
// healFromBytes is the same but the clean source is in-memory (a tar backup).
|
||||
func healFromBytes(regionDir, fn string, data []byte, quarantineDir, stamp string) error {
|
||||
_ = os.MkdirAll(quarantineDir, 0o755)
|
||||
live := filepath.Join(regionDir, fn)
|
||||
_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
|
||||
tmp := live + ".medic-new"
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, live)
|
||||
}
|
||||
|
||||
// copyFile copies src→dst atomically (temp+rename), same filesystem assumed for
|
||||
// the rename target dir. Used for snapshots + quarantine (rolling dir).
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
tmp := dst + ".tmp-copy"
|
||||
out, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
// copyFileInPlace overwrites dst with src via a temp file in dst's own dir, so
|
||||
// the rename is same-filesystem even when src lives elsewhere (a rolling slot).
|
||||
func copyFileInPlace(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
tmp := dst + ".medic-new"
|
||||
out, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
_ "time/tzdata" // embed the IANA tz database so America/Los_Angeles resolves
|
||||
// even in debian:12-slim, which ships no /usr/share/zoneinfo.
|
||||
)
|
||||
|
||||
// resolveRegionDir finds the active world's Region dir under a 7DTD saves root
|
||||
// (the dir mounted as the saves volume — holds serverconfig.xml + .local/...).
|
||||
// Prefers serverconfig GameWorld/GameName when that world has real data
|
||||
// (main.ttw); else falls back to the most-recently-written Saves/*/<GameName>
|
||||
// with data — handles RWG seed-named worlds.
|
||||
func resolveRegionDir(savesRoot string) (string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(savesRoot, "serverconfig.xml"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read serverconfig.xml: %w", err)
|
||||
}
|
||||
gw := xmlProp(string(raw), "GameWorld")
|
||||
gn := xmlProp(string(raw), "GameName")
|
||||
base := filepath.Join(savesRoot, ".local", "share", "7DaysToDie", "Saves")
|
||||
hasWorld := func(d string) bool {
|
||||
if _, e := os.Stat(filepath.Join(d, "main.ttw")); e == nil {
|
||||
return true
|
||||
}
|
||||
fi, e := os.Stat(filepath.Join(d, "Region"))
|
||||
return e == nil && fi.IsDir()
|
||||
}
|
||||
if gw != "" && gn != "" {
|
||||
lit := filepath.Join(base, gw, gn)
|
||||
if hasWorld(lit) {
|
||||
return filepath.Join(lit, "Region"), nil
|
||||
}
|
||||
}
|
||||
if gn != "" { // RWG fallback: newest world dir with this GameName.
|
||||
worlds, _ := os.ReadDir(base)
|
||||
var newest string
|
||||
var newestT time.Time
|
||||
for _, w := range worlds {
|
||||
if !w.IsDir() {
|
||||
continue
|
||||
}
|
||||
cand := filepath.Join(base, w.Name(), gn)
|
||||
if !hasWorld(cand) {
|
||||
continue
|
||||
}
|
||||
if fi, e := os.Stat(cand); e == nil && fi.ModTime().After(newestT) {
|
||||
newestT, newest = fi.ModTime(), cand
|
||||
}
|
||||
}
|
||||
if newest != "" {
|
||||
return filepath.Join(newest, "Region"), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("could not resolve active Region dir (GameWorld=%q GameName=%q)", gw, gn)
|
||||
}
|
||||
|
||||
// xmlProp pulls the value from <property name="X" value="Y" /> (tolerant of
|
||||
// attribute order + quote style; skips comments).
|
||||
func xmlProp(content, name string) string {
|
||||
for _, raw := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "<!--") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(line, `name="`+name+`"`) {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(line, "value=")
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
rest := line[i+len("value="):]
|
||||
if len(rest) < 1 {
|
||||
continue
|
||||
}
|
||||
q := rest[0]
|
||||
rest = rest[1:]
|
||||
if end := strings.IndexByte(rest, q); end >= 0 {
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// postDiscordHealReport posts a rich embed heal summary to a Discord channel via
|
||||
// a bot token read from tokenFile. No-op when token/channel missing or nothing
|
||||
// healed. All times are rendered in Pacific (America/Los_Angeles → PST/PDT).
|
||||
func postDiscordHealReport(tokenFile, channel, label string, rep ahReport) error {
|
||||
if tokenFile == "" || channel == "" || len(rep.Healed) == 0 {
|
||||
return nil
|
||||
}
|
||||
tb, err := os.ReadFile(tokenFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read discord token: %w", err)
|
||||
}
|
||||
token := strings.TrimSpace(string(tb))
|
||||
if token == "" {
|
||||
return nil
|
||||
}
|
||||
if label == "" {
|
||||
label = "7DTD server"
|
||||
}
|
||||
loc, lerr := time.LoadLocation("America/Los_Angeles")
|
||||
if lerr != nil || loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
pst := func(rfc string) string {
|
||||
if t, e := time.Parse(time.RFC3339, rfc); e == nil {
|
||||
return t.In(loc).Format("Jan 2, 3:04 PM MST")
|
||||
}
|
||||
return rfc
|
||||
}
|
||||
srcLabel := func(s string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(s, "rolling:"):
|
||||
return "rolling snapshot"
|
||||
case strings.HasPrefix(s, "backup:"):
|
||||
return "panel backup"
|
||||
default:
|
||||
return s
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
for _, h := range rep.Healed {
|
||||
lines = append(lines, fmt.Sprintf("• `%s` ← %s _(saved %s)_", h.Region, srcLabel(h.Source), pst(h.SourceTime)))
|
||||
}
|
||||
color := 0x57F287 // green — everything healed
|
||||
fields := []map[string]any{{
|
||||
"name": fmt.Sprintf("✅ Healed %d region(s)", len(rep.Healed)),
|
||||
"value": strings.Join(lines, "\n"),
|
||||
"inline": false,
|
||||
}}
|
||||
if len(rep.Unhealable) > 0 {
|
||||
color = 0xED4245 // red — some couldn't be healed
|
||||
fields = append(fields, map[string]any{
|
||||
"name": fmt.Sprintf("⚠️ Could NOT heal %d (no clean copy anywhere)", len(rep.Unhealable)),
|
||||
"value": "`" + strings.Join(rep.Unhealable, "`, `") + "`",
|
||||
"inline": false,
|
||||
})
|
||||
}
|
||||
nowPST := time.Now().In(loc).Format("Mon Jan 2, 3:04 PM MST")
|
||||
embed := map[string]any{
|
||||
"title": "🩺 Region Medic — " + label,
|
||||
"description": fmt.Sprintf("Detected and repaired **%d** corrupt region file(s) on reboot, before the world loaded.", rep.CorruptFound),
|
||||
"color": color,
|
||||
"fields": fields,
|
||||
"footer": map[string]any{"text": "Region Medic · " + nowPST},
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"embeds": []any{embed}})
|
||||
req, err := http.NewRequest(http.MethodPost, "https://discord.com/api/v10/channels/"+channel+"/messages", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bot "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "RefugeBot-RegionMedic (https://refugegaming.org, 1.0)")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("discord HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// savesMedicConfig is the per-server medic config the panel writes (via FsWrite,
|
||||
// no container recreate) into the saves volume at region-medic.json. Enabled is a
|
||||
// pointer so an absent field defaults to ON.
|
||||
type savesMedicConfig struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
Keep int `json:"keep"`
|
||||
World string `json:"world"` // "<World>/<GameName>"; empty = auto-detect active world
|
||||
DiscordChannel string `json:"discord_channel"`
|
||||
}
|
||||
|
||||
// readSavesMedicConfig reads <savesRoot>/region-medic.json; ok=false if absent or
|
||||
// unparseable (the caller then keeps its flag defaults).
|
||||
func readSavesMedicConfig(savesRoot string) (savesMedicConfig, bool) {
|
||||
b, err := os.ReadFile(filepath.Join(savesRoot, "region-medic.json"))
|
||||
if err != nil {
|
||||
return savesMedicConfig{}, false
|
||||
}
|
||||
var c savesMedicConfig
|
||||
if json.Unmarshal(b, &c) != nil {
|
||||
return savesMedicConfig{}, false
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
// Command region-medic is a standalone operator tool that detects and heals
|
||||
// corrupted 7DTD ".7rg" region files from the panel's own backups, using the
|
||||
// shared github.com/dbledeez/panel/pkg/regionmedic engine.
|
||||
//
|
||||
// It is the same logic that ships inside the agent's RegionScan/RegionHeal RPCs,
|
||||
// exposed as a CLI so it can be run by hand against a stopped instance without
|
||||
// redeploying the controller. Subcommands:
|
||||
//
|
||||
// region-medic validate <file.7rg> [-deep]
|
||||
// region-medic scan <region-dir>
|
||||
// region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
|
||||
// region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply
|
||||
// [-deep] [-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
|
||||
// [-controller URL -instance ID] # stop before / start after (panel-safe)
|
||||
// [-container NAME] # post-start: watch for ready + new corruption
|
||||
// region-medic verify <region-dir> # count error_backups now
|
||||
//
|
||||
// stop/start go through the controller HTTP API (never raw docker), so panel
|
||||
// state stays correct; readiness/state polling reads docker locally.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
var err error
|
||||
switch os.Args[1] {
|
||||
case "validate":
|
||||
err = cmdValidate(os.Args[2:])
|
||||
case "scan":
|
||||
err = cmdScan(os.Args[2:])
|
||||
case "plan":
|
||||
err = cmdHeal(os.Args[2:], false)
|
||||
case "heal":
|
||||
err = cmdHeal(os.Args[2:], true)
|
||||
case "verify":
|
||||
err = cmdVerify(os.Args[2:])
|
||||
case "autoheal":
|
||||
err = cmdAutoheal(os.Args[2:])
|
||||
case "-h", "--help", "help":
|
||||
usage()
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1])
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `region-medic — detect & heal corrupt 7DTD .7rg regions from panel backups
|
||||
|
||||
region-medic validate <file.7rg> [-deep]
|
||||
region-medic scan <region-dir>
|
||||
region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
|
||||
region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply [-deep]
|
||||
[-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
|
||||
[-controller URL -instance ID] [-container NAME]
|
||||
region-medic verify <region-dir>
|
||||
`)
|
||||
}
|
||||
|
||||
func printJSON(v any) {
|
||||
b, _ := json.MarshalIndent(v, "", " ")
|
||||
fmt.Println(string(b))
|
||||
}
|
||||
|
||||
// --- validate ---------------------------------------------------------------
|
||||
|
||||
func cmdValidate(args []string) error {
|
||||
deep := false
|
||||
var file string
|
||||
for _, a := range args {
|
||||
if a == "-deep" || a == "--deep" {
|
||||
deep = true
|
||||
} else if !strings.HasPrefix(a, "-") {
|
||||
file = a
|
||||
}
|
||||
}
|
||||
if file == "" {
|
||||
return fmt.Errorf("usage: region-medic validate <file.7rg> [-deep]")
|
||||
}
|
||||
rep := regionmedic.ValidateRegionFile(file, deep)
|
||||
printJSON(rep)
|
||||
if rep.Healthy() {
|
||||
fmt.Fprintf(os.Stderr, "HEALTHY: %d present chunks, %d sectors\n", rep.PresentChunks, rep.TotalSectors)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "UNHEALTHY: err=%q badChunks=%d\n", rep.Err, len(rep.BadChunks))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- scan -------------------------------------------------------------------
|
||||
|
||||
func cmdScan(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("usage: region-medic scan <region-dir>")
|
||||
}
|
||||
scan, err := regionmedic.ScanRegionDir(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printJSON(scan)
|
||||
fmt.Fprintf(os.Stderr, "%d error_backups across %d affected region(s)\n", scan.ErrorBackups, len(scan.Affected))
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- verify -----------------------------------------------------------------
|
||||
|
||||
func cmdVerify(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("usage: region-medic verify <region-dir>")
|
||||
}
|
||||
scan, err := regionmedic.ScanRegionDir(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "error_backups now: %d (affected regions: %d)\n", scan.ErrorBackups, len(scan.Affected))
|
||||
printJSON(scan.Affected)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- plan / heal ------------------------------------------------------------
|
||||
|
||||
type healFlags struct {
|
||||
regionDir, region, backupsDir string
|
||||
snapshotDir, quarantineDir string
|
||||
deep, apply bool
|
||||
notAfter string
|
||||
controller, instance, container string
|
||||
}
|
||||
|
||||
func parseHealFlags(args []string) (healFlags, error) {
|
||||
f := healFlags{}
|
||||
for i := 0; i < len(args); i++ {
|
||||
a := args[i]
|
||||
next := func() string {
|
||||
if i+1 < len(args) {
|
||||
i++
|
||||
return args[i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
switch a {
|
||||
case "-region-dir":
|
||||
f.regionDir = next()
|
||||
case "-region":
|
||||
f.region = next()
|
||||
case "-backups-dir":
|
||||
f.backupsDir = next()
|
||||
case "-snapshot-dir":
|
||||
f.snapshotDir = next()
|
||||
case "-quarantine-dir":
|
||||
f.quarantineDir = next()
|
||||
case "-not-after":
|
||||
f.notAfter = next()
|
||||
case "-controller":
|
||||
f.controller = next()
|
||||
case "-instance":
|
||||
f.instance = next()
|
||||
case "-container":
|
||||
f.container = next()
|
||||
case "-deep", "--deep":
|
||||
f.deep = true
|
||||
case "-apply", "--apply":
|
||||
f.apply = true
|
||||
default:
|
||||
return f, fmt.Errorf("unknown flag %q", a)
|
||||
}
|
||||
}
|
||||
if f.regionDir == "" || f.region == "" || f.backupsDir == "" {
|
||||
return f, fmt.Errorf("required: -region-dir, -region, -backups-dir")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func cmdHeal(args []string, allowApply bool) error {
|
||||
f, err := parseHealFlags(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
region, ok := regionmedic.ParseRegionFileName(regionArg(f.region))
|
||||
if !ok {
|
||||
return fmt.Errorf("bad -region %q (want r.X.Z or r.X.Z.7rg)", f.region)
|
||||
}
|
||||
var notAfter time.Time
|
||||
if f.notAfter != "" {
|
||||
notAfter, err = time.Parse(time.RFC3339, f.notAfter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bad -not-after: %w", err)
|
||||
}
|
||||
}
|
||||
backups, err := regionmedic.ListPanelBackups(f.backupsDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list backups: %w", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "found %d backups in %s\n", len(backups), f.backupsDir)
|
||||
|
||||
plan, err := regionmedic.PlanHeal(f.regionDir, region, backups, f.deep, notAfter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summarizePlan(plan)
|
||||
|
||||
if !allowApply || !f.apply {
|
||||
fmt.Fprintln(os.Stderr, "(dry run — re-run `heal ... -apply` to execute)")
|
||||
return nil
|
||||
}
|
||||
if !plan.Ready() {
|
||||
return fmt.Errorf("refusing to heal: no clean backup copy found for %s", region)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// Orchestrate stop (panel-safe) if controller configured.
|
||||
if f.controller != "" && f.instance != "" {
|
||||
fmt.Fprintf(os.Stderr, "stopping %s via panel...\n", f.instance)
|
||||
if err := panelAction(ctx, f.controller, f.instance, "stop"); err != nil {
|
||||
return fmt.Errorf("panel stop: %w", err)
|
||||
}
|
||||
if f.container != "" {
|
||||
if err := waitContainer(ctx, f.container, false, 90*time.Second); err != nil {
|
||||
return fmt.Errorf("wait stopped: %w", err)
|
||||
}
|
||||
} else {
|
||||
time.Sleep(8 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
snapDir := orDefault(f.snapshotDir, f.regionDir+"/_medic_snapshots")
|
||||
qDir := orDefault(f.quarantineDir, f.regionDir+"/_medic_quarantine")
|
||||
res, err := regionmedic.ApplyHeal(plan, snapDir, qDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("apply heal: %w", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "HEALED %s: wrote %d bytes from %s; snapshot=%s; quarantined=%d\n",
|
||||
res.Region, res.BytesWritten, plan.Source.Backup.ID, res.SnapshotPath, len(res.QuarantinedFiles))
|
||||
printJSON(res)
|
||||
|
||||
// Start back up (panel-safe) + verify.
|
||||
if f.controller != "" && f.instance != "" {
|
||||
fmt.Fprintf(os.Stderr, "starting %s via panel...\n", f.instance)
|
||||
if err := panelAction(ctx, f.controller, f.instance, "start"); err != nil {
|
||||
return fmt.Errorf("panel start: %w", err)
|
||||
}
|
||||
if f.container != "" {
|
||||
fmt.Fprintln(os.Stderr, "watching boot for readiness + new corruption...")
|
||||
if err := watchBoot(ctx, f.container, f.regionDir, region, 240*time.Second); err != nil {
|
||||
return fmt.Errorf("post-heal verify: %w", err)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "VERIFY OK: instance ready, no new Wrong chunk header for the healed region")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func summarizePlan(plan regionmedic.HealPlan) {
|
||||
fmt.Fprintf(os.Stderr, "region %s: live healthy=%v (badChunks=%d, err=%q)\n",
|
||||
plan.Region, plan.LiveReport.Healthy(), len(plan.LiveReport.BadChunks), plan.LiveReport.Err)
|
||||
fmt.Fprintf(os.Stderr, "error_backups for region: %d\n", len(plan.ErrorBackups))
|
||||
for _, c := range plan.Evaluated {
|
||||
status := "no region in backup"
|
||||
if c.Found {
|
||||
if c.Report.Healthy() {
|
||||
status = fmt.Sprintf("CLEAN (%d chunks)", c.Report.PresentChunks)
|
||||
} else {
|
||||
status = fmt.Sprintf("corrupt (%d bad chunks, err=%q)", len(c.Report.BadChunks), c.Report.Err)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " %s %s %s\n", c.Backup.CreatedAt.Format(time.RFC3339), c.Backup.ID, status)
|
||||
}
|
||||
if plan.Ready() {
|
||||
fmt.Fprintf(os.Stderr, "=> heal source: %s (%s)\n", plan.Source.Backup.ID, plan.Source.Backup.CreatedAt.Format(time.RFC3339))
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "=> NO clean source found")
|
||||
}
|
||||
}
|
||||
|
||||
// --- orchestration helpers --------------------------------------------------
|
||||
|
||||
func panelAction(ctx context.Context, controller, instance, action string) error {
|
||||
url := strings.TrimRight(controller, "/") + "/api/instances/" + instance + "/" + action
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, bytes.TrimSpace(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitContainer polls `docker inspect` until the container's Running state
|
||||
// matches want, or timeout.
|
||||
func waitContainer(ctx context.Context, name string, want bool, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
out, _ := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.State.Running}}", name).Output()
|
||||
running := strings.TrimSpace(string(out)) == "true"
|
||||
if running == want {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for %s running=%v", name, want)
|
||||
}
|
||||
|
||||
// watchBoot tails the container log until the 7DTD ready marker appears, while
|
||||
// asserting no "Wrong chunk header!" reappears AND no new error_backup files
|
||||
// land for the healed region.
|
||||
func watchBoot(ctx context.Context, container, regionDir string, region regionmedic.RegionCoord, timeout time.Duration) error {
|
||||
if err := waitContainer(ctx, container, true, 60*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
readyRe := []string{"StartGame done", "GameStartDone", "by Telnet from"}
|
||||
for time.Now().Before(deadline) {
|
||||
out, _ := exec.CommandContext(ctx, "docker", "logs", "--since", "5m", container).CombinedOutput()
|
||||
s := string(out)
|
||||
if strings.Contains(s, "Wrong chunk header") {
|
||||
// Only fail if it's for our region — re-scan the dir to be sure.
|
||||
scan, _ := regionmedic.ScanRegionDir(regionDir)
|
||||
for _, a := range scan.Affected {
|
||||
if a.Region == region {
|
||||
return fmt.Errorf("healed region %s threw new error_backups (%d) after restart", region, a.ErrorBackups)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range readyRe {
|
||||
if strings.Contains(s, m) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(4 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for ready marker")
|
||||
}
|
||||
|
||||
func regionArg(s string) string {
|
||||
if strings.HasSuffix(s, ".7rg") {
|
||||
return s
|
||||
}
|
||||
return s + ".7rg"
|
||||
}
|
||||
|
||||
func orDefault(s, def string) string {
|
||||
if s == "" {
|
||||
return def
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
module github.com/dbledeez/panel/agent
|
||||
|
||||
go 1.26
|
||||
|
||||
require (
|
||||
github.com/bodgit/sevenzip v1.6.2
|
||||
github.com/dbledeez/panel/pkg v0.0.0
|
||||
github.com/dbledeez/panel/proto v0.0.0
|
||||
github.com/docker/docker v28.5.2+incompatible
|
||||
github.com/docker/go-connections v0.5.0
|
||||
github.com/moby/go-archive v0.2.0
|
||||
github.com/nwaples/rardecode/v2 v2.2.2
|
||||
github.com/ulikunitz/xz v0.5.15
|
||||
golang.org/x/net v0.52.0
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/patternmatcher v0.6.0 // indirect
|
||||
github.com/moby/sys/atomicwriter v0.1.0 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/morikuni/aec v1.1.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.26 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go4.org v0.0.0-20260112195520-a5071408f32f // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
replace (
|
||||
github.com/dbledeez/panel/pkg => ../pkg
|
||||
github.com/dbledeez/panel/proto => ../proto
|
||||
)
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
|
||||
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
|
||||
github.com/bodgit/sevenzip v1.6.2 h1:6/0mwj5KaRXpuf9iSiE+VpG7VpzFJ8D60P53VjxRv34=
|
||||
github.com/bodgit/sevenzip v1.6.2/go.mod h1:q8DktB7GbvNn0Q6u4Iq6zULE0vo3rWtRHQg5L1XmjuU=
|
||||
github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
|
||||
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
|
||||
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
|
||||
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
|
||||
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
|
||||
github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU=
|
||||
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
|
||||
github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
|
||||
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
|
||||
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
|
||||
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
|
||||
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
|
||||
github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs=
|
||||
github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
|
||||
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
|
||||
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
|
||||
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
|
||||
github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU=
|
||||
github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
|
||||
github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
||||
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
|
||||
go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
|
||||
go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw=
|
||||
go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
@@ -0,0 +1,388 @@
|
||||
// Package archive walks and re-encodes archive files (zip / tar /
|
||||
// tar.gz / tar.bz2 / tar.xz / 7z / rar) fully in-process — no native
|
||||
// deps. It is shared by the file browser's extract/compress endpoints
|
||||
// (internal/dispatch) and the direct update provider
|
||||
// (internal/updater), which extracts archive downloads into the
|
||||
// instance instead of dumping the raw archive bytes.
|
||||
//
|
||||
// 7z and rar use pure-Go libraries (bodgit/sevenzip,
|
||||
// nwaples/rardecode/v2). Encrypted rars and solid 7z volumes are
|
||||
// surfaced as errors, not silently skipped.
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bodgit/sevenzip"
|
||||
"github.com/nwaples/rardecode/v2"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// Entry-count / byte caps so a malformed or hostile archive can't pin
|
||||
// the agent forever walking phantom records.
|
||||
const (
|
||||
MaxEntries = 200000
|
||||
MaxTotalBytes = 8 * 1024 * 1024 * 1024
|
||||
EntrySoftBytes = 4 * 1024 * 1024 * 1024
|
||||
)
|
||||
|
||||
// EntryFunc is called once per archive entry. body is a streaming
|
||||
// reader (nil for directories); the callback may read fully or skip.
|
||||
type EntryFunc func(name string, mode int64, isDir bool, body io.Reader) error
|
||||
|
||||
// Walk opens `data` as an archive (auto-detected via signature plus
|
||||
// filename hint) and calls fn for each entry. Returns entry and byte
|
||||
// counts alongside any error.
|
||||
func Walk(data []byte, name string, fn EntryFunc) (int64, int64, error) {
|
||||
switch DetectFormat(data, name) {
|
||||
case "zip":
|
||||
return walkZip(data, fn)
|
||||
case "tar.gz":
|
||||
gz, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("gzip open: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
return walkTar(gz, fn)
|
||||
case "tar.bz2":
|
||||
return walkTar(bzip2.NewReader(bytes.NewReader(data)), fn)
|
||||
case "tar.xz":
|
||||
xr, err := xz.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("xz open: %w", err)
|
||||
}
|
||||
return walkTar(xr, fn)
|
||||
case "tar":
|
||||
return walkTar(bytes.NewReader(data), fn)
|
||||
case "7z":
|
||||
return walk7z(data, fn)
|
||||
case "rar":
|
||||
return walkRar(data, fn)
|
||||
default:
|
||||
return 0, 0, fmt.Errorf("unsupported archive format for %q (panel knows zip / tar / tar.gz / tar.bz2 / tar.xz / 7z / rar)", name)
|
||||
}
|
||||
}
|
||||
|
||||
// WriteAsTar parses an archive from `data` and re-emits its entries as
|
||||
// a tar stream into `out`. Used by container-side extract paths so the
|
||||
// agent can feed Docker's CopyToContainer in a single API call.
|
||||
func WriteAsTar(data []byte, originalName string, out io.Writer) (int64, int64, error) {
|
||||
tw := tar.NewWriter(out)
|
||||
defer tw.Close()
|
||||
var entries, bytesOut int64
|
||||
_, _, err := Walk(data, originalName, func(name string, mode int64, isDir bool, body io.Reader) error {
|
||||
safe := SafeEntryPath(name)
|
||||
if safe == "" {
|
||||
return nil
|
||||
}
|
||||
// Buffer the entry body so we can compute its length before
|
||||
// writing the tar header. zip / gzip readers return streaming
|
||||
// readers; the tar writer needs Size up front.
|
||||
var bodyBuf bytes.Buffer
|
||||
var sz int64
|
||||
if !isDir {
|
||||
n, err := io.Copy(&bodyBuf, body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("entry %q: %w", name, err)
|
||||
}
|
||||
sz = n
|
||||
if sz > EntrySoftBytes {
|
||||
return fmt.Errorf("entry %q exceeds %d-byte cap", name, int64(EntrySoftBytes))
|
||||
}
|
||||
}
|
||||
hdr := &tar.Header{Name: safe, ModTime: time.Now(), Mode: 0o644}
|
||||
if mode != 0 {
|
||||
hdr.Mode = mode & 0o777
|
||||
}
|
||||
if isDir {
|
||||
hdr.Typeflag = tar.TypeDir
|
||||
hdr.Mode = 0o755
|
||||
if !strings.HasSuffix(hdr.Name, "/") {
|
||||
hdr.Name += "/"
|
||||
}
|
||||
} else {
|
||||
hdr.Typeflag = tar.TypeReg
|
||||
hdr.Size = sz
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isDir {
|
||||
if _, err := tw.Write(bodyBuf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
entries++
|
||||
bytesOut += sz
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
return entries, bytesOut, tw.Close()
|
||||
}
|
||||
|
||||
// DetectFormat classifies `data` by magic bytes first (handles renamed
|
||||
// extensions), then falls back to the filename hint. Returns "" when
|
||||
// the content is not a recognized archive.
|
||||
func DetectFormat(data []byte, name string) string {
|
||||
lower := strings.ToLower(name)
|
||||
// Magic bytes win when present — handles renamed extensions.
|
||||
if len(data) >= 4 && data[0] == 0x50 && data[1] == 0x4b && (data[2] == 0x03 || data[2] == 0x05 || data[2] == 0x07) {
|
||||
return "zip"
|
||||
}
|
||||
if len(data) >= 3 && data[0] == 0x1f && data[1] == 0x8b && data[2] == 0x08 {
|
||||
return "tar.gz"
|
||||
}
|
||||
if len(data) >= 3 && data[0] == 'B' && data[1] == 'Z' && data[2] == 'h' {
|
||||
return "tar.bz2"
|
||||
}
|
||||
// 7z signature: 37 7A BC AF 27 1C
|
||||
if len(data) >= 6 && data[0] == 0x37 && data[1] == 0x7a && data[2] == 0xbc && data[3] == 0xaf && data[4] == 0x27 && data[5] == 0x1c {
|
||||
return "7z"
|
||||
}
|
||||
// RAR 5.0 signature: 52 61 72 21 1A 07 01 00 ("Rar!\x1a\x07\x01\x00")
|
||||
// RAR 1.5–4.x: 52 61 72 21 1A 07 00 ("Rar!\x1a\x07\x00")
|
||||
if len(data) >= 7 && data[0] == 'R' && data[1] == 'a' && data[2] == 'r' && data[3] == '!' && data[4] == 0x1a && data[5] == 0x07 {
|
||||
return "rar"
|
||||
}
|
||||
// xz signature: FD 37 7A 58 5A 00
|
||||
if len(data) >= 6 && data[0] == 0xfd && data[1] == '7' && data[2] == 'z' && data[3] == 'X' && data[4] == 'Z' && data[5] == 0x00 {
|
||||
return "tar.xz"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") {
|
||||
return "tar.gz"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.bz2") || strings.HasSuffix(lower, ".tbz2") || strings.HasSuffix(lower, ".tbz") {
|
||||
return "tar.bz2"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.xz") || strings.HasSuffix(lower, ".txz") {
|
||||
return "tar.xz"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar") {
|
||||
return "tar"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".zip") {
|
||||
return "zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".7z") {
|
||||
return "7z"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".rar") {
|
||||
return "rar"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func walkZip(data []byte, fn EntryFunc) (int64, int64, error) {
|
||||
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("zip open: %w", err)
|
||||
}
|
||||
var entries, bytesOut int64
|
||||
var totalBytes int64
|
||||
for _, f := range zr.File {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
isDir := f.FileInfo().IsDir() || strings.HasSuffix(f.Name, "/")
|
||||
if isDir {
|
||||
if err := fn(f.Name, int64(f.Mode()), true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("zip entry %q: %w", f.Name, err)
|
||||
}
|
||||
// Cap each entry by `EntrySoftBytes` so a malformed/zip-bomb
|
||||
// entry can't run away with memory. We DO NOT trust
|
||||
// f.UncompressedSize64 — some zip writers leave it 0 for streamed
|
||||
// entries; using it for accounting (bytesOut += size) caused the
|
||||
// per-archive total to be charged the 4 GB soft cap per such
|
||||
// entry, tripping MaxTotalBytes after just a couple of
|
||||
// files and aborting the extract mid-stream — silently leaving
|
||||
// the first N files extracted and the rest dropped (this was the
|
||||
// root cause of the 7DTD mod-upload "Config/ folders missing"
|
||||
// bug). Use a counting reader so accounting reflects real bytes.
|
||||
cr := &countingReader{r: io.LimitReader(rc, EntrySoftBytes)}
|
||||
if err := fn(f.Name, int64(f.Mode()), false, cr); err != nil {
|
||||
_ = rc.Close()
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
_ = rc.Close()
|
||||
entries++
|
||||
bytesOut += cr.n
|
||||
totalBytes += cr.n
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
|
||||
// countingReader wraps an io.Reader and tracks the byte count actually
|
||||
// consumed. Used by walkZip / walkTar accounting so the per-archive
|
||||
// total reflects real-world bytes, not zip-header declared sizes (which
|
||||
// some encoders leave as 0).
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func walkTar(r io.Reader, fn EntryFunc) (int64, int64, error) {
|
||||
tr := tar.NewReader(r)
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("tar next: %w", err)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := fn(hdr.Name, hdr.Mode, true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
if err := fn(hdr.Name, hdr.Mode, false, tr); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
bytesOut += hdr.Size
|
||||
totalBytes += hdr.Size
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func walk7z(data []byte, fn EntryFunc) (int64, int64, error) {
|
||||
zr, err := sevenzip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("7z open: %w", err)
|
||||
}
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for _, f := range zr.File {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
isDir := f.FileInfo().IsDir() || strings.HasSuffix(f.Name, "/")
|
||||
if isDir {
|
||||
if err := fn(f.Name, int64(f.Mode()), true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("7z entry %q: %w", f.Name, err)
|
||||
}
|
||||
// See countingReader rationale in walkZip — header-declared
|
||||
// size can be 0 for streamed entries, charging the soft-cap per
|
||||
// entry would falsely abort the extract.
|
||||
cr := &countingReader{r: io.LimitReader(rc, EntrySoftBytes)}
|
||||
if err := fn(f.Name, int64(f.Mode()), false, cr); err != nil {
|
||||
_ = rc.Close()
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
_ = rc.Close()
|
||||
entries++
|
||||
bytesOut += cr.n
|
||||
totalBytes += cr.n
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
|
||||
func walkRar(data []byte, fn EntryFunc) (int64, int64, error) {
|
||||
rr, err := rardecode.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("rar open: %w", err)
|
||||
}
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
hdr, err := rr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("rar next: %w", err)
|
||||
}
|
||||
mode := int64(hdr.Mode().Perm())
|
||||
if hdr.IsDir {
|
||||
if err := fn(hdr.Name, mode, true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
continue
|
||||
}
|
||||
// See countingReader rationale in walkZip.
|
||||
cr := &countingReader{r: io.LimitReader(rr, EntrySoftBytes)}
|
||||
if err := fn(hdr.Name, mode, false, cr); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
bytesOut += cr.n
|
||||
totalBytes += cr.n
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SafeEntryPath strips leading slashes / drive letters and
|
||||
// rejects any path that tries to ../ out of the destination. Returns
|
||||
// "" on rejected entries — caller skips them silently.
|
||||
func SafeEntryPath(name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
// Normalize separators + strip leading drive / slash.
|
||||
clean := strings.ReplaceAll(name, "\\", "/")
|
||||
for strings.HasPrefix(clean, "/") {
|
||||
clean = clean[1:]
|
||||
}
|
||||
if len(clean) >= 2 && clean[1] == ':' {
|
||||
clean = clean[2:]
|
||||
}
|
||||
clean = path.Clean(clean)
|
||||
if clean == "." || clean == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(clean, "../") || clean == ".." || strings.Contains(clean, "/../") {
|
||||
return ""
|
||||
}
|
||||
return clean
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// buildTar writes a small dir+file tree as a tar stream.
|
||||
func buildTar(t *testing.T, w io.Writer) {
|
||||
t.Helper()
|
||||
tw := tar.NewWriter(w)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "factorio/", Typeflag: tar.TypeDir, Mode: 0o755}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("#!/bin/sh\necho hi\n")
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "factorio/bin/x64/factorio", Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func makeTarXz(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
xw, err := xz.NewWriter(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buildTar(t, xw)
|
||||
if err := xw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func makeTarGz(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
buildTar(t, gw)
|
||||
if err := gw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func makeZip(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
f, err := zw.Create("factorio/bin/x64/factorio")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.Write([]byte("#!/bin/sh\necho hi\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestDetectFormatMagicBytes(t *testing.T) {
|
||||
// Extensionless names — magic bytes must carry detection (factorio's
|
||||
// download URL has no file extension).
|
||||
cases := []struct {
|
||||
data []byte
|
||||
want string
|
||||
}{
|
||||
{makeTarXz(t), "tar.xz"},
|
||||
{makeTarGz(t), "tar.gz"},
|
||||
{makeZip(t), "zip"},
|
||||
{[]byte("plain text, not an archive"), ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := DetectFormat(c.data, "https://example.com/get-download/stable/headless/linux64"); got != c.want {
|
||||
t.Errorf("DetectFormat = %q, want %q", got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectFormatExtensionFallback(t *testing.T) {
|
||||
// Content too short for magic — extension hint decides.
|
||||
if got := DetectFormat([]byte{}, "thing.tar.xz"); got != "tar.xz" {
|
||||
t.Errorf("extension fallback = %q, want tar.xz", got)
|
||||
}
|
||||
if got := DetectFormat([]byte{}, "thing.tgz"); got != "tar.gz" {
|
||||
t.Errorf("extension fallback = %q, want tar.gz", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkAllFormats(t *testing.T) {
|
||||
for name, data := range map[string][]byte{
|
||||
"tar.xz": makeTarXz(t),
|
||||
"tar.gz": makeTarGz(t),
|
||||
"zip": makeZip(t),
|
||||
} {
|
||||
var files []string
|
||||
var content []byte
|
||||
entries, _, err := Walk(data, "blob", func(n string, mode int64, isDir bool, body io.Reader) error {
|
||||
if isDir {
|
||||
return nil
|
||||
}
|
||||
files = append(files, n)
|
||||
b, err := io.ReadAll(body)
|
||||
content = b
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: Walk: %v", name, err)
|
||||
}
|
||||
if entries == 0 || len(files) != 1 || files[0] != "factorio/bin/x64/factorio" {
|
||||
t.Fatalf("%s: unexpected entries %d files %v", name, entries, files)
|
||||
}
|
||||
if string(content) != "#!/bin/sh\necho hi\n" {
|
||||
t.Fatalf("%s: content mismatch: %q", name, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteAsTarRoundTrip(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
entries, bytesOut, err := WriteAsTar(makeTarXz(t), "blob", &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entries != 2 {
|
||||
t.Fatalf("entries = %d, want 2", entries)
|
||||
}
|
||||
if bytesOut == 0 {
|
||||
t.Fatal("bytesOut = 0")
|
||||
}
|
||||
// Re-walk the emitted tar and check the file mode survived.
|
||||
tr := tar.NewReader(&out)
|
||||
var sawFile bool
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeReg {
|
||||
sawFile = true
|
||||
if hdr.Name != "factorio/bin/x64/factorio" {
|
||||
t.Fatalf("name = %q", hdr.Name)
|
||||
}
|
||||
if hdr.Mode&0o111 == 0 {
|
||||
t.Fatalf("exec bit lost: mode %o", hdr.Mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !sawFile {
|
||||
t.Fatal("no regular file in re-emitted tar")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeEntryPathRejectsTraversal(t *testing.T) {
|
||||
for _, bad := range []string{"../evil", "a/../../evil", "..", ""} {
|
||||
if got := SafeEntryPath(bad); got != "" {
|
||||
t.Errorf("SafeEntryPath(%q) = %q, want rejection", bad, got)
|
||||
}
|
||||
}
|
||||
if got := SafeEntryPath("/abs/path"); got != "abs/path" {
|
||||
t.Errorf("leading slash strip = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package dispatch
|
||||
|
||||
// ArkSaveRestore: swap a SavedArks/<subdir>/ rolling backup into the
|
||||
// active <subdir>.ark slot. The previously-active save is moved aside
|
||||
// (NEVER deleted) — even abandoned attempts are recoverable from the
|
||||
// timestamped *_replaced.ark filename.
|
||||
//
|
||||
// Sequence (all server-side, atomic from the operator's POV):
|
||||
// 1. Validate inputs.
|
||||
// 2. Confirm the main container is NOT running. We refuse the swap
|
||||
// if it is, because mid-write file ops in ARK's save dir corrupt
|
||||
// the rolling backup chain.
|
||||
// 3. mv <subdir>/<subdir>.ark -> <subdir>/<subdir>_<dd.mm.yyyy>_<HH.MM.SS>_replaced.ark
|
||||
// 4. cp <subdir>/<target_filename> -> <subdir>/<subdir>.ark
|
||||
// For .arkrbf rolling backups the dest still ends in .ark — the
|
||||
// engine doesn't care about the source file's extension, only the
|
||||
// slot name.
|
||||
//
|
||||
// Caller (controller) is responsible for stopping the container before
|
||||
// this RPC. If the operator started the container in the meantime,
|
||||
// this handler short-circuits cleanly with a friendly error.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// arkSavedArksContainerRoot is the relative path to ARK's rolling-save
|
||||
// directory FROM the ark-sa module's BrowseableRoot. The module roots
|
||||
// browsing at ".../ShooterGame/Saved" (see modules/ark-sa/module.yaml),
|
||||
// so this is the segment beneath that — NOT the full container path.
|
||||
const arkSavedArksContainerRoot = "SavedArks"
|
||||
|
||||
func (d *Dispatcher) handleArkSaveRestore(corrID string, req *panelv1.ArkSaveRestoreRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
subdir := strings.TrimSpace(req.SavedArksSubdir)
|
||||
target := strings.TrimSpace(req.TargetFilename)
|
||||
if subdir == "" || target == "" {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "saved_arks_subdir and target_filename are required")
|
||||
return
|
||||
}
|
||||
// Path-safety: subdir is one segment under SavedArks/, target is
|
||||
// one segment under that. Reject any traversal attempt up front.
|
||||
if strings.ContainsAny(subdir, "/\\") || subdir == "." || subdir == ".." {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "saved_arks_subdir must be a single directory name")
|
||||
return
|
||||
}
|
||||
if strings.ContainsAny(target, "/\\") || target == "." || target == ".." {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "target_filename must be a single file name")
|
||||
return
|
||||
}
|
||||
// Sanity-check extension. Active slot is .ark; sources may be .ark
|
||||
// or .arkrbf (rolling backup). .bak is ARK's anti-corruption file
|
||||
// and we don't restore from it directly — operators rename to .ark
|
||||
// manually if they need to.
|
||||
lowerTarget := strings.ToLower(target)
|
||||
if !strings.HasSuffix(lowerTarget, ".ark") && !strings.HasSuffix(lowerTarget, ".arkrbf") {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "target_filename must end in .ark or .arkrbf")
|
||||
return
|
||||
}
|
||||
|
||||
activeName := subdir + ".ark"
|
||||
if target == activeName {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "target is already the active save")
|
||||
return
|
||||
}
|
||||
|
||||
// Stamp the aside name with current wall time. We use the user's
|
||||
// "dd.mm.yyyy_HH.MM.SS" format from ARK's own rolling backups so
|
||||
// the aside file sorts naturally next to genuine engine backups
|
||||
// in the file browser. Suffix "_replaced" disambiguates them.
|
||||
// Uses UTC because that's what the engine writes — keeps the
|
||||
// frontend's parser (which assumes UTC) honest for both sources.
|
||||
now := time.Now().UTC()
|
||||
asideName := fmt.Sprintf("%s_%s_replaced.ark",
|
||||
subdir, now.Format("02.01.2006_15.04.05"))
|
||||
|
||||
if d.useContainerOps(rec) {
|
||||
d.arkRestoreInContainer(corrID, rec, subdir, target, activeName, asideName)
|
||||
return
|
||||
}
|
||||
d.arkRestoreOnHost(corrID, rec, subdir, target, activeName, asideName)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) arkRestoreInContainer(corrID string, rec *instanceRecord, subdir, target, activeName, asideName string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// Refuse if the main game container is still running. We don't
|
||||
// want file ops happening inside SavedArks while the engine is
|
||||
// writing rolling backups.
|
||||
mainName := "panel-" + rec.InstanceID
|
||||
if st, err := d.runtime.InspectByName(ctx, mainName); err == nil && st.Status == "running" {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "",
|
||||
"refusing to restore while server is running — stop the server first")
|
||||
return
|
||||
}
|
||||
|
||||
subdirRel := path.Join(arkSavedArksContainerRoot, subdir)
|
||||
subdirAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), subdirRel)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
activeAbs := path.Join(subdirAbs, activeName)
|
||||
asideAbs := path.Join(subdirAbs, asideName)
|
||||
targetAbs := path.Join(subdirAbs, target)
|
||||
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Verify the target file exists before we touch the active slot.
|
||||
// `test -f` returns exit 1 (no file), 0 (exists), other (error).
|
||||
_, _, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"test -f " + shellQuote(targetAbs)})
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", fmt.Sprintf("target save %q not found", target))
|
||||
return
|
||||
}
|
||||
|
||||
// Move the previously-active save aside (only if it exists; some
|
||||
// ops may run on a freshly-installed instance with no active save).
|
||||
finalAside := ""
|
||||
_, _, code, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"test -f " + shellQuote(activeAbs)})
|
||||
if code == 0 {
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mv -- " + shellQuote(activeAbs) + " " + shellQuote(asideAbs)})
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("mv aside exited %d", code)
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", msg)
|
||||
return
|
||||
}
|
||||
finalAside = asideName
|
||||
}
|
||||
|
||||
// Copy the chosen backup into the active slot. cp preserves the
|
||||
// source so the operator can pick it again later if they change
|
||||
// their mind. -p preserves mtime so the engine sees the original
|
||||
// save's modification time.
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"cp -p -- " + shellQuote(targetAbs) + " " + shellQuote(activeAbs)})
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("cp exited %d", code)
|
||||
}
|
||||
// Roll the aside back if cp failed mid-flight.
|
||||
if finalAside != "" {
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mv -- " + shellQuote(asideAbs) + " " + shellQuote(activeAbs)})
|
||||
finalAside = ""
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", msg)
|
||||
return
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, activeName, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) arkRestoreOnHost(corrID string, rec *instanceRecord, subdir, target, activeName, asideName string) {
|
||||
if rec.DataPath == "" {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "no file storage available")
|
||||
return
|
||||
}
|
||||
subdirRel := filepath.Join(arkSavedArksContainerRoot, subdir)
|
||||
subdirAbs, err := safeJoinHost(rec.DataPath, subdirRel)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
activeAbs := filepath.Join(subdirAbs, activeName)
|
||||
asideAbs := filepath.Join(subdirAbs, asideName)
|
||||
targetAbs := filepath.Join(subdirAbs, target)
|
||||
|
||||
if _, err := os.Stat(targetAbs); err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", fmt.Sprintf("target save %q not found", target))
|
||||
return
|
||||
}
|
||||
finalAside := ""
|
||||
if _, err := os.Stat(activeAbs); err == nil {
|
||||
if err := os.Rename(activeAbs, asideAbs); err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
finalAside = asideName
|
||||
}
|
||||
in, err := os.Open(targetAbs)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.OpenFile(activeAbs, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
// Roll aside back so we don't leave the server with no save.
|
||||
if finalAside != "" {
|
||||
_ = os.Rename(asideAbs, activeAbs)
|
||||
finalAside = ""
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
_ = out.Close()
|
||||
if finalAside != "" {
|
||||
_ = os.Remove(activeAbs)
|
||||
_ = os.Rename(asideAbs, activeAbs)
|
||||
finalAside = ""
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, activeName, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendArkSaveRestoreResult(corrID, asideFilename, installedFilename, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_ArkSaveRestoreResult{
|
||||
ArkSaveRestoreResult: &panelv1.ArkSaveRestoreResult{
|
||||
AsideFilename: asideFilename,
|
||||
InstalledFilename: installedFilename,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,505 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// buildBackupTarCmd assembles the tar command run inside the sidecar.
|
||||
// Manifest BackupSpec wins when Include is set: the tar runs against
|
||||
// the listed paths (each relative to BrowseableRoot, missing ones
|
||||
// silently skipped). Otherwise it falls back to a whole-tree archive
|
||||
// of BrowseableRoot. Excludes are passed through tar's --exclude.
|
||||
//
|
||||
// Resulting tarball preserves the directory layout under BrowseableRoot
|
||||
// so restore can extract straight into the same root.
|
||||
func buildBackupTarCmd(manifest *modulepkgManifest, root, fileName string, keep int) string {
|
||||
if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
|
||||
root = manifest.Backup.Root
|
||||
}
|
||||
var base string
|
||||
if manifest == nil || manifest.Backup == nil || len(manifest.Backup.Include) == 0 {
|
||||
// Fallback: whole tree.
|
||||
base = fmt.Sprintf("cd %q && tar czf /backup/%s . && stat -c %%s /backup/%s",
|
||||
root, shellEscape(fileName), shellEscape(fileName))
|
||||
} else {
|
||||
// Per-game include list. Missing paths are non-fatal — players will
|
||||
// create new instances on a fresh server before the save dir exists,
|
||||
// and we don't want their first Back-up-now to fail the run.
|
||||
excludes := ""
|
||||
for _, e := range manifest.Backup.Exclude {
|
||||
excludes += " --exclude=" + shellEscape(e)
|
||||
}
|
||||
// Build "test -e <path> && echo <path>" for each include — only
|
||||
// existing paths get added to the tar argv, so the operator doesn't
|
||||
// see a confusing "Cannot stat: No such file" on first run.
|
||||
//
|
||||
// Newline-delimited (NOT NUL-delimited) because the sidecar image is
|
||||
// alpine, which ships busybox tar — and busybox tar doesn't grok
|
||||
// `--null`. Manifest paths never contain whitespace or newlines (we
|
||||
// hand-curate them per game), so a plain newline list is safe.
|
||||
var existsChecks []string
|
||||
for _, p := range manifest.Backup.Include {
|
||||
clean := strings.TrimPrefix(p, "./")
|
||||
clean = strings.TrimPrefix(clean, "/")
|
||||
existsChecks = append(existsChecks, fmt.Sprintf("[ -e %s ] && printf '%%s\\n' %s", shellEscape(clean), shellEscape(clean)))
|
||||
}
|
||||
listExpr := "{ " + strings.Join(existsChecks, " ; ") + " ; }"
|
||||
base = fmt.Sprintf("cd %q && %s | tar czf /backup/%s%s -T - && stat -c %%s /backup/%s",
|
||||
root, listExpr, shellEscape(fileName), excludes, shellEscape(fileName))
|
||||
}
|
||||
return base + backupPruneSuffix(keep)
|
||||
}
|
||||
|
||||
// backupPruneSuffix appends a best-effort retention prune that keeps only the
|
||||
// newest `keep` *.tar.gz in /backup (by mtime) and deletes the rest. It runs in
|
||||
// the SAME root sidecar that just wrote the new tarball — the backup files are
|
||||
// root-owned, so the unprivileged agent can't prune them itself. keep<=0 leaves
|
||||
// pruning off. busybox-safe (alpine): ls -1t + tail -n +N + while-read — no
|
||||
// `head -n -N` (GNU-only) and no `xargs -r`.
|
||||
func backupPruneSuffix(keep int) string {
|
||||
if keep <= 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" ; echo 'prune: keeping newest %d backups' ; ls -1t /backup/*.tar.gz 2>/dev/null | tail -n +%d | while IFS= read -r f; do echo \"prune: rm $f\" ; rm -f \"$f\" ; done",
|
||||
keep, keep+1)
|
||||
}
|
||||
|
||||
// readBackupKeep returns the backup retention for this instance: the integer in
|
||||
// <dataPath>/backup-keep when present, else the PANEL_BACKUP_KEEP env default,
|
||||
// else 10. A rolling cap stops the unbounded pileup that filled 90 GB on a busy
|
||||
// 7DTD season. Set per-instance by writing the file (the Backups UI exposes it).
|
||||
func readBackupKeep(dataPath string) int {
|
||||
def := 10
|
||||
if v := os.Getenv("PANEL_BACKUP_KEEP"); v != "" {
|
||||
if n, e := strconv.Atoi(strings.TrimSpace(v)); e == nil && n > 0 {
|
||||
def = n
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dataPath, "backup-keep"))
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
if n, e := strconv.Atoi(strings.TrimSpace(string(b))); e == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// modulepkgManifest is a local alias so the helper signature below
|
||||
// doesn't drag modulepkg into its return type at the file top.
|
||||
type modulepkgManifest = modulepkg.Manifest
|
||||
|
||||
// Backup / restore via sidecar containers. Two operations:
|
||||
//
|
||||
// - Backup: tar -czf /backup/<file>.tar.gz -C /src . (volume readonly at /src)
|
||||
// - Restore: tar -xzf /backup/<file>.tar.gz -C /dst (volume read-write at /dst)
|
||||
//
|
||||
// The sidecar uses alpine:3.21 + busybox tar. Backup files live on the
|
||||
// agent host under $BACKUP_DIR/<instance_id>/<ts>-<bkpID>.tar.gz; both
|
||||
// sidecars bind-mount $BACKUP_DIR/<instance_id> at /backup.
|
||||
//
|
||||
// For restore we refuse if the main container is running: extracting
|
||||
// files under a live server would corrupt world state. Operator stops
|
||||
// first. (Symmetrical to the SteamCMD updater's running-container check.)
|
||||
|
||||
const (
|
||||
backupImage = "alpine:3.21"
|
||||
backupSidecarTTL = 30 * time.Minute
|
||||
)
|
||||
|
||||
// ---- Backup ----
|
||||
|
||||
func (d *Dispatcher) handleBackup(ctx context.Context, corrID string, req *panelv1.BackupRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||||
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, "", "", 0, err.Error())
|
||||
return
|
||||
}
|
||||
if rec.BrowseableRoot == "" {
|
||||
d.sendBackupResult(corrID, "", "", 0, "instance has no BrowseableRoot to tar (module declares no volumes)")
|
||||
return
|
||||
}
|
||||
if d.backupDir == "" {
|
||||
d.sendBackupResult(corrID, "", "", 0, "agent has no --backup-dir configured")
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare the agent-side target directory.
|
||||
bkpID := "bkp_" + randHex(8)
|
||||
hostDir := filepath.Join(d.backupDir, req.InstanceId)
|
||||
if err := os.MkdirAll(hostDir, 0o755); err != nil {
|
||||
d.sendBackupResult(corrID, "", "", 0, fmt.Sprintf("mkdir %s: %s", hostDir, err.Error()))
|
||||
return
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
fileName := fmt.Sprintf("%s-%s.tar.gz", ts, bkpID)
|
||||
hostFile := filepath.Join(hostDir, fileName)
|
||||
|
||||
log.Info("backup starting", "backup_id", bkpID, "host_file", hostFile)
|
||||
|
||||
// Build sidecar spec that mounts the same volumes as the instance,
|
||||
// plus the host backup directory at /backup.
|
||||
volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
|
||||
// Mark volumes read-only for backup safety.
|
||||
for i := range volumes {
|
||||
volumes[i].ReadOnly = true
|
||||
}
|
||||
// Add the host backup dir as a (read-write) bind.
|
||||
volumes = append(volumes, runtime.VolumeSpec{
|
||||
Type: "bind",
|
||||
HostPath: hostDir,
|
||||
ContainerPath: "/backup",
|
||||
})
|
||||
|
||||
// Per-game backup paths. If the manifest declares Backup.Include,
|
||||
// only those paths get archived (saves-only — keeps ARK SA tarballs
|
||||
// at ~250 MB instead of the 18 GB whole-install). Without a spec,
|
||||
// fall back to the original "tar everything under BrowseableRoot"
|
||||
// behavior so older modules don't regress.
|
||||
manifest := d.modulesMustGet(rec.ModuleID)
|
||||
keep := readBackupKeep(rec.DataPath)
|
||||
tarCmd := buildBackupTarCmd(manifest, rec.BrowseableRoot, fileName, keep)
|
||||
if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
|
||||
log.Info("backup using manifest-declared paths", "include_count", len(manifest.Backup.Include))
|
||||
} else {
|
||||
log.Warn("backup falling back to whole BrowseableRoot — module manifest has no `backup.include`",
|
||||
"module_id", rec.ModuleID, "root", rec.BrowseableRoot)
|
||||
}
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: req.InstanceId + "-backup",
|
||||
IsSidecar: true,
|
||||
Image: backupImage,
|
||||
Entrypoint: []string{"sh"},
|
||||
Command: []string{"-c", tarCmd},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, "create sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, "start sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
// Stream log lines for visibility.
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID, Stream: "backup",
|
||||
At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, "wait: "+err.Error())
|
||||
return
|
||||
}
|
||||
if exitCode != 0 {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, fmt.Sprintf("sidecar exited %d", exitCode))
|
||||
return
|
||||
}
|
||||
|
||||
info, err := os.Stat(hostFile)
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, hostFile, 0, "stat result: "+err.Error())
|
||||
return
|
||||
}
|
||||
log.Info("backup complete", "backup_id", bkpID, "size_bytes", info.Size())
|
||||
d.sendBackupResult(corrID, bkpID, hostFile, info.Size(), "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendBackupResult(corrID, bkpID, path string, size int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_BackupResult{
|
||||
BackupResult: &panelv1.BackupResult{
|
||||
BackupId: bkpID,
|
||||
Path: path,
|
||||
SizeBytes: size,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- Wipe paths (used by the env-config recreate flow) ----
|
||||
//
|
||||
// Spawns an alpine sidecar that mounts the same volumes the instance
|
||||
// uses, runs `rm -rf` on each path (relative to BrowseableRoot), then
|
||||
// exits. Designed for the ARK map-switch flow — operator picked a new
|
||||
// SERVER_MAP and checked "wipe existing world data" so the old map's
|
||||
// SavedArks/<MapName>/ goes away. Generic enough that any future
|
||||
// env-config field can declare its own wipe paths.
|
||||
func (d *Dispatcher) runWipeSidecar(ctx context.Context, manifest *modulepkg.Manifest, instanceID, dataPath string, paths []string) error {
|
||||
if manifest == nil {
|
||||
return errors.New("nil manifest")
|
||||
}
|
||||
root := resolveBrowseableRoot(manifest)
|
||||
if root == "" {
|
||||
return errors.New("module declares no browseable_root — wipe needs a base path")
|
||||
}
|
||||
volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
|
||||
// Build the rm -rf command. Each path is rooted under
|
||||
// BrowseableRoot. Missing paths are silently skipped (first run).
|
||||
var rmParts []string
|
||||
for _, p := range paths {
|
||||
clean := p
|
||||
// Defensive: reject absolute paths and parent traversal so a
|
||||
// rogue config_values value can't `rm -rf /` the container.
|
||||
if strings.HasPrefix(clean, "/") || strings.Contains(clean, "..") {
|
||||
return fmt.Errorf("refusing wipe of suspicious path %q", p)
|
||||
}
|
||||
full := root + "/" + clean
|
||||
rmParts = append(rmParts, fmt.Sprintf("rm -rf %s", shellEscape(full)))
|
||||
}
|
||||
cmd := strings.Join(rmParts, " && ") + " && echo wiped"
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: instanceID + "-wipe",
|
||||
IsSidecar: true,
|
||||
Image: backupImage,
|
||||
Entrypoint: []string{"sh"},
|
||||
Command: []string{"-c", cmd},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create wipe sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return fmt.Errorf("start wipe sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: instanceID, Stream: "wipe",
|
||||
At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return fmt.Errorf("wait wipe sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return fmt.Errorf("wipe sidecar exited %d", exitCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- Restore ----
|
||||
|
||||
func (d *Dispatcher) handleRestore(ctx context.Context, corrID string, req *panelv1.RestoreRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "backup_id", req.BackupId, "correlation_id", corrID)
|
||||
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendRestoreResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if rec.BrowseableRoot == "" {
|
||||
d.sendRestoreResult(corrID, 0, "instance has no BrowseableRoot to restore into")
|
||||
return
|
||||
}
|
||||
if req.BackupPath == "" {
|
||||
d.sendRestoreResult(corrID, 0, "backup_path required")
|
||||
return
|
||||
}
|
||||
// Refuse if container is running.
|
||||
if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil && state.Status == "running" {
|
||||
d.sendRestoreResult(corrID, 0, "stop the instance first — main container is running")
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(req.BackupPath)
|
||||
if err != nil || info.IsDir() {
|
||||
d.sendRestoreResult(corrID, 0, "backup file missing or is a directory")
|
||||
return
|
||||
}
|
||||
|
||||
volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
|
||||
// Add the backup file's parent as a read-only bind at /backup.
|
||||
hostDir := filepath.Dir(req.BackupPath)
|
||||
fileName := filepath.Base(req.BackupPath)
|
||||
volumes = append(volumes, runtime.VolumeSpec{
|
||||
Type: "bind",
|
||||
HostPath: hostDir,
|
||||
ContainerPath: "/backup",
|
||||
ReadOnly: true,
|
||||
})
|
||||
|
||||
// Extract strategy depends on whether the backup is per-game-paths
|
||||
// or a whole-tree archive. When the manifest declares Backup.Include,
|
||||
// only THOSE paths are wiped before extraction — preserves the
|
||||
// 18 GB game install while still giving a clean save dir. Without a
|
||||
// spec, fall back to wiping all of BrowseableRoot (legacy whole-tree
|
||||
// behavior).
|
||||
manifest := d.modulesMustGet(rec.ModuleID)
|
||||
// Resolve the SAME root the backup wrote against. buildBackupTarCmd
|
||||
// archives relative to manifest.Backup.Root when set (e.g. 7DTD's
|
||||
// /game-saves, where world data actually lives — distinct from the
|
||||
// /game install volume that is BrowseableRoot). Restore MUST extract
|
||||
// into that same root or the tarball lands in the wrong volume and the
|
||||
// live server never sees it (silent data loss: restore "succeeds" but
|
||||
// players' bases are gone). Mirror the backup's resolution exactly.
|
||||
restoreRoot := rec.BrowseableRoot
|
||||
if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
|
||||
restoreRoot = manifest.Backup.Root
|
||||
}
|
||||
var wipeCmd string
|
||||
if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
|
||||
// Wipe only the manifest-declared include paths so we don't
|
||||
// nuke unrelated install files. Each path is rm -rf'd; missing
|
||||
// ones are skipped silently (first-run case).
|
||||
var parts []string
|
||||
for _, p := range manifest.Backup.Include {
|
||||
clean := strings.TrimPrefix(strings.TrimPrefix(p, "./"), "/")
|
||||
parts = append(parts, fmt.Sprintf("rm -rf %s/%s", shellEscape(restoreRoot), shellEscape(clean)))
|
||||
}
|
||||
wipeCmd = strings.Join(parts, " ; ") + " 2>/dev/null"
|
||||
log.Info("restore: wiping manifest-declared paths only", "root", restoreRoot, "include_count", len(manifest.Backup.Include))
|
||||
} else {
|
||||
wipeCmd = fmt.Sprintf("rm -rf %q/* %q/.[!.]* 2>/dev/null", restoreRoot, restoreRoot)
|
||||
log.Warn("restore: wiping entire root (whole-tree backup)", "root", restoreRoot)
|
||||
}
|
||||
extractCmd := fmt.Sprintf(
|
||||
"%s ; tar xzf /backup/%s -C %q && du -sb %q | cut -f1",
|
||||
wipeCmd,
|
||||
shellEscape(fileName),
|
||||
restoreRoot,
|
||||
restoreRoot,
|
||||
)
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: req.InstanceId + "-restore",
|
||||
IsSidecar: true,
|
||||
Image: backupImage,
|
||||
Entrypoint: []string{"sh"},
|
||||
Command: []string{"-c", extractCmd},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
d.sendRestoreResult(corrID, 0, "create sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
d.sendRestoreResult(corrID, 0, "start sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID, Stream: "restore",
|
||||
At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
d.sendRestoreResult(corrID, 0, "wait: "+err.Error())
|
||||
return
|
||||
}
|
||||
if exitCode != 0 {
|
||||
d.sendRestoreResult(corrID, 0, fmt.Sprintf("sidecar exited %d", exitCode))
|
||||
return
|
||||
}
|
||||
log.Info("restore complete")
|
||||
d.sendRestoreResult(corrID, info.Size(), "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendRestoreResult(corrID string, bytesRestored int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_RestoreResult{
|
||||
RestoreResult: &panelv1.RestoreResult{BytesRestored: bytesRestored, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
func randHex(n int) string {
|
||||
b := make([]byte, n)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// shellEscape wraps a filename in single quotes for safe inclusion in an
|
||||
// sh -c command. Assumes filename never contains a single-quote (ours
|
||||
// never do — timestamps + bkp_<hex>).
|
||||
func shellEscape(s string) string { return "'" + s + "'" }
|
||||
|
||||
// modulesMustGet returns the dispatcher's manifest for the given id, or
|
||||
// nil if it's gone missing (shouldn't happen for a live instance).
|
||||
func (d *Dispatcher) modulesMustGet(id string) *modulepkgManifest {
|
||||
m, _ := d.modules.Get(id)
|
||||
return m
|
||||
}
|
||||
|
||||
// Keep errors imported for future error-wrapping.
|
||||
var _ = errors.New
|
||||
@@ -0,0 +1,198 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Backup browse — list / read entries inside a backup tarball without
|
||||
// extracting it. The agent has direct host access to the .tar.gz under
|
||||
// $BACKUP_DIR, so we don't need a sidecar; standard library archive/tar
|
||||
// + compress/gzip handles both the list and the single-file extract.
|
||||
//
|
||||
// Sized for safety: BackupReadFile caps the in-memory buffer per-call
|
||||
// (default 8 MiB) so a malicious or corrupt tar entry can't OOM the
|
||||
// agent. Truncation is reported back to the caller so the UI can show
|
||||
// "(truncated, download to see full file)".
|
||||
|
||||
const (
|
||||
defaultBackupReadCap = 8 << 20 // 8 MiB
|
||||
maxBackupReadCap = 64 << 20 // 64 MiB hard ceiling regardless of caller request
|
||||
binarySniffLen = 8 << 10 // 8 KiB
|
||||
)
|
||||
|
||||
func (d *Dispatcher) handleBackupListEntries(corrID string, req *panelv1.BackupListEntriesRequest) {
|
||||
d.log.Info("backup list entries: start", "correlation_id", corrID, "backup_path", req.BackupPath, "instance_id", req.InstanceId)
|
||||
if req.BackupPath == "" {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "backup_path required")
|
||||
return
|
||||
}
|
||||
f, err := os.Open(req.BackupPath)
|
||||
if err != nil {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "open backup: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "gzip: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(gz)
|
||||
|
||||
var entries []*panelv1.BackupTarEntry
|
||||
var totalBytes int64
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "tar header: "+err.Error())
|
||||
return
|
||||
}
|
||||
isDir := hdr.Typeflag == tar.TypeDir || strings.HasSuffix(hdr.Name, "/")
|
||||
clean := strings.TrimPrefix(hdr.Name, "./")
|
||||
clean = strings.TrimPrefix(clean, "/")
|
||||
entry := &panelv1.BackupTarEntry{
|
||||
Path: clean,
|
||||
Size: hdr.Size,
|
||||
IsDir: isDir,
|
||||
Mode: hdr.FileInfo().Mode().String(),
|
||||
ModTime: timestamppb.New(hdr.ModTime),
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
if !isDir {
|
||||
totalBytes += hdr.Size
|
||||
}
|
||||
}
|
||||
d.log.Info("backup list entries: ok", "correlation_id", corrID, "count", len(entries), "total_bytes", totalBytes)
|
||||
d.sendBackupListEntriesResult(corrID, entries, totalBytes, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) handleBackupReadFile(corrID string, req *panelv1.BackupReadFileRequest) {
|
||||
if req.BackupPath == "" {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "backup_path required")
|
||||
return
|
||||
}
|
||||
if req.EntryPath == "" {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "entry_path required")
|
||||
return
|
||||
}
|
||||
maxRead := req.MaxBytes
|
||||
if maxRead <= 0 {
|
||||
maxRead = defaultBackupReadCap
|
||||
}
|
||||
if maxRead > maxBackupReadCap {
|
||||
maxRead = maxBackupReadCap
|
||||
}
|
||||
|
||||
f, err := os.Open(req.BackupPath)
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "open backup: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "gzip: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(gz)
|
||||
|
||||
// Match by exact path; tarballs sometimes prefix entries with "./" so
|
||||
// also accept that variant.
|
||||
want := strings.TrimPrefix(req.EntryPath, "/")
|
||||
wantAlt := "./" + want
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, fmt.Sprintf("entry %q not found in backup", req.EntryPath))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "tar header: "+err.Error())
|
||||
return
|
||||
}
|
||||
if hdr.Name != want && hdr.Name != wantAlt {
|
||||
continue
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeDir || strings.HasSuffix(hdr.Name, "/") {
|
||||
d.sendBackupReadFileResult(corrID, nil, hdr.Size, false, false, "entry is a directory")
|
||||
return
|
||||
}
|
||||
// Read up to maxRead+1 to detect truncation cleanly.
|
||||
buf := &bytes.Buffer{}
|
||||
_, err = io.Copy(buf, io.LimitReader(tr, maxRead+1))
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, hdr.Size, false, false, "read entry: "+err.Error())
|
||||
return
|
||||
}
|
||||
truncated := false
|
||||
content := buf.Bytes()
|
||||
if int64(len(content)) > maxRead {
|
||||
content = content[:maxRead]
|
||||
truncated = true
|
||||
}
|
||||
isBinary := looksBinary(content)
|
||||
d.sendBackupReadFileResult(corrID, content, hdr.Size, truncated, isBinary, "")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// looksBinary returns true if the first binarySniffLen bytes contain a
|
||||
// NUL byte. Standard heuristic — text files don't contain NULs, binaries
|
||||
// almost always do (executables, images, compressed save formats).
|
||||
func looksBinary(b []byte) bool {
|
||||
limit := binarySniffLen
|
||||
if len(b) < limit {
|
||||
limit = len(b)
|
||||
}
|
||||
for i := 0; i < limit; i++ {
|
||||
if b[i] == 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendBackupListEntriesResult(corrID string, entries []*panelv1.BackupTarEntry, totalBytes int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_BackupListEntriesResult{
|
||||
BackupListEntriesResult: &panelv1.BackupListEntriesResult{
|
||||
Entries: entries,
|
||||
TotalBytes: totalBytes,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendBackupReadFileResult(corrID string, content []byte, size int64, truncated, isBinary bool, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_BackupReadFileResult{
|
||||
BackupReadFileResult: &panelv1.BackupReadFileResult{
|
||||
Content: content,
|
||||
Size: size,
|
||||
Truncated: truncated,
|
||||
IsBinary: isBinary,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
"github.com/dbledeez/panel/agent/internal/updater"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// installBaseMods runs a SteamCMD validate (if the module has a steamcmd
|
||||
// update provider) followed by an alpine sidecar that downloads + installs
|
||||
// the third-party 7DTD base-mod set:
|
||||
//
|
||||
// - Allocs Server Fixes — tarball from illy.bz (canonical upstream).
|
||||
// Contains Allocs_CommandExtensions + Allocs_WebAndMapRendering +
|
||||
// Alloc_XmlPatch, all under a Mods/ prefix when extracted to /game.
|
||||
// - PrismaCore — latest GitHub release .zip from Prisma501/PrismaCore.
|
||||
//
|
||||
// SteamCMD validate step (new):
|
||||
// - If the module declares at least one update provider with kind="steamcmd",
|
||||
// the first such provider is used to run `app_update <app_id> validate`
|
||||
// with anonymous login BEFORE the alpine sidecar runs. This re-downloads
|
||||
// or restores stock TFP mods (TFP_*, 0_TFP_*, xMarkers) that ship with
|
||||
// the dedicated server install.
|
||||
// - If the SteamCMD validate fails, the function returns an error and does
|
||||
// NOT proceed to the alpine step.
|
||||
// - If there is no steamcmd provider (e.g., Conan Exiles), the validate is
|
||||
// skipped and the function goes straight to the alpine path.
|
||||
//
|
||||
// What the alpine step does NOT touch (and does not need to):
|
||||
// - TFP_* / 0_TFP_* — shipped by SteamCMD as part of the dedicated
|
||||
// server install. TFP_Harmony is the runtime patcher; wiping it
|
||||
// would brick the engine. Always present in /game/Mods/ after a
|
||||
// fresh `app_update 294420`.
|
||||
// - xMarkers — also ships with the SteamCMD 7DTD install per the
|
||||
// operator (2026-05-04). Already in /game/Mods/ after install.
|
||||
// - Custom mods (RefugeBot, AGF-VP-*, AsylumRoboticInbox, anything not
|
||||
// in the recognized base-mod name set). Those are operator territory.
|
||||
//
|
||||
// Behavior on existing installs: any directory with a name that matches
|
||||
// the incoming mod gets renamed to `<name>.bak-<UTCstamp>` first so a
|
||||
// re-run is reversible. Direct port of the safety pattern in
|
||||
// refugebotserver's DependencyInstaller.cs.
|
||||
//
|
||||
// Caller responsibility: the instance container MUST be stopped before
|
||||
// calling this — the game holds DLLs in /game/Mods open while running,
|
||||
// and rename-on-extract will fail or corrupt under that.
|
||||
func (d *Dispatcher) installBaseMods(ctx context.Context, instanceID string, manifest *modulepkg.Manifest, dataPath string) error {
|
||||
if manifest == nil || len(manifest.UpdateProviders) == 0 {
|
||||
return errors.New("install-base-mods: module has no update provider")
|
||||
}
|
||||
installPath := manifest.UpdateProviders[0].InstallPath
|
||||
if installPath == "" {
|
||||
installPath = manifest.Runtime.Docker.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return errors.New("install-base-mods: install_path missing on module")
|
||||
}
|
||||
volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
|
||||
gameVol := volumeNameForPath(volumes, installPath)
|
||||
if gameVol == "" {
|
||||
return fmt.Errorf("install-base-mods: install_path %q not in module volumes", installPath)
|
||||
}
|
||||
|
||||
// ── SteamCMD validate step ──────────────────────────────────────────
|
||||
// If the module has a steamcmd update provider, run app_update validate
|
||||
// with anonymous login FIRST to re-download/restore stock TFP mods.
|
||||
// This ensures TFP_*, 0_TFP_*, and xMarkers are present before the
|
||||
// alpine sidecar installs third-party base mods.
|
||||
var steamcmdSpec *modulepkg.UpdateProvider
|
||||
for i := range manifest.UpdateProviders {
|
||||
if manifest.UpdateProviders[i].Kind == "steamcmd" {
|
||||
steamcmdSpec = &manifest.UpdateProviders[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if steamcmdSpec != nil {
|
||||
d.log.Info("install-base-mods: steamcmd validate starting", "instance_id", instanceID, "app_id", steamcmdSpec.AppID)
|
||||
prov, err := updater.Get("steamcmd")
|
||||
if err != nil {
|
||||
return fmt.Errorf("install-base-mods: get steamcmd provider: %w", err)
|
||||
}
|
||||
// Build a log sink that streams through the same "install-base-mods log" logger.
|
||||
sink := func(line string) {
|
||||
d.log.Info("install-base-mods log", "instance_id", instanceID, "line", line)
|
||||
}
|
||||
uc := &updater.Context{
|
||||
InstanceID: instanceID,
|
||||
Manifest: manifest,
|
||||
BrowseableRoot: installPath,
|
||||
DataPath: dataPath,
|
||||
Runtime: d.runtime,
|
||||
Log: sink,
|
||||
SteamUsername: "", // anonymous login
|
||||
SteamPassword: "", // anonymous login
|
||||
}
|
||||
if err := prov.Update(ctx, uc, steamcmdSpec); err != nil {
|
||||
return fmt.Errorf("basemods: steamcmd validate failed: %w", err)
|
||||
}
|
||||
d.log.Info("install-base-mods: steamcmd validate succeeded", "instance_id", instanceID)
|
||||
}
|
||||
|
||||
// alpine:3.19 is a 7 MB image with apk available. apk add fetches
|
||||
// curl + unzip + tar + jq in ~5s on a warm cache. Cheaper than
|
||||
// shipping our own base image and good enough for occasional runs.
|
||||
script := `set -e
|
||||
apk add --no-cache --quiet curl tar unzip jq ca-certificates >/dev/null
|
||||
STAMP=$(date -u +%Y%m%d%H%M%S)
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
backup_if_exists() {
|
||||
for d in "$@"; do
|
||||
if [ -d "$d" ] && [ ! -L "$d" ]; then
|
||||
mv "$d" "$d.bak-$STAMP" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ── Allocs Server Fixes ───────────────────────────────────────────────
|
||||
echo "[basemods] downloading Allocs from illy.bz"
|
||||
curl -fsSL -o "$TMP/allocs.tgz" "https://illy.bz/fi/7dtd/server_fixes.tar.gz"
|
||||
# Magic-byte check: gzip starts with 0x1f 0x8b — protects against
|
||||
# CDN-served HTML 404 pages landing as a "valid" archive.
|
||||
head -c 2 "$TMP/allocs.tgz" | od -An -tx1 | tr -d ' \n' | grep -q '^1f8b' \
|
||||
|| { echo "[basemods] Allocs download wasn't gzip — aborting"; exit 1; }
|
||||
backup_if_exists /game/Mods/Allocs_CommandExtensions /game/Mods/Allocs_WebAndMapRendering /game/Mods/Alloc_XmlPatch
|
||||
tar -xzf "$TMP/allocs.tgz" -C /game
|
||||
echo "[basemods] Allocs installed"
|
||||
|
||||
# ── PrismaCore (latest GitHub release zip) ─────────────────────────────
|
||||
echo "[basemods] resolving latest PrismaCore release"
|
||||
PRISMA_JSON="$TMP/prisma.json"
|
||||
curl -fsSL -A "panel-basemods/1.0" -H "Accept: application/vnd.github+json" \
|
||||
-o "$PRISMA_JSON" "https://api.github.com/repos/Prisma501/PrismaCore/releases/latest"
|
||||
PRISMA_TAG=$(jq -r '.tag_name // empty' "$PRISMA_JSON")
|
||||
PRISMA_URL=$(jq -r '.assets[]? | select(.name | test("\\.zip$"; "i")) | .browser_download_url' "$PRISMA_JSON" | head -1)
|
||||
if [ -z "$PRISMA_URL" ]; then
|
||||
echo "[basemods] no PrismaCore .zip asset — skipping"
|
||||
else
|
||||
echo "[basemods] downloading PrismaCore $PRISMA_TAG from $PRISMA_URL"
|
||||
curl -fsSL -A "panel-basemods/1.0" -o "$TMP/prisma.zip" "$PRISMA_URL"
|
||||
head -c 2 "$TMP/prisma.zip" | od -An -tx1 | tr -d ' \n' | grep -q '^504b' \
|
||||
|| { echo "[basemods] PrismaCore download wasn't a zip — aborting"; exit 1; }
|
||||
for d in /game/Mods/Prisma*; do backup_if_exists "$d"; done
|
||||
unzip -o -q "$TMP/prisma.zip" -d /game/Mods/
|
||||
echo "[basemods] PrismaCore installed"
|
||||
fi
|
||||
|
||||
# Note: TFP_* and xMarkers are shipped as part of the SteamCMD dedicated
|
||||
# server install (app 294420). They're already in /game/Mods/ after a
|
||||
# fresh install and we don't touch them here — wiping TFP_Harmony would
|
||||
# brick the engine.
|
||||
echo BASEMODS_OK
|
||||
`
|
||||
|
||||
sidecarID := fmt.Sprintf("%s-basemods", instanceID)
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: "alpine:3.19",
|
||||
Command: []string{"sh", "-c", script},
|
||||
Volumes: []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: gameVol, ContainerPath: installPath},
|
||||
},
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
d.log.Info("install-base-mods: starting sidecar", "instance_id", instanceID, "vol", gameVol)
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("install-base-mods: create: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return fmt.Errorf("install-base-mods: start: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.log.Info("install-base-mods log", "instance_id", instanceID, "line", line)
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return fmt.Errorf("install-base-mods: wait: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return fmt.Errorf("install-base-mods: sidecar exited %d", exitCode)
|
||||
}
|
||||
d.log.Info("install-base-mods: success", "instance_id", instanceID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Cluster player-save snapshots.
|
||||
//
|
||||
// A 7DTD cluster shares ONE Player/ folder across every member server (each
|
||||
// member's active-world Player dir is a symlink to the shared /cluster/Player
|
||||
// bind). That shared dir holds every character's .ttp (inventory, skills,
|
||||
// position, quests) — the single most irreplaceable per-player state. But it
|
||||
// is NOT captured by the panel tar backup: the backup tars /game-saves, where
|
||||
// the active Player dir is a symlink pointing OUT to /cluster, so tar stores
|
||||
// the dangling symlink, not the contents. Region Medic only heals world
|
||||
// regions, not player files. Net result before this: one corrupt write to a
|
||||
// .ttp and that character was gone with no restore path.
|
||||
//
|
||||
// Fix (mirrors the region-medic rolling-snapshot pattern): on every 7DTD
|
||||
// start, if the instance is clustered, copy the shared Player dir to a fresh
|
||||
// rolling slot keyed by cluster id and prune to keep N. All members write to
|
||||
// the same per-cluster rolling history, so a member's 4×/day restart cadence
|
||||
// gives a dense set of restore points. Best-effort: never blocks the start.
|
||||
|
||||
func clusterPlayerRollingRoot() string {
|
||||
return medicEnvOr("PANEL_CLUSTER_PLAYER_ROLLING_DIR", "/home/refuge/cluster-player-rolling")
|
||||
}
|
||||
|
||||
func clusterPlayerKeep() int {
|
||||
if v := os.Getenv("PANEL_CLUSTER_PLAYER_KEEP"); v != "" {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 12
|
||||
}
|
||||
|
||||
// snapshotClusterPlayerOnStart resolves the instance's /cluster bind to its
|
||||
// host path, and if it's a real shared bind holding player data, snapshots
|
||||
// <bind>/Player into a rolling slot keyed by the cluster id (= the bind dir's
|
||||
// basename, e.g. cl_37e3a7f72cdc). No-op for standalone servers (whose
|
||||
// /cluster is an empty per-instance named volume, not a bind).
|
||||
func (d *Dispatcher) snapshotClusterPlayerOnStart(ctx context.Context, rec *instanceRecord) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
log := d.log.With("instance_id", rec.InstanceID, "cluster_player_snapshot", true)
|
||||
|
||||
mounts, err := d.runtime.ContainerMounts(ctx, "panel-"+rec.InstanceID)
|
||||
if err != nil {
|
||||
log.Warn("cluster-player snapshot: inspect mounts failed (skipping)", "err", err)
|
||||
return
|
||||
}
|
||||
var bind string
|
||||
for _, m := range mounts {
|
||||
if m.Destination == "/cluster" && m.Type == "bind" && m.Source != "" {
|
||||
bind = m.Source
|
||||
break
|
||||
}
|
||||
}
|
||||
if bind == "" {
|
||||
return // standalone server (named-volume /cluster) — nothing shared to snapshot
|
||||
}
|
||||
playerDir := filepath.Join(bind, "Player")
|
||||
if st, err := os.Stat(playerDir); err != nil || !st.IsDir() {
|
||||
return // cluster joined but no Player dir yet (first boot)
|
||||
}
|
||||
if empty, _ := dirIsEmpty(playerDir); empty {
|
||||
return
|
||||
}
|
||||
|
||||
clusterID := filepath.Base(bind)
|
||||
destRoot := filepath.Join(clusterPlayerRollingRoot(), clusterID)
|
||||
if err := os.MkdirAll(destRoot, 0o755); err != nil {
|
||||
log.Warn("cluster-player snapshot: mkdir rolling root failed", "err", err)
|
||||
return
|
||||
}
|
||||
slot := filepath.Join(destRoot, "slot-"+strconv.FormatInt(time.Now().Unix(), 10))
|
||||
dst := filepath.Join(slot, "Player")
|
||||
if err := os.MkdirAll(slot, 0o755); err != nil {
|
||||
log.Warn("cluster-player snapshot: mkdir slot failed", "err", err)
|
||||
return
|
||||
}
|
||||
// cp -a preserves perms/acls/timestamps and the dir tree (player subdirs
|
||||
// of .7rm map files). figaro is Linux; cp is always present. Best-effort.
|
||||
cmd := exec.CommandContext(ctx, "cp", "-a", playerDir+"/.", dst)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
log.Warn("cluster-player snapshot: copy failed", "err", err, "out", strings.TrimSpace(string(out)))
|
||||
_ = os.RemoveAll(slot) // don't leave a half-copied slot that prune would keep
|
||||
return
|
||||
}
|
||||
keep := clusterPlayerKeep()
|
||||
pruned := pruneRollingSlots(destRoot, keep)
|
||||
log.Info("cluster-player snapshot taken", "cluster", clusterID, "slot", filepath.Base(slot), "keep", keep, "pruned", pruned)
|
||||
}
|
||||
|
||||
// dirIsEmpty reports whether dir has no entries.
|
||||
func dirIsEmpty(dir string) (bool, error) {
|
||||
f, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer f.Close()
|
||||
names, err := f.Readdirnames(1)
|
||||
if err != nil {
|
||||
// io.EOF → empty.
|
||||
return len(names) == 0, nil
|
||||
}
|
||||
return len(names) == 0, nil
|
||||
}
|
||||
|
||||
// pruneRollingSlots keeps the newest `keep` slot-* dirs under root (by name,
|
||||
// which embeds a unix timestamp so lexical == chronological) and removes the
|
||||
// rest. Returns how many it deleted.
|
||||
func pruneRollingSlots(root string, keep int) int {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var slots []string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && strings.HasPrefix(e.Name(), "slot-") {
|
||||
slots = append(slots, e.Name())
|
||||
}
|
||||
}
|
||||
if len(slots) <= keep {
|
||||
return 0
|
||||
}
|
||||
sort.Strings(slots) // oldest first
|
||||
removed := 0
|
||||
for _, name := range slots[:len(slots)-keep] {
|
||||
if err := os.RemoveAll(filepath.Join(root, name)); err == nil {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package dispatch
|
||||
|
||||
// DayZ mod install/uninstall + FsSymlink handlers. The controller drives
|
||||
// the high-level workflow (SteamCMD workshop_download runs controller-side,
|
||||
// writing to the shared panel-dayz-workshop volume); the agent's job is to
|
||||
// wire the downloaded content into the instance's /game tree:
|
||||
//
|
||||
// 1. Create /game/<folder_name> as a symlink to
|
||||
// /game/steamapps/workshop/content/221100/<workshop_id>
|
||||
// 2. Copy every .bikey from that mod's keys/ (or Keys/) subfolder into
|
||||
// /game/keys/ so BattlEye signs mod content on boot.
|
||||
// 3. On uninstall, remove the symlink + every bikey whose content hash
|
||||
// matches a bikey in the workshop mod's keys dir (so we don't remove
|
||||
// keys that belong to OTHER installed mods of the same vendor).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// ---- FsSymlink ----
|
||||
//
|
||||
// The symlink is created inside the instance's container namespace so that
|
||||
// any `/game/@Mod` symlink resolves correctly from the DayZServer binary's
|
||||
// perspective.
|
||||
|
||||
func (d *Dispatcher) handleFsSymlink(corrID string, req *panelv1.FsSymlinkRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsSymlinkResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsSymlinkResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
// `ln -sfn` replaces an existing symlink atomically. We require
|
||||
// abs paths for both target and link_path and refuse anything outside
|
||||
// the configured browseable roots for link_path (the symlink itself;
|
||||
// target can point outside for workshop shares).
|
||||
if !strings.HasPrefix(req.LinkPath, "/") || !strings.HasPrefix(req.Target, "/") {
|
||||
d.sendFsSymlinkResult(corrID, "target and link_path must be absolute")
|
||||
return
|
||||
}
|
||||
if _, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.LinkPath); err != nil {
|
||||
d.sendFsSymlinkResult(corrID, "link_path outside browseable roots: "+err.Error())
|
||||
return
|
||||
}
|
||||
// `ln` errors with "No such file or directory" if the parent doesn't
|
||||
// exist yet — common on first mod install for modules whose entrypoint
|
||||
// hasn't lazily created the Mods/ folder yet (Conan Exiles). `mkdir -p`
|
||||
// the parent before linking so the symlink-then-game-restart flow works
|
||||
// even on a fresh recreate. Idempotent: mkdir on existing dir is a
|
||||
// no-op, and the safeJoinAny check above already proved link_path is
|
||||
// inside a declared root so we're not creating dirs in arbitrary spots.
|
||||
parentDir := path.Dir(req.LinkPath)
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("mkdir -p %q && ln -sfn %q %q", parentDir, req.Target, req.LinkPath)})
|
||||
if err != nil {
|
||||
d.sendFsSymlinkResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendFsSymlinkResult(corrID, fmt.Sprintf("ln exited %d: %s", code, strings.TrimSpace(string(stderr))))
|
||||
return
|
||||
}
|
||||
d.sendFsSymlinkResult(corrID, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsSymlinkResult(corrID, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsSymlinkResult{
|
||||
FsSymlinkResult: &panelv1.FsSymlinkResult{Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- DayzModInstall ----
|
||||
|
||||
func (d *Dispatcher) handleDayzModInstall(corrID string, req *panelv1.DayzModInstallRequest) {
|
||||
d.log.Info("dayz mod install", "instance_id", req.InstanceId, "workshop_id", req.WorkshopId, "folder", req.FolderName)
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, err.Error(), nil, "")
|
||||
return
|
||||
}
|
||||
if req.WorkshopId == "" || req.FolderName == "" {
|
||||
d.sendDayzModInstallResult(corrID, "workshop_id and folder_name are required", nil, "")
|
||||
return
|
||||
}
|
||||
// Normalize folder name: must start with '@', no slashes, no whitespace.
|
||||
folder := strings.TrimSpace(req.FolderName)
|
||||
if !strings.HasPrefix(folder, "@") {
|
||||
folder = "@" + folder
|
||||
}
|
||||
if strings.ContainsAny(folder, "/\\ \t\n") {
|
||||
d.sendDayzModInstallResult(corrID, "folder_name must not contain slashes or whitespace", nil, "")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, err.Error(), nil, "")
|
||||
return
|
||||
}
|
||||
|
||||
modPath := "/game/steamapps/workshop/content/221100/" + req.WorkshopId
|
||||
linkPath := "/game/" + folder
|
||||
|
||||
// 1. Verify workshop content exists.
|
||||
_, _, code, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("test -d %q", modPath)})
|
||||
if code != 0 {
|
||||
d.sendDayzModInstallResult(corrID, "workshop content not found at "+modPath+" — did SteamCMD download succeed?", nil, modPath)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Create symlink.
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("ln -sfn %q %q", modPath, linkPath)})
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, "symlink: "+err.Error(), nil, modPath)
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendDayzModInstallResult(corrID, "symlink failed: "+strings.TrimSpace(string(stderr)), nil, modPath)
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Copy bikeys. DayZ mods ship them under keys/ OR Keys/ inside the
|
||||
// workshop tree. Copy every .bikey we find there into /game/keys/.
|
||||
// Keep a manifest of copied filenames so uninstall can remove them.
|
||||
// Use `find` + `cp` for portability; busybox and GNU both understand.
|
||||
stdout, stderr2, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf(`mkdir -p /game/keys && \
|
||||
for src in %q/keys %q/Keys; do \
|
||||
[ -d "$src" ] || continue; \
|
||||
for f in "$src"/*.bikey "$src"/*.BIKEY; do \
|
||||
[ -f "$f" ] || continue; \
|
||||
base=$(basename "$f"); \
|
||||
cp -f "$f" /game/keys/"$base"; \
|
||||
printf '%%s\n' "$base"; \
|
||||
done; \
|
||||
done`, modPath, modPath)})
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, "copy bikeys: "+err.Error(), nil, modPath)
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendDayzModInstallResult(corrID, "copy bikeys failed: "+strings.TrimSpace(string(stderr2)), nil, modPath)
|
||||
return
|
||||
}
|
||||
var bikeys []string
|
||||
for _, ln := range strings.Split(string(stdout), "\n") {
|
||||
ln = strings.TrimSpace(strings.TrimPrefix(ln, `\n`))
|
||||
if ln == "" {
|
||||
continue
|
||||
}
|
||||
bikeys = append(bikeys, ln)
|
||||
}
|
||||
d.sendDayzModInstallResult(corrID, "", bikeys, modPath)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendDayzModInstallResult(corrID, errMsg string, bikeys []string, resolvedPath string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_DayzModInstallResult{
|
||||
DayzModInstallResult: &panelv1.DayzModInstallResult{
|
||||
Error: errMsg,
|
||||
BikeysCopied: bikeys,
|
||||
ResolvedModPath: resolvedPath,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- DayzModUninstall ----
|
||||
|
||||
func (d *Dispatcher) handleDayzModUninstall(corrID string, req *panelv1.DayzModUninstallRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendDayzModUninstallResult(corrID, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
folder := strings.TrimSpace(req.FolderName)
|
||||
if !strings.HasPrefix(folder, "@") || strings.ContainsAny(folder, "/\\ \t\n") {
|
||||
d.sendDayzModUninstallResult(corrID, "invalid folder_name", nil)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendDayzModUninstallResult(corrID, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
linkPath := "/game/" + folder
|
||||
|
||||
// Resolve the symlink to find the workshop directory.
|
||||
stdout, _, code, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("readlink -f %q 2>/dev/null || true", linkPath)})
|
||||
modPath := strings.TrimSpace(string(stdout))
|
||||
if code != 0 || modPath == "" {
|
||||
// Symlink is already gone or broken — just drop stale /game/keys entries.
|
||||
modPath = ""
|
||||
}
|
||||
|
||||
// Remove bikeys whose *name* matches a bikey in the workshop mod's keys.
|
||||
// Matching by name (not hash) is what DayZ tooling does — the server
|
||||
// looks up bikeys by exact basename, and mods typically ship unique
|
||||
// key names.
|
||||
var removed []string
|
||||
if modPath != "" {
|
||||
out, _, _, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf(`for d in %q/keys %q/Keys; do \
|
||||
[ -d "$d" ] || continue; \
|
||||
for f in "$d"/*.bikey "$d"/*.BIKEY; do \
|
||||
[ -f "$f" ] || continue; \
|
||||
base=$(basename "$f"); \
|
||||
if [ -f "/game/keys/$base" ]; then \
|
||||
rm -f "/game/keys/$base"; \
|
||||
echo "$base"; \
|
||||
fi; \
|
||||
done; \
|
||||
done`, modPath, modPath)})
|
||||
for _, ln := range strings.Split(string(out), "\n") {
|
||||
ln = strings.TrimSpace(ln)
|
||||
if ln == "" {
|
||||
continue
|
||||
}
|
||||
removed = append(removed, ln)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the symlink itself.
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("rm -f %q", linkPath)})
|
||||
if err != nil {
|
||||
d.sendDayzModUninstallResult(corrID, "rm symlink: "+err.Error(), removed)
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendDayzModUninstallResult(corrID, "rm symlink failed: "+strings.TrimSpace(string(stderr)), removed)
|
||||
return
|
||||
}
|
||||
d.sendDayzModUninstallResult(corrID, "", removed)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendDayzModUninstallResult(corrID, errMsg string, bikeys []string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_DayzModUninstallResult{
|
||||
DayzModUninstallResult: &panelv1.DayzModUninstallResult{
|
||||
Error: errMsg,
|
||||
BikeysRemoved: bikeys,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Auto-provision a freshly-generated clustered 7DTD world (zero-touch create),
|
||||
// AND keep a visible "setting up" status until it is actually done so an operator
|
||||
// who refreshes the dashboard sees it is not ready yet (don't restart it).
|
||||
//
|
||||
// A new cluster world bakes its decorations (decoration.7dt / multiblocks.7dt)
|
||||
// with the engine's NATIVE block ids at generation. The entrypoint stamps the
|
||||
// cluster CANONICAL .nim over the world's table every boot so shared inventory
|
||||
// (.ttp) decodes uniformly — but native-baked decorations then resolve to a null
|
||||
// Block under canonical -> DecoManager NRE -> the world loads but never reaches
|
||||
// GameStartDone ("online but unjoinable"). The entrypoint fixes this by remapping
|
||||
// the decorations native->canonical on the FIRST boot it sees them in the native
|
||||
// band — which can only be a boot AFTER the gen boot (the running server writes
|
||||
// decoration.7dt; the entrypoint can't see it until next start).
|
||||
//
|
||||
// This watcher (a) closes that 2-phase gap so create is zero-touch — it bounces
|
||||
// the container ONCE so the entrypoint's remap runs — and (b) holds a persistent
|
||||
// "Finishing cluster setup…" status detail (the dashboard renders it as a "setting
|
||||
// up" pill instead of a green "running" one) until the world carries the
|
||||
// .canon-provisioned marker, so a page refresh always shows it isn't done yet.
|
||||
//
|
||||
// Strict no-op for non-cluster servers (no /cluster bind -> the watcher exits) and
|
||||
// already-canon worlds (deco in canon band -> "CANON" on the first probe -> the
|
||||
// indicator clears immediately). Single-flight; the entrypoint is the backstop if
|
||||
// the watcher ever misses (the world still provisions on its next restart).
|
||||
// Mirrors the hang-guard single-flight container-restart pattern (hangguard.go).
|
||||
|
||||
const (
|
||||
provisionPollInterval = 20 * time.Second
|
||||
provisionPollTimeout = 15 * time.Minute
|
||||
provisionExecTimeout = 20 * time.Second
|
||||
provisionRestartGrace = 30 * time.Second
|
||||
)
|
||||
|
||||
// provisionDetail is the persistent status detail shown while a clustered world is
|
||||
// still aligning to the cluster canonical. The dashboard matches this text to paint
|
||||
// a "setting up" pill, so a refresh shows the server isn't ready yet.
|
||||
const provisionDetail = "Finishing cluster setup — aligning decorations to the cluster (it auto-restarts once); please wait, don't restart it."
|
||||
|
||||
// provisionProbeScript runs inside the container and classifies the active cluster
|
||||
// world's provisioning state. Emits exactly one token on stdout:
|
||||
//
|
||||
// NOWORLD no recorded save dir (RWG world not yet picked in the Cluster tab)
|
||||
// PROVISIONED .canon-provisioned marker present (already aligned)
|
||||
// GENERATING no main.ttw yet (world still generating)
|
||||
// CANON decorations already in the canon band (>=24000) -> nothing to do
|
||||
// ATTEMPTED already bounced once and still not aligned -> stop (avoid a loop)
|
||||
// NEEDS_ALIGN gen done; decorations native-banded OR not baked yet -> bounce once
|
||||
//
|
||||
// Gated on main.ttw (= gen finished) not on decoration.7dt existing, because some
|
||||
// worlds only write decoration.7dt on the first save — the bounce's graceful save
|
||||
// writes it (native), then the entrypoint remaps it on the align boot. First deco
|
||||
// block-id = u16 @ offset 17 (hdr 5 + record id-offset 12), masked to 15 bits.
|
||||
const provisionProbeScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
|
||||
`[ -z "$SB" ] && { echo NOWORLD; exit 0; }; ` +
|
||||
`[ -f "$SB/.canon-provisioned" ] && { echo PROVISIONED; exit 0; }; ` +
|
||||
`[ -f "$SB/main.ttw" ] || { echo GENERATING; exit 0; }; ` +
|
||||
`D="$SB/decoration.7dt"; ` +
|
||||
`if [ -f "$D" ]; then raw="$(od -An -tu2 -j17 -N2 "$D" 2>/dev/null | tr -d ' ')"; id=$(( ${raw:-0} & 0x7FFF )); [ "$id" -ge 24000 ] && { echo CANON; exit 0; }; fi; ` +
|
||||
`[ -f "$SB/.panel-provision-attempted" ] && { echo ATTEMPTED; exit 0; }; ` +
|
||||
`echo NEEDS_ALIGN`
|
||||
|
||||
// provisionAttemptScript drops a durable one-bounce guard so a world that doesn't
|
||||
// align after its single bounce is never bounced again (the entrypoint remains the
|
||||
// backstop on ordinary restarts).
|
||||
const provisionAttemptScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
|
||||
`[ -n "$SB" ] && touch "$SB/.panel-provision-attempted" 2>/dev/null; true`
|
||||
|
||||
// autoProvisionClusterWorldOnStart starts (single-flight) the provision watcher for
|
||||
// a just-started 7DTD instance. Returns immediately; the watch runs off a background
|
||||
// goroutine bounded by provisionPollTimeout.
|
||||
func (d *Dispatcher) autoProvisionClusterWorldOnStart(rec *instanceRecord) {
|
||||
if rec.ModuleID != "7dtd" {
|
||||
return
|
||||
}
|
||||
if !rec.provisionGuard.CompareAndSwap(false, true) {
|
||||
return // a watcher is already running for this instance
|
||||
}
|
||||
go func() {
|
||||
defer rec.provisionGuard.Store(false)
|
||||
log := d.log.With("instance_id", rec.InstanceID, "auto_provision", true)
|
||||
|
||||
// Only clustered worlds provision (and show the indicator). Resolve the
|
||||
// /cluster bind like clusterplayer.go; bail out for standalone servers.
|
||||
mctx, mcancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
mounts, _ := d.runtime.ContainerMounts(mctx, "panel-"+rec.InstanceID)
|
||||
mcancel()
|
||||
clustered := false
|
||||
for _, m := range mounts {
|
||||
if m.Destination == "/cluster" && m.Type == "bind" && m.Source != "" {
|
||||
clustered = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !clustered {
|
||||
return
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(provisionPollTimeout)
|
||||
delay := 5 * time.Second // first poll soon, to override a premature green "running"
|
||||
var lastDetail string
|
||||
bounced := false
|
||||
emit := func(detail string) {
|
||||
if detail == lastDetail {
|
||||
return
|
||||
}
|
||||
lastDetail = detail
|
||||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING, 0, detail)
|
||||
}
|
||||
for {
|
||||
time.Sleep(delay)
|
||||
delay = provisionPollInterval
|
||||
if time.Now().After(deadline) {
|
||||
return // gave up; the entrypoint aligns on the next ordinary restart
|
||||
}
|
||||
ectx, cancel := context.WithTimeout(context.Background(), provisionExecTimeout)
|
||||
stdout, _, code, err := d.runtime.ExecCapture(ectx, rec.ContainerID, []string{"sh", "-c", provisionProbeScript})
|
||||
cancel()
|
||||
if err != nil || code != 0 {
|
||||
continue // mid-boot / exec hiccup — leave the agent's own status alone
|
||||
}
|
||||
switch strings.TrimSpace(string(stdout)) {
|
||||
case "CANON", "PROVISIONED":
|
||||
emit("running") // fully set up — clear the "setting up" indicator
|
||||
return
|
||||
case "NEEDS_ALIGN":
|
||||
emit(provisionDetail)
|
||||
if !bounced {
|
||||
bounced = true
|
||||
actx, acancel := context.WithTimeout(context.Background(), provisionExecTimeout)
|
||||
_, _, _, _ = d.runtime.ExecCapture(actx, rec.ContainerID, []string{"sh", "-c", provisionAttemptScript})
|
||||
acancel()
|
||||
log.Info("auto-provision: aligning new cluster world to canonical (one-time bounce)")
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID, Stream: "stdout", At: timestamppb.Now(),
|
||||
Line: "[panel] auto-provision: aligning the new world's decorations to the cluster (one-time restart)…",
|
||||
}},
|
||||
})
|
||||
rctx, rcancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
if err := d.runtime.Restart(rctx, rec.ContainerID, provisionRestartGrace); err != nil {
|
||||
log.Error("auto-provision restart failed (entrypoint aligns on the next ordinary restart)", "err", err)
|
||||
}
|
||||
rcancel()
|
||||
}
|
||||
continue // keep watching across the bounce until the align boot marks it provisioned
|
||||
default: // GENERATING / NOWORLD / ATTEMPTED — still setting up
|
||||
emit(provisionDetail)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// handleDelete tears down the instance record. If the container is still
|
||||
// running, it's stopped with the requested grace first. Optionally
|
||||
// removes every Docker volume labeled for this instance ("purge").
|
||||
//
|
||||
// Never deletes the host data_path directory — operators may want to
|
||||
// keep configs/logs that landed on the bind mount. If they want a full
|
||||
// wipe, they can delete the dir manually after.
|
||||
func (d *Dispatcher) handleDelete(ctx context.Context, corrID string, req *panelv1.InstanceDelete) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||||
|
||||
d.mu.Lock()
|
||||
rec, exists := d.instances[req.InstanceId]
|
||||
d.mu.Unlock()
|
||||
|
||||
// Cancel any in-flight steamcmd / updater goroutine FIRST. The
|
||||
// updater spins up its own sidecar container (panel-<id>-steamcmd-<n>)
|
||||
// that mounts the same data volumes. If we don't cancel it before
|
||||
// trying to remove volumes, the sidecar holds them open and the
|
||||
// volume-rm fails with "volume is in use" — operator sees the
|
||||
// instance stuck on "Deleting…" while a download finishes in the
|
||||
// background.
|
||||
d.cancelUpdater(req.InstanceId)
|
||||
// Whatever state the main container is in, any Files-tab helper that
|
||||
// was running against this instance is now orphaned. Clean it up
|
||||
// before touching anything else so its volume-holds don't interfere
|
||||
// with the main container's stop/remove.
|
||||
d.teardownFsHelper(req.InstanceId)
|
||||
|
||||
grace := time.Duration(req.GraceSeconds) * time.Second
|
||||
if grace == 0 {
|
||||
grace = 15 * time.Second
|
||||
}
|
||||
|
||||
// If we have a record, use its container_id directly. Otherwise try
|
||||
// to find the container by its canonical name (may exist from a
|
||||
// previous session that skipped rehydrate).
|
||||
containerID := ""
|
||||
if exists {
|
||||
containerID = rec.ContainerID
|
||||
} else {
|
||||
if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil {
|
||||
containerID = state.ContainerID
|
||||
}
|
||||
}
|
||||
|
||||
if containerID != "" {
|
||||
// Best-effort stop; ignore errors since container may already be stopped.
|
||||
_ = d.runtime.Stop(ctx, containerID, grace)
|
||||
removeErr := d.runtime.Remove(ctx, containerID)
|
||||
// d.runtime.Remove already swallows not-found, so a non-nil
|
||||
// removeErr means the remove genuinely failed. Before we tear down
|
||||
// any bookkeeping (in-memory record + sidecar metadata) we must be
|
||||
// SURE the container is gone — otherwise we orphan a live container
|
||||
// the controller still routes to, and the agent forgets it on the
|
||||
// next rehydrate. The classic trigger: a delete racing with agent
|
||||
// shutdown, where the Docker connection dies mid-remove
|
||||
// ("terminated signal received"). The remove errors, and the
|
||||
// follow-up existence check ALSO errors — fate unknown, not gone.
|
||||
if removeErr != nil {
|
||||
exists, existErr := d.runtime.ContainerExists(ctx, "panel-"+req.InstanceId)
|
||||
switch {
|
||||
case existErr != nil:
|
||||
// Couldn't even ask (daemon down / connection dropped /
|
||||
// ctx cancelled). Do NOT delete the sidecar — bail and let
|
||||
// the operator retry once the daemon is reachable. Leaving
|
||||
// the sidecar means the next rehydrate re-adopts the
|
||||
// instance instead of orphaning it.
|
||||
log.Error("delete: container remove failed and existence is unknown — aborting to avoid orphan",
|
||||
"remove_err", removeErr, "exists_err", existErr)
|
||||
d.replyError(corrID, req.InstanceId, "container_remove",
|
||||
fmt.Sprintf("container panel-%s remove failed and existence unverifiable: %s", req.InstanceId, removeErr.Error()))
|
||||
return
|
||||
case exists:
|
||||
// Container is confirmed still present. The controller must
|
||||
// learn — otherwise the DB row gets purged while a zombie
|
||||
// container lives on, and the next create with the same id
|
||||
// fails on a "name already in use" docker error.
|
||||
log.Error("delete: container still exists after remove", "err", removeErr)
|
||||
d.replyError(corrID, req.InstanceId, "container_remove",
|
||||
fmt.Sprintf("container panel-%s still exists after remove: %s", req.InstanceId, removeErr.Error()))
|
||||
return
|
||||
default:
|
||||
// Confirmed gone (e.g. autoremove fired between Stop and
|
||||
// Remove). Safe to proceed with bookkeeping cleanup.
|
||||
log.Warn("delete: remove returned error but container is confirmed gone", "err", removeErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Signal any live trackers / log streams to stop.
|
||||
if exists {
|
||||
if rec.TrackerCancel != nil {
|
||||
rec.TrackerCancel()
|
||||
}
|
||||
if rec.LogCancel != nil {
|
||||
rec.LogCancel()
|
||||
}
|
||||
}
|
||||
|
||||
// Purge volumes if requested.
|
||||
if req.PurgeVolumes {
|
||||
dr, ok := d.runtime.(*runtime.DockerRuntime)
|
||||
if !ok {
|
||||
log.Warn("delete: purge requested but runtime is not Docker; skipping volume cleanup")
|
||||
} else {
|
||||
// Sweep every container that mounts one of this instance's
|
||||
// volumes — running OR exited. Without this, a stranded
|
||||
// steamcmd / backup / fs-helper sidecar from a previous
|
||||
// agent crash keeps the volume "in use" and volume-rm
|
||||
// fails repeatedly. Operator sees the instance stuck
|
||||
// "Deleting…" forever.
|
||||
holders, err := dr.ListContainersHoldingInstanceVolumes(ctx, req.InstanceId)
|
||||
if err != nil {
|
||||
log.Warn("delete: scan for orphan sidecars failed", "err", err)
|
||||
}
|
||||
for _, hid := range holders {
|
||||
if err := dr.ForceRemoveContainer(ctx, hid); err != nil {
|
||||
log.Warn("delete: force-remove sidecar failed", "container_id", hid, "err", err)
|
||||
} else {
|
||||
log.Info("delete: removed orphan sidecar", "container_id", short(hid))
|
||||
}
|
||||
}
|
||||
vols, err := dr.ListVolumesByInstance(ctx, req.InstanceId)
|
||||
if err != nil {
|
||||
log.Warn("delete: list volumes failed", "err", err)
|
||||
}
|
||||
for _, v := range vols {
|
||||
// Force=true: there's no scenario where we want a panel
|
||||
// volume to survive a purge=true delete after we've
|
||||
// scrubbed every container that referenced it.
|
||||
if err := dr.RemoveVolume(ctx, v, true); err != nil {
|
||||
log.Warn("delete: remove volume failed", "volume", v, "err", err)
|
||||
d.replyError(corrID, req.InstanceId, "volume_remove", fmt.Sprintf("volume %s: %s", v, err.Error()))
|
||||
return
|
||||
}
|
||||
log.Info("delete: purged volume", "volume", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove in-memory record + sidecar metadata.
|
||||
d.mu.Lock()
|
||||
delete(d.instances, req.InstanceId)
|
||||
d.mu.Unlock()
|
||||
d.deleteMeta(req.InstanceId)
|
||||
|
||||
// Leave host data_path alone (see doc comment); operators can rm -rf
|
||||
// manually. We do NOT attempt os.RemoveAll here — too destructive to
|
||||
// do automatically on a shared multi-instance directory layout.
|
||||
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "deleted")
|
||||
d.replyOK(corrID)
|
||||
log.Info("instance deleted", "purge_volumes", req.PurgeVolumes)
|
||||
_ = os.Stat // kept imported for future host-data_path removal flag
|
||||
}
|
||||
|
||||
// sendDelete-specific correlation reply uses the existing replyOK/err pattern.
|
||||
var _ = timestamppb.Now // keep import hint while ctx-based future expansions land
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,526 @@
|
||||
package dispatch
|
||||
|
||||
// Per-player POI discovery query against the empyrion server's
|
||||
// global.db SQLite store. Spawns an alpine+sqlite sidecar with the
|
||||
// instance's saves volume mounted read-only, runs a parameterized
|
||||
// join query, returns the rows.
|
||||
//
|
||||
// Schema reference (DiscoveredPOIs / Entities / Playfields / SolarSystems
|
||||
// / LoginLogoff) was reverse-engineered from a live empyrion install on
|
||||
// 2026-04-30. Eleon's mod API does NOT expose per-player discovery —
|
||||
// Request_GlobalStructure_List returns the server-wide aggregate that
|
||||
// EAH shows. The DB has the finer per-player data the in-game map uses.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// sqlite3's CLI doesn't support `?` parameter binding (that's a
|
||||
// prepared-statement-only feature on the C API). We string-interpolate
|
||||
// the SteamID64 — already validated as 17 digits server-side, but we
|
||||
// also belt-and-suspenders re-validate at the shell layer (digits only).
|
||||
//
|
||||
// Two query modes:
|
||||
// "personal" — POIs the player personally first-walked-into
|
||||
// "faction" — POIs anyone in the player's faction has discovered
|
||||
// (matches the in-game shared map)
|
||||
const sqlPersonalTpl = `
|
||||
SELECT
|
||||
COALESCE(ss.name, '') AS solar_system,
|
||||
COALESCE(pf.name, '') AS playfield,
|
||||
COALESCE(poi.name, '') AS poi_name,
|
||||
COALESCE(poi.etype, 0) AS poi_type,
|
||||
COALESCE(dp.gametime, 0) AS game_time,
|
||||
COALESCE(ll.playername, '') AS discoverer_name,
|
||||
COALESCE(ll.playerid, '') AS discoverer_steam_id,
|
||||
COALESCE(dp.facgroup, 0) AS facgroup,
|
||||
COALESCE(dp.facid, 0) AS facid
|
||||
FROM DiscoveredPOIs dp
|
||||
JOIN Entities poi ON dp.poiid = poi.entityid
|
||||
LEFT JOIN Playfields pf ON dp.pfid = pf.pfid
|
||||
LEFT JOIN SolarSystems ss ON pf.ssid = ss.ssid
|
||||
LEFT JOIN LoginLogoff ll ON dp.entityid = ll.entityid
|
||||
WHERE dp.entityid IN (
|
||||
SELECT DISTINCT entityid FROM LoginLogoff WHERE playerid='%SID%'
|
||||
)
|
||||
ORDER BY dp.gametime DESC
|
||||
LIMIT 5000;
|
||||
`
|
||||
|
||||
const sqlFactionTpl = `
|
||||
SELECT
|
||||
COALESCE(ss.name, '') AS solar_system,
|
||||
COALESCE(pf.name, '') AS playfield,
|
||||
COALESCE(poi.name, '') AS poi_name,
|
||||
COALESCE(poi.etype, 0) AS poi_type,
|
||||
COALESCE(dp.gametime, 0) AS game_time,
|
||||
COALESCE(ll.playername, '') AS discoverer_name,
|
||||
COALESCE(ll.playerid, '') AS discoverer_steam_id,
|
||||
COALESCE(dp.facgroup, 0) AS facgroup,
|
||||
COALESCE(dp.facid, 0) AS facid
|
||||
FROM DiscoveredPOIs dp
|
||||
JOIN Entities poi ON dp.poiid = poi.entityid
|
||||
LEFT JOIN Playfields pf ON dp.pfid = pf.pfid
|
||||
LEFT JOIN SolarSystems ss ON pf.ssid = ss.ssid
|
||||
LEFT JOIN LoginLogoff ll ON dp.entityid = ll.entityid
|
||||
WHERE (dp.facgroup, dp.facid) IN (
|
||||
SELECT DISTINCT pe.facgroup, pe.facid
|
||||
FROM Entities pe
|
||||
WHERE pe.entityid IN (
|
||||
SELECT DISTINCT entityid FROM LoginLogoff WHERE playerid='%SID%'
|
||||
)
|
||||
)
|
||||
ORDER BY dp.gametime DESC
|
||||
LIMIT 5000;
|
||||
`
|
||||
|
||||
const sqlPlayerNameTpl = `
|
||||
SELECT playername FROM LoginLogoff
|
||||
WHERE playerid='%SID%'
|
||||
ORDER BY loginticks DESC
|
||||
LIMIT 1;
|
||||
`
|
||||
|
||||
func (d *Dispatcher) handleEmpyrionDiscoveries(corrID string, req *panelv1.EmpyrionDiscoveriesRequest) {
|
||||
if req == nil || req.InstanceId == "" || req.SteamId == "" {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "instance_id and steam_id required",
|
||||
})
|
||||
return
|
||||
}
|
||||
go d.runEmpyrionDiscoveriesJob(corrID, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) runEmpyrionDiscoveriesJob(corrID string, req *panelv1.EmpyrionDiscoveriesRequest) {
|
||||
mode := strings.ToLower(strings.TrimSpace(req.Mode))
|
||||
if mode == "" || mode == "personal" {
|
||||
mode = "personal"
|
||||
} else if mode != "faction" {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "mode must be 'personal' or 'faction'", Mode: req.Mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
dockerBin, err := exec.LookPath("docker")
|
||||
if err != nil {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "docker binary not found on agent host", Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Saves volume name follows the panel convention: panel-<id>-saves.
|
||||
savesVol := "panel-" + req.InstanceId + "-saves"
|
||||
// Pre-flight: volume must exist (instance exists on this agent).
|
||||
if err := exec.Command(dockerBin, "volume", "inspect", savesVol).Run(); err != nil {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: fmt.Sprintf("instance saves volume %q not found on this agent", savesVol),
|
||||
Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Validate steam_id is digits-only — defense in depth, controller
|
||||
// already checks 17-digit format. Belt-and-suspenders against any
|
||||
// future caller that bypasses the controller's validation.
|
||||
for _, c := range req.SteamId {
|
||||
if c < '0' || c > '9' {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "steam_id must be digits only", Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
tpl := sqlPersonalTpl
|
||||
if mode == "faction" {
|
||||
tpl = sqlFactionTpl
|
||||
}
|
||||
q := strings.ReplaceAll(tpl, "%SID%", req.SteamId)
|
||||
nameQ := strings.ReplaceAll(sqlPlayerNameTpl, "%SID%", req.SteamId)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// alpine+sqlite sidecar. We don't need to discover the savegame
|
||||
// folder name dynamically — the entrypoint creates exactly one
|
||||
// (via dedicated.yaml.SaveDirectory which defaults to "DediGame";
|
||||
// the panel's empyrion module pins this). We pick `*/global.db`
|
||||
// at the shell layer so renames don't break the query.
|
||||
//
|
||||
// Note: -2 disables stdin which avoids the `command timed out`
|
||||
// path that empty stdin can hit on some Alpine builds. We feed the
|
||||
// SQL via the third positional arg.
|
||||
rowsScript := `set -e
|
||||
apk add --no-cache sqlite >/dev/null 2>&1
|
||||
DB=$(ls /saves/Saves/Games/*/global.db 2>/dev/null | head -1)
|
||||
if [ -z "$DB" ]; then echo '{"err":"no global.db found in saves volume"}' >&2; exit 2; fi
|
||||
NAME=$(sqlite3 "$DB" "$1" 2>/dev/null || echo "")
|
||||
ROWS=$(sqlite3 -json "$DB" "$2" 2>&1)
|
||||
if [ -z "$ROWS" ]; then ROWS='[]'; fi
|
||||
NAME_JSON=$(printf '%s' "$NAME" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g')
|
||||
printf '{"player_name":"%s","rows":%s}\n' "$NAME_JSON" "$ROWS"
|
||||
`
|
||||
|
||||
out, err := exec.CommandContext(ctx, dockerBin,
|
||||
"run", "--rm",
|
||||
"-v", savesVol+":/saves:ro",
|
||||
"--entrypoint", "sh",
|
||||
"alpine:3.20", "-c", rowsScript, "_", nameQ, q,
|
||||
).Output()
|
||||
if err != nil {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "sqlite query failed: " + err.Error(),
|
||||
Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Parse the wrapper json {"player_name": "...", "rows": [...]}.
|
||||
type wireRow struct {
|
||||
SolarSystem string `json:"solar_system"`
|
||||
Playfield string `json:"playfield"`
|
||||
PoiName string `json:"poi_name"`
|
||||
PoiType int32 `json:"poi_type"`
|
||||
GameTime int64 `json:"game_time"`
|
||||
DiscovererName string `json:"discoverer_name"`
|
||||
DiscovererSteamID string `json:"discoverer_steam_id"`
|
||||
Facgroup int32 `json:"facgroup"`
|
||||
Facid int32 `json:"facid"`
|
||||
}
|
||||
var wrap struct {
|
||||
PlayerName string `json:"player_name"`
|
||||
Rows []wireRow `json:"rows"`
|
||||
}
|
||||
// Strip leading whitespace; sqlite3 -json sometimes prepends a BOM-ish blank line.
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &wrap); err != nil {
|
||||
// Tolerate "no rows" — sqlite3 -json prints `[]` (good) or empty
|
||||
// string when no DB (bad — caught above) or `null` for some
|
||||
// builds. Fall back to empty-result on parse failure.
|
||||
var rowsOnly []wireRow
|
||||
if err2 := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &rowsOnly); err2 == nil {
|
||||
wrap.Rows = rowsOnly
|
||||
} else {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "parse sqlite output: " + err.Error(),
|
||||
Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
pois := make([]*panelv1.DiscoveredPOI, 0, len(wrap.Rows))
|
||||
for _, r := range wrap.Rows {
|
||||
pois = append(pois, &panelv1.DiscoveredPOI{
|
||||
SolarSystem: r.SolarSystem,
|
||||
Playfield: r.Playfield,
|
||||
PoiName: r.PoiName,
|
||||
PoiType: r.PoiType,
|
||||
GameTime: r.GameTime,
|
||||
DiscovererName: r.DiscovererName,
|
||||
DiscovererSteamId: r.DiscovererSteamID,
|
||||
FactionGroup: r.Facgroup,
|
||||
FactionId: r.Facid,
|
||||
})
|
||||
}
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: true,
|
||||
Pois: pois,
|
||||
Mode: mode,
|
||||
SteamId: req.SteamId,
|
||||
PlayerName: strings.Trim(wrap.PlayerName, "\""),
|
||||
})
|
||||
_ = errors.New
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendEmpyrionDiscoveriesResult(corrID string, res *panelv1.EmpyrionDiscoveriesResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_EmpyrionDiscoveriesResult{
|
||||
EmpyrionDiscoveriesResult: res,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Player Summary — stats + sessions + inventory in one query
|
||||
// ============================================================================
|
||||
|
||||
// playerSummaryScript is one big sh script run inside an alpine sidecar.
|
||||
// It opens global.db, resolves the player's entityid via LoginLogoff,
|
||||
// then runs four sqlite queries with -json output. The output is one
|
||||
// JSON object with named keys for each query — we marshal-decode it
|
||||
// directly into the proto result.
|
||||
//
|
||||
// The 4 queries:
|
||||
// 1. summary — Entities row + LoginLogoff name lookup + last position
|
||||
// 2. stats — PlayerStatistics row
|
||||
// 3. sessions — recent LoginLogoff rows
|
||||
// 4. inv — most recent PlayerInventory snapshot + items
|
||||
//
|
||||
// We accept that some of these may return [] (player has no stats row
|
||||
// yet, never logged in this savegame, etc.) — JSON parser handles it.
|
||||
func playerSummaryScript() string {
|
||||
return `set -e
|
||||
apk add --no-cache sqlite >/dev/null 2>&1
|
||||
DB=$(ls /saves/Saves/Games/*/global.db 2>/dev/null | head -1)
|
||||
if [ -z "$DB" ]; then echo '{"err":"no global.db found in saves volume"}' >&2; exit 2; fi
|
||||
SID="$1"
|
||||
LIMIT="${2:-50}"
|
||||
|
||||
# Resolve entityid for this Steam ID. Use the most recent LoginLogoff
|
||||
# row to handle character resets that re-bind the same Steam ID to a
|
||||
# new entityid.
|
||||
EID=$(sqlite3 "$DB" "SELECT entityid FROM LoginLogoff WHERE playerid='${SID}' ORDER BY loginticks DESC LIMIT 1;")
|
||||
if [ -z "$EID" ]; then
|
||||
printf '{"steam_id":"%s","entity_id":0,"player_name":"","stats":null,"sessions":[],"inventory":[]}\n' "$SID"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
NAME=$(sqlite3 "$DB" "SELECT playername FROM LoginLogoff WHERE entityid=${EID} ORDER BY loginticks DESC LIMIT 1;")
|
||||
FG=$(sqlite3 "$DB" "SELECT facgroup FROM Entities WHERE entityid=${EID};")
|
||||
FI=$(sqlite3 "$DB" "SELECT facid FROM Entities WHERE entityid=${EID};")
|
||||
|
||||
# Last position from PlayerData.
|
||||
PD=$(sqlite3 -json "$DB" "
|
||||
SELECT
|
||||
COALESCE(pf.name, '') AS last_playfield,
|
||||
COALESCE(pd.posx, 0) AS last_x,
|
||||
COALESCE(pd.posy, 0) AS last_y,
|
||||
COALESCE(pd.posz, 0) AS last_z,
|
||||
COALESCE(pd.credits, 0) AS credits
|
||||
FROM PlayerData pd
|
||||
LEFT JOIN Playfields pf ON pd.pfid = pf.pfid
|
||||
WHERE pd.entityid=${EID};
|
||||
")
|
||||
|
||||
STATS=$(sqlite3 -json "$DB" "
|
||||
SELECT
|
||||
COALESCE(killedenemies, 0) AS killed_enemies,
|
||||
COALESCE(killedallied, 0) AS killed_allied,
|
||||
COALESCE(killedanimals, 0) AS killed_animals,
|
||||
COALESCE(killeddrones, 0) AS killed_drones,
|
||||
COALESCE(killedplayers, 0) AS killed_players,
|
||||
COALESCE(killedalliedplayers, 0) AS killed_allied_players,
|
||||
COALESCE(died, 0) AS died,
|
||||
COALESCE(score, 0) AS score,
|
||||
COALESCE(blocksplaced, 0) AS blocks_placed,
|
||||
COALESCE(blocksdigged, 0) AS blocks_digged,
|
||||
COALESCE(walkedmeters, 0) AS walked_meters,
|
||||
COALESCE(jetpackmeters, 0) AS jetpack_meters,
|
||||
COALESCE(hvmeters, 0) AS hv_meters,
|
||||
COALESCE(svmeters, 0) AS sv_meters,
|
||||
COALESCE(cvmeters, 0) AS cv_meters,
|
||||
COALESCE(playtime, 0.0) AS playtime,
|
||||
COALESCE(travly, 0.0) AS light_years,
|
||||
COALESCE(travau, 0.0) AS astronomical_units
|
||||
FROM PlayerStatistics WHERE entityid=${EID};
|
||||
")
|
||||
|
||||
SESS=$(sqlite3 -json "$DB" "
|
||||
SELECT
|
||||
COALESCE(loginticks, 0) AS login_ticks,
|
||||
COALESCE(logoffticks, 0) AS logoff_ticks,
|
||||
COALESCE(playername, '') AS player_name,
|
||||
COALESCE(buildnr, 0) AS build_nr,
|
||||
COALESCE(ip, '') AS ip,
|
||||
COALESCE(os, '') AS os,
|
||||
COALESCE(gfx, '') AS gfx,
|
||||
COALESCE(cpu, '') AS cpu
|
||||
FROM LoginLogoff WHERE entityid=${EID}
|
||||
ORDER BY loginticks DESC LIMIT ${LIMIT};
|
||||
")
|
||||
|
||||
INV_PIID=$(sqlite3 "$DB" "SELECT piid FROM PlayerInventory WHERE entityid=${EID} ORDER BY gametime DESC LIMIT 1;")
|
||||
INV_GT=0
|
||||
INV='[]'
|
||||
if [ -n "$INV_PIID" ]; then
|
||||
INV_GT=$(sqlite3 "$DB" "SELECT gametime FROM PlayerInventory WHERE piid=${INV_PIID};")
|
||||
INV=$(sqlite3 -json "$DB" "
|
||||
SELECT item AS item_id, count
|
||||
FROM PlayerInventoryItems WHERE piid=${INV_PIID};
|
||||
")
|
||||
if [ -z "$INV" ]; then INV='[]'; fi
|
||||
fi
|
||||
|
||||
if [ -z "$STATS" ]; then STATS='[]'; fi
|
||||
if [ -z "$SESS" ]; then SESS='[]'; fi
|
||||
if [ -z "$PD" ]; then PD='[]'; fi
|
||||
|
||||
printf '{"steam_id":"%s","player_name":%s,"entity_id":%s,"faction_group":%s,"faction_id":%s,"position":%s,"stats":%s,"sessions":%s,"inventory_gametime":%s,"inventory":%s}\n' \
|
||||
"$SID" \
|
||||
"$(printf '%%s' "$NAME" | sqlite3 ':memory:' 'select json_quote(?1);' 2>/dev/null || echo '""')" \
|
||||
"${EID:-0}" "${FG:-0}" "${FI:-0}" \
|
||||
"$PD" "$STATS" "$SESS" "${INV_GT:-0}" "$INV"
|
||||
`
|
||||
}
|
||||
|
||||
func (d *Dispatcher) handleEmpyrionPlayerSummary(corrID string, req *panelv1.EmpyrionPlayerSummaryRequest) {
|
||||
if req == nil || req.InstanceId == "" || req.SteamId == "" {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "instance_id and steam_id required",
|
||||
})
|
||||
return
|
||||
}
|
||||
go d.runEmpyrionPlayerSummaryJob(corrID, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) runEmpyrionPlayerSummaryJob(corrID string, req *panelv1.EmpyrionPlayerSummaryRequest) {
|
||||
dockerBin, err := exec.LookPath("docker")
|
||||
if err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "docker binary not found on agent host", SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
savesVol := "panel-" + req.InstanceId + "-saves"
|
||||
if err := exec.Command(dockerBin, "volume", "inspect", savesVol).Run(); err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: fmt.Sprintf("instance saves volume %q not found on this agent", savesVol), SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
limit := req.MaxLoginRows
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, dockerBin,
|
||||
"run", "--rm",
|
||||
"-v", savesVol+":/saves:ro",
|
||||
"--entrypoint", "sh",
|
||||
"alpine:3.20", "-c", playerSummaryScript(), "_",
|
||||
req.SteamId, fmt.Sprintf("%d", limit),
|
||||
).Output()
|
||||
if err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "sqlite query failed: " + err.Error(), SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
type wireStats struct {
|
||||
KilledEnemies int64 `json:"killed_enemies"`
|
||||
KilledAllied int64 `json:"killed_allied"`
|
||||
KilledAnimals int64 `json:"killed_animals"`
|
||||
KilledDrones int64 `json:"killed_drones"`
|
||||
KilledPlayers int64 `json:"killed_players"`
|
||||
KilledAlliedPlayers int64 `json:"killed_allied_players"`
|
||||
Died int64 `json:"died"`
|
||||
Score int64 `json:"score"`
|
||||
BlocksPlaced int64 `json:"blocks_placed"`
|
||||
BlocksDigged int64 `json:"blocks_digged"`
|
||||
WalkedMeters int64 `json:"walked_meters"`
|
||||
JetpackMeters int64 `json:"jetpack_meters"`
|
||||
HvMeters int64 `json:"hv_meters"`
|
||||
SvMeters int64 `json:"sv_meters"`
|
||||
CvMeters int64 `json:"cv_meters"`
|
||||
Playtime float64 `json:"playtime"`
|
||||
LightYears float64 `json:"light_years"`
|
||||
AstroUnits float64 `json:"astronomical_units"`
|
||||
}
|
||||
type wireSession struct {
|
||||
LoginTicks int64 `json:"login_ticks"`
|
||||
LogoffTicks int64 `json:"logoff_ticks"`
|
||||
PlayerName string `json:"player_name"`
|
||||
BuildNr int64 `json:"build_nr"`
|
||||
IP string `json:"ip"`
|
||||
OS string `json:"os"`
|
||||
GFX string `json:"gfx"`
|
||||
CPU string `json:"cpu"`
|
||||
}
|
||||
type wireInvItem struct {
|
||||
ItemID int32 `json:"item_id"`
|
||||
Count int32 `json:"count"`
|
||||
}
|
||||
type wirePos struct {
|
||||
LastPlayfield string `json:"last_playfield"`
|
||||
LastX float64 `json:"last_x"`
|
||||
LastY float64 `json:"last_y"`
|
||||
LastZ float64 `json:"last_z"`
|
||||
Credits int64 `json:"credits"`
|
||||
}
|
||||
type wire struct {
|
||||
SteamID string `json:"steam_id"`
|
||||
PlayerName string `json:"player_name"`
|
||||
EntityID int64 `json:"entity_id"`
|
||||
FactionGroup int32 `json:"faction_group"`
|
||||
FactionID int32 `json:"faction_id"`
|
||||
Position []wirePos `json:"position"`
|
||||
Stats []wireStats `json:"stats"`
|
||||
Sessions []wireSession `json:"sessions"`
|
||||
InventoryGametime int64 `json:"inventory_gametime"`
|
||||
Inventory []wireInvItem `json:"inventory"`
|
||||
}
|
||||
var w wire
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &w); err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "parse sqlite output: " + err.Error(), SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
res := &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: true,
|
||||
SteamId: w.SteamID,
|
||||
PlayerName: strings.Trim(w.PlayerName, "\""),
|
||||
EntityId: w.EntityID,
|
||||
FactionGroup: w.FactionGroup,
|
||||
FactionId: w.FactionID,
|
||||
InventoryGametime: w.InventoryGametime,
|
||||
}
|
||||
if len(w.Position) > 0 {
|
||||
res.LastPlayfield = w.Position[0].LastPlayfield
|
||||
res.LastX = w.Position[0].LastX
|
||||
res.LastY = w.Position[0].LastY
|
||||
res.LastZ = w.Position[0].LastZ
|
||||
res.Credits = w.Position[0].Credits
|
||||
}
|
||||
if len(w.Stats) > 0 {
|
||||
s := w.Stats[0]
|
||||
res.Stats = &panelv1.PlayerStatsRow{
|
||||
KilledEnemies: s.KilledEnemies, KilledAllied: s.KilledAllied,
|
||||
KilledAnimals: s.KilledAnimals, KilledDrones: s.KilledDrones,
|
||||
KilledPlayers: s.KilledPlayers, KilledAlliedPlayers: s.KilledAlliedPlayers,
|
||||
Died: s.Died, Score: s.Score,
|
||||
BlocksPlaced: s.BlocksPlaced, BlocksDigged: s.BlocksDigged,
|
||||
WalkedMeters: s.WalkedMeters, JetpackMeters: s.JetpackMeters,
|
||||
HvMeters: s.HvMeters, SvMeters: s.SvMeters, CvMeters: s.CvMeters,
|
||||
Playtime: s.Playtime, LightYears: s.LightYears, AstronomicalUnits: s.AstroUnits,
|
||||
}
|
||||
}
|
||||
for _, s := range w.Sessions {
|
||||
res.Sessions = append(res.Sessions, &panelv1.PlayerLoginRow{
|
||||
LoginTicks: s.LoginTicks, LogoffTicks: s.LogoffTicks,
|
||||
PlayerName: s.PlayerName, BuildNr: s.BuildNr,
|
||||
Ip: s.IP, Os: s.OS, Gfx: s.GFX, Cpu: s.CPU,
|
||||
})
|
||||
}
|
||||
for _, it := range w.Inventory {
|
||||
res.Inventory = append(res.Inventory, &panelv1.PlayerInventoryItem{
|
||||
ItemId: it.ItemID, Count: it.Count,
|
||||
})
|
||||
}
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, res)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendEmpyrionPlayerSummaryResult(corrID string, res *panelv1.EmpyrionPlayerSummaryResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_EmpyrionPlayerSummaryResult{
|
||||
EmpyrionPlayerSummaryResult: res,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package dispatch
|
||||
|
||||
// Agent-side Empyrion workshop-scenario installer. Runs entirely on the
|
||||
// local Docker daemon, so a multi-agent panel can install Reforged Eden 2
|
||||
// (and any other workshop scenario) into an Empyrion instance regardless
|
||||
// of which agent owns it.
|
||||
//
|
||||
// Flow:
|
||||
// 1. SteamCMD sidecar mounts panel-empyrion-workshop (an agent-local
|
||||
// cache volume, lazily created on first install) and downloads the
|
||||
// workshop item.
|
||||
// 2. Alpine helper sidecar mounts the workshop volume + the instance's
|
||||
// `panel-<INSTANCE>-game` volume and copies the scenario files into
|
||||
// Content/Scenarios/<scenario_name>/.
|
||||
// 3. A marker file (.panel-installed) is touched so the empyrion
|
||||
// entrypoint can confirm the install.
|
||||
//
|
||||
// Progress is streamed back to the controller as LogLine envelopes with
|
||||
// stream="scenario" so the panel UI's scenario job log shows what's
|
||||
// happening live.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
func (d *Dispatcher) handleEmpyrionScenarioInstall(corrID string, req *panelv1.EmpyrionScenarioInstallRequest) {
|
||||
if req == nil || req.JobId == "" || req.InstanceId == "" || req.WorkshopId == "" || req.ScenarioName == "" {
|
||||
d.sendEmpyrionScenarioResult(corrID, req.GetJobId(), false, "missing required fields", req.GetScenarioName())
|
||||
return
|
||||
}
|
||||
appID := req.AppId
|
||||
if appID == "" {
|
||||
appID = "530870"
|
||||
}
|
||||
go d.runEmpyrionScenarioJob(corrID, req, appID)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) runEmpyrionScenarioJob(corrID string, req *panelv1.EmpyrionScenarioInstallRequest, appID string) {
|
||||
// Redact the password from any line we forward upstream, since
|
||||
// SteamCMD CAN echo `+login user PASS` back in error paths.
|
||||
redact := func(s string) string {
|
||||
if req.SteamPassword == "" || len(req.SteamPassword) < 4 {
|
||||
return s
|
||||
}
|
||||
return strings.ReplaceAll(s, req.SteamPassword, "[REDACTED]")
|
||||
}
|
||||
emit := func(line string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: req.InstanceId,
|
||||
Stream: "scenario",
|
||||
At: timestamppb.Now(),
|
||||
Line: redact(line),
|
||||
}},
|
||||
})
|
||||
}
|
||||
finish := func(ok bool, msg string) {
|
||||
d.sendEmpyrionScenarioResult(corrID, req.JobId, ok, msg, req.ScenarioName)
|
||||
}
|
||||
|
||||
emit(fmt.Sprintf("[scenario] starting workshop download: app=%s item=%s scenario=%q", appID, req.WorkshopId, req.ScenarioName))
|
||||
|
||||
dockerBin, err := exec.LookPath("docker")
|
||||
if err != nil {
|
||||
finish(false, "docker binary not found on agent host")
|
||||
return
|
||||
}
|
||||
|
||||
// Phase 1 — workshop download. The volume name is per-agent (Docker
|
||||
// daemons don't share volumes); it's created lazily by the first run.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// Build the SteamCMD argv for an attempt — login portion is the only
|
||||
// variable. `+app_license_request` refreshes Steam's entitlement
|
||||
// cache before the workshop download (cheap fix for stale-license
|
||||
// "Access Denied" on owned games).
|
||||
buildArgs := func(loginArgs []string) []string {
|
||||
base := []string{
|
||||
"run", "--rm",
|
||||
"-v", "panel-steamcmd-auth:/root/.local/share/Steam",
|
||||
"-v", "panel-empyrion-workshop:/empyrion-workshop/steamapps/workshop",
|
||||
"steamcmd/steamcmd:latest",
|
||||
"+@sSteamCmdForcePlatformType", "windows",
|
||||
"+force_install_dir", "/empyrion-workshop",
|
||||
}
|
||||
base = append(base, loginArgs...)
|
||||
base = append(base,
|
||||
"+app_license_request", appID,
|
||||
"+workshop_download_item", appID, req.WorkshopId, "validate",
|
||||
"+quit",
|
||||
)
|
||||
return base
|
||||
}
|
||||
|
||||
var out string
|
||||
switch {
|
||||
case req.SteamUsername == "":
|
||||
emit("[scenario] no Steam credentials provided — trying anonymous (will fail for app workshops that require ownership)")
|
||||
out, err = runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", "anonymous"}), emit)
|
||||
default:
|
||||
// Try cached refresh-token login first (no password). If the
|
||||
// agent's panel-steamcmd-auth volume has a valid token from a
|
||||
// prior install, this skips Steam Guard entirely — no push
|
||||
// notification fires. Only when the cache is missing or expired
|
||||
// (Logon failure / no cached credentials in the SteamCMD output)
|
||||
// do we fall back to full password auth, which DOES require
|
||||
// approving a Steam Guard push but then re-primes the cache for
|
||||
// future runs.
|
||||
emit(fmt.Sprintf("[scenario] login attempt 1: %q via cached refresh token (no Steam Guard push if cache valid)", req.SteamUsername))
|
||||
cachedOut, cachedErr := runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", req.SteamUsername}), emit)
|
||||
needsPwd := cachedErr != nil ||
|
||||
strings.Contains(cachedOut, "Logon failure") ||
|
||||
strings.Contains(cachedOut, "No cached credentials") ||
|
||||
strings.Contains(cachedOut, "FAILED login") ||
|
||||
strings.Contains(cachedOut, "Login failure")
|
||||
if !needsPwd {
|
||||
out, err = cachedOut, cachedErr
|
||||
break
|
||||
}
|
||||
if req.SteamPassword == "" {
|
||||
finish(false, "Steam refresh-token cache is missing/expired on this agent and no password is available to re-prime it. Re-enter credentials via the Steam login modal.")
|
||||
return
|
||||
}
|
||||
emit(fmt.Sprintf("[scenario] login attempt 2: %q with password (Steam Guard push will fire — approve on phone)", req.SteamUsername))
|
||||
out, err = runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", req.SteamUsername, req.SteamPassword}), emit)
|
||||
}
|
||||
out = redact(out)
|
||||
if err != nil {
|
||||
finish(false, "steamcmd: "+err.Error())
|
||||
return
|
||||
}
|
||||
// Steam reports "Access Denied" when the logged-in account can't
|
||||
// pull this workshop item. Common causes (in order of likelihood):
|
||||
// 1. Account doesn't own the app — most common
|
||||
// 2. License cache stale — we already issued +app_license_request
|
||||
// above, so if it still errors that's not it
|
||||
// 3. Workshop item is private / friends-only / restricted by author
|
||||
// 4. Workshop item requires the operator to "subscribe" to it via
|
||||
// the desktop Steam client at least once before SteamCMD can
|
||||
// pull it (some apps gate this)
|
||||
// 5. Family Library Sharing — the account borrows Empyrion via
|
||||
// sharing instead of owning a license; SteamCMD requires real
|
||||
// ownership, not borrowed licenses
|
||||
if strings.Contains(out, "ERROR! Download item") && strings.Contains(out, "Access Denied") {
|
||||
// Account confirmed-owner of the app but still denied — this is
|
||||
// almost always a publisher-side restriction on workshop_download_item
|
||||
// (Empyrion / Eleon is the canonical example: the desktop Steam
|
||||
// client uses a different API path that's allowed, but SteamCMD's
|
||||
// path is denied for this app's workshop). NOT a panel bug, NOT an
|
||||
// account bug. Manual upload is the only reliable path.
|
||||
ownsApp := strings.Contains(out, "already owned")
|
||||
var msg string
|
||||
if ownsApp {
|
||||
msg = fmt.Sprintf(
|
||||
"Workshop item %s denied even though account %q owns app %s (Steam confirmed: \"AppID %s already owned\"). "+
|
||||
"This is a publisher-side restriction on SteamCMD's workshop API — common for Empyrion (Eleon disables CLI workshop downloads; the desktop Steam client uses a different allowed path). "+
|
||||
"Workaround: subscribe to the item in the desktop Steam client (Workshop → Subscribe), let it download to %%STEAM%%/steamapps/workshop/content/%s/%s/, then upload that folder into the instance's panel-empyrion-game volume at Content/Scenarios/<scenario_name>/. The panel can't bypass this — Steam policy enforced server-side.",
|
||||
req.WorkshopId, req.SteamUsername, appID, appID, appID, req.WorkshopId,
|
||||
)
|
||||
} else {
|
||||
msg = fmt.Sprintf(
|
||||
"Steam returned Access Denied for workshop item %s after logging in as %q. "+
|
||||
"Likely causes: (a) account doesn't actually own app %s (Family Library Sharing borrows don't count — needs a purchase license); "+
|
||||
"(b) workshop item is private/friends-only/restricted by author; "+
|
||||
"(c) item requires desktop-Steam-client subscription first. "+
|
||||
"Workaround: download via desktop Steam client and drop Content/Scenarios/<folder> into the instance volume manually.",
|
||||
req.WorkshopId, req.SteamUsername, appID,
|
||||
)
|
||||
}
|
||||
finish(false, msg)
|
||||
return
|
||||
}
|
||||
if !strings.Contains(out, "Success. Downloaded item") {
|
||||
// Don't fail outright on a missing success marker — some
|
||||
// SteamCMD versions return without that exact string when the
|
||||
// item is already cached. Sanity-check the file actually
|
||||
// exists in the volume before deciding.
|
||||
probe := []string{"run", "--rm", "-v", "panel-empyrion-workshop:/ws", "alpine:3.20",
|
||||
"sh", "-c", fmt.Sprintf("test -d /ws/content/%s/%s && echo cached", appID, req.WorkshopId)}
|
||||
probeOut, _ := exec.CommandContext(ctx, dockerBin, probe...).CombinedOutput()
|
||||
if !strings.Contains(string(probeOut), "cached") {
|
||||
finish(false, "steamcmd did not download item — see log for details")
|
||||
return
|
||||
}
|
||||
emit("[scenario] workshop item already cached on this agent — skipping re-download")
|
||||
}
|
||||
|
||||
// Phase 2 — copy into instance volume.
|
||||
emit("[scenario] copying scenario files into instance volume…")
|
||||
gameVol := "panel-" + req.InstanceId + "-game"
|
||||
dst := "/dst/Content/Scenarios/" + req.ScenarioName
|
||||
|
||||
// Verify the instance volume exists; if not, the instance was
|
||||
// never created on this agent.
|
||||
probe := []string{"volume", "inspect", gameVol}
|
||||
if err := exec.CommandContext(ctx, dockerBin, probe...).Run(); err != nil {
|
||||
finish(false, fmt.Sprintf("instance volume %q not found on this agent — is the empyrion instance created here?", gameVol))
|
||||
return
|
||||
}
|
||||
|
||||
helperArgs := []string{
|
||||
"run", "--rm",
|
||||
"-v", "panel-empyrion-workshop:/src",
|
||||
"-v", gameVol + ":/dst",
|
||||
"alpine:3.20", "sh", "-c",
|
||||
fmt.Sprintf(
|
||||
"mkdir -p %s && rm -rf %s/* %s/.* 2>/dev/null; "+
|
||||
"cp -a /src/content/%s/%s/. %s/ && date > %s/.panel-installed && echo OK",
|
||||
shQuote(dst), shQuote(dst), shQuote(dst),
|
||||
appID, req.WorkshopId, shQuote(dst), shQuote(dst),
|
||||
),
|
||||
}
|
||||
out, err = runWithLineEmit(ctx, dockerBin, helperArgs, emit)
|
||||
if err != nil {
|
||||
finish(false, "scenario copy failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if !strings.Contains(out, "OK") {
|
||||
finish(false, "scenario copy did not confirm OK — see log for details")
|
||||
return
|
||||
}
|
||||
|
||||
emit(fmt.Sprintf("[scenario] %q installed into %s", req.ScenarioName, req.InstanceId))
|
||||
finish(true, "")
|
||||
}
|
||||
|
||||
// shQuote single-quotes a path for safe inclusion in a `sh -c` script.
|
||||
// Embedded single quotes are escaped via the standard '\'' trick. Used
|
||||
// for scenario_name which is operator input — even though the controller
|
||||
// already rejects path-traversal characters, defense-in-depth at the
|
||||
// agent layer is cheap.
|
||||
func shQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
// runWithLineEmit spawns docker, captures combined stdout/stderr,
|
||||
// emits each line as it arrives, and returns the full combined output
|
||||
// for post-mortem checks (success markers etc.) when the process exits.
|
||||
func runWithLineEmit(ctx context.Context, bin string, args []string, emit func(string)) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, bin, args...)
|
||||
pipe, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
cmd.Stderr = cmd.Stdout
|
||||
if err := cmd.Start(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var buf strings.Builder
|
||||
scanCh := make(chan struct{})
|
||||
go func() {
|
||||
defer close(scanCh)
|
||||
readBuf := make([]byte, 4096)
|
||||
var line strings.Builder
|
||||
for {
|
||||
n, err := pipe.Read(readBuf)
|
||||
if n > 0 {
|
||||
chunk := readBuf[:n]
|
||||
buf.Write(chunk)
|
||||
for _, b := range chunk {
|
||||
if b == '\n' {
|
||||
emit(strings.TrimRight(line.String(), "\r"))
|
||||
line.Reset()
|
||||
} else {
|
||||
line.WriteByte(b)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if line.Len() > 0 {
|
||||
emit(strings.TrimRight(line.String(), "\r"))
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
waitErr := cmd.Wait()
|
||||
<-scanCh
|
||||
if waitErr != nil {
|
||||
return buf.String(), waitErr
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendEmpyrionScenarioResult(corrID, jobID string, ok bool, errMsg, scenarioName string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_EmpyrionScenarioInstallResult{
|
||||
EmpyrionScenarioInstallResult: &panelv1.EmpyrionScenarioInstallResult{
|
||||
JobId: jobID,
|
||||
Ok: ok,
|
||||
Error: errMsg,
|
||||
ScenarioName: scenarioName,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,662 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// File management: operations are rooted at each instance's declared
|
||||
// BrowseableRoot inside the container when the container is running +
|
||||
// the runtime supports container-path ops (Docker). Otherwise we fall
|
||||
// back to the host-side DataPath bind mount. Either path is safe-joined
|
||||
// to prevent escape.
|
||||
|
||||
const (
|
||||
// fsMaxReadBytes caps a single file-read RPC. Big enough for full
|
||||
// world saves and most ARK/Empyrion scenario zips (a few-hundred MB
|
||||
// is common). True streaming would remove the cap entirely; until
|
||||
// then 4 GiB matches the controller-side upload cap.
|
||||
fsMaxReadBytes = 4 * 1024 * 1024 * 1024
|
||||
fsMaxListSize = 5000
|
||||
|
||||
// fsCopyTimeout bounds a single CopyToContainer / CopyFromContainer
|
||||
// call. A multi-GB upload over the loopback Docker socket can take
|
||||
// 30+ seconds, and an extract that ships the whole archive's
|
||||
// contents back to the container can be larger still — give it
|
||||
// real headroom so we don't surface spurious timeouts.
|
||||
fsCopyTimeout = 10 * time.Minute
|
||||
)
|
||||
|
||||
// ---- Path safety ----
|
||||
|
||||
// safeJoinHost joins rel onto root using OS path semantics, rejecting
|
||||
// anything that canonicalizes outside of root.
|
||||
func safeJoinHost(root, rel string) (string, error) {
|
||||
rel = strings.TrimPrefix(rel, "/")
|
||||
rel = strings.TrimPrefix(rel, "\\")
|
||||
if rel == "" {
|
||||
return root, nil
|
||||
}
|
||||
joined := filepath.Join(root, rel)
|
||||
absRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
absJoined, err := filepath.Abs(joined)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sep := string(filepath.Separator)
|
||||
if absJoined != absRoot && !strings.HasPrefix(absJoined+sep, absRoot+sep) {
|
||||
return "", fmt.Errorf("path escapes instance data directory")
|
||||
}
|
||||
return absJoined, nil
|
||||
}
|
||||
|
||||
// safeJoinContainer joins rel onto root using Linux container semantics
|
||||
// (always forward-slash). Returns the cleaned absolute path.
|
||||
func safeJoinContainer(root, rel string) (string, error) {
|
||||
rel = strings.TrimPrefix(rel, "/")
|
||||
if rel == "" {
|
||||
return root, nil
|
||||
}
|
||||
joined := path.Clean(path.Join(root, rel))
|
||||
if joined != root && !strings.HasPrefix(joined+"/", root+"/") {
|
||||
return "", fmt.Errorf("path escapes instance root")
|
||||
}
|
||||
return joined, nil
|
||||
}
|
||||
|
||||
// rootPaths returns the full list of allowed root paths for a record
|
||||
// (flattens BrowseableRoots → container paths, or falls back to the
|
||||
// singular BrowseableRoot). Always returns at least one entry.
|
||||
func rootPaths(rec *instanceRecord) []string {
|
||||
if len(rec.BrowseableRoots) > 0 {
|
||||
out := make([]string, 0, len(rec.BrowseableRoots))
|
||||
for _, r := range rec.BrowseableRoots {
|
||||
if r.Path != "" {
|
||||
out = append(out, r.Path)
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
if rec.BrowseableRoot != "" {
|
||||
return []string{rec.BrowseableRoot}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// safeJoinAny resolves a caller-supplied path against ANY of the module's
|
||||
// declared roots. Semantics:
|
||||
//
|
||||
// - If p starts with "/", it must be EQUAL TO or UNDER one of the
|
||||
// roots; we return it cleaned as-is. Lets the UI send container-
|
||||
// absolute paths when it knows the full layout (multi-root modules).
|
||||
// - Otherwise p is relative, resolved against `defaultRoot`. Same
|
||||
// safeJoinContainer behaviour as before.
|
||||
//
|
||||
// Returns an error if the path can't be mapped to any root or tries to
|
||||
// escape via `..`.
|
||||
func safeJoinAny(defaultRoot string, roots []string, p string) (string, error) {
|
||||
if strings.HasPrefix(p, "/") {
|
||||
cleaned := path.Clean(p)
|
||||
for _, r := range roots {
|
||||
if cleaned == r || strings.HasPrefix(cleaned+"/", r+"/") {
|
||||
return cleaned, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("path %q is not under any declared root", cleaned)
|
||||
}
|
||||
return safeJoinContainer(defaultRoot, p)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) lookupRecord(instanceID string) (*instanceRecord, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
rec, ok := d.instances[instanceID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("instance %q not on this target", instanceID)
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// useContainerOps decides whether to go through the container runtime
|
||||
// (true) or fall back to the host bind mount (false). Any module that
|
||||
// declares a browseable root uses container ops — if the main container
|
||||
// isn't running at the moment, we spin up a helper sidecar (see
|
||||
// getFsTargetContainerID). This lets operators browse/edit files on
|
||||
// stopped instances.
|
||||
func (d *Dispatcher) useContainerOps(rec *instanceRecord) bool {
|
||||
return rec.BrowseableRoot != ""
|
||||
}
|
||||
|
||||
// ---- FS helper sidecar: file ops on stopped containers ----
|
||||
|
||||
// fsHelperImage is the base for the helper container. We use debian-slim
|
||||
// rather than alpine because our `ls --time-style=+%s` call relies on GNU
|
||||
// coreutils — busybox's ls doesn't accept --time-style. debian:12-slim is
|
||||
// already on disk for every panel-native game module, so no extra pull.
|
||||
const fsHelperImage = "debian:12-slim"
|
||||
|
||||
// fsHelperIdleTTL is how long a helper container is kept alive after its
|
||||
// last file-op access. Balance: long enough that click-around browsing
|
||||
// reuses the same helper without respawning; short enough that a forgotten
|
||||
// Files tab doesn't pin volumes forever.
|
||||
const fsHelperIdleTTL = 5 * time.Minute
|
||||
|
||||
// getFsTargetContainerID picks the container to route a file op through:
|
||||
// the main instance container if it's running, or a helper sidecar
|
||||
// (lazy-created + started on demand) if it isn't. The helper reuses the
|
||||
// same named volumes as the main container, so the operator sees the
|
||||
// same files they would see with the game running.
|
||||
func (d *Dispatcher) getFsTargetContainerID(ctx context.Context, rec *instanceRecord) (string, error) {
|
||||
mainName := "panel-" + rec.InstanceID
|
||||
st, err := d.runtime.InspectByName(ctx, mainName)
|
||||
if err == nil && st.Status == "running" {
|
||||
// Main is alive — tear down any lingering helper so volumes
|
||||
// aren't held by two containers simultaneously during live play.
|
||||
d.teardownFsHelper(rec.InstanceID)
|
||||
return st.ContainerID, nil
|
||||
}
|
||||
return d.ensureFsHelper(ctx, rec)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) ensureFsHelper(ctx context.Context, rec *instanceRecord) (string, error) {
|
||||
helperName := "panel-" + rec.InstanceID + "-fs"
|
||||
|
||||
// Reuse existing helper if it's still healthy.
|
||||
d.mu.Lock()
|
||||
h, ok := d.fsHelpers[rec.InstanceID]
|
||||
d.mu.Unlock()
|
||||
if ok {
|
||||
st, err := d.runtime.InspectByName(ctx, helperName)
|
||||
if err == nil && st.Status == "running" {
|
||||
d.mu.Lock()
|
||||
h.lastAccess = time.Now()
|
||||
d.mu.Unlock()
|
||||
return h.containerID, nil
|
||||
}
|
||||
// Stale record — container gone or in a bad state. Drop + rebuild.
|
||||
d.teardownFsHelper(rec.InstanceID)
|
||||
}
|
||||
|
||||
// Check for an orphaned helper container on Docker's side from a
|
||||
// previous agent run. When the agent restarts, in-memory fsHelpers
|
||||
// empties but the container persists. Adopt it if it's running,
|
||||
// remove it if not (Create will then succeed).
|
||||
if st, err := d.runtime.InspectByName(ctx, helperName); err == nil {
|
||||
if st.Status == "running" {
|
||||
d.log.Info("fs helper adopted from prior agent run",
|
||||
"instance_id", rec.InstanceID, "container_id", st.ContainerID)
|
||||
d.mu.Lock()
|
||||
d.fsHelpers[rec.InstanceID] = &fsHelper{containerID: st.ContainerID, lastAccess: time.Now()}
|
||||
d.mu.Unlock()
|
||||
return st.ContainerID, nil
|
||||
}
|
||||
d.log.Info("removing stale fs helper before recreate",
|
||||
"instance_id", rec.InstanceID, "container_id", st.ContainerID, "status", st.Status)
|
||||
rmCtx, rmCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = d.runtime.Remove(rmCtx, st.ContainerID)
|
||||
rmCancel()
|
||||
}
|
||||
|
||||
// Resolve the instance's volumes from its manifest so the helper sees
|
||||
// exactly what the game would.
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("module %q not loaded; can't mount helper volumes", rec.ModuleID)
|
||||
}
|
||||
volumes := agentmodule.ResolveVolumes(manifest, rec.InstanceID, rec.DataPath)
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: rec.InstanceID + "-fs",
|
||||
IsSidecar: true,
|
||||
Image: fsHelperImage,
|
||||
Command: []string{"sleep", "infinity"},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
containerID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create fs helper: %w", err)
|
||||
}
|
||||
if err := d.runtime.Start(ctx, containerID); err != nil {
|
||||
// Clean up the half-created container so next call doesn't see a
|
||||
// zombie from `Created` state.
|
||||
rmCtx, rmCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer rmCancel()
|
||||
_ = d.runtime.Remove(rmCtx, containerID)
|
||||
return "", fmt.Errorf("start fs helper: %w", err)
|
||||
}
|
||||
d.mu.Lock()
|
||||
d.fsHelpers[rec.InstanceID] = &fsHelper{containerID: containerID, lastAccess: time.Now()}
|
||||
d.mu.Unlock()
|
||||
d.log.Info("fs helper started", "instance_id", rec.InstanceID, "container_id", containerID)
|
||||
return containerID, nil
|
||||
}
|
||||
|
||||
// teardownFsHelper stops and removes the helper sidecar for an instance,
|
||||
// if one exists. Safe to call at any time — no-ops on absent helpers.
|
||||
// Called on main-container start (to free volumes) and on instance
|
||||
// delete (cleanup).
|
||||
func (d *Dispatcher) teardownFsHelper(instanceID string) {
|
||||
d.mu.Lock()
|
||||
h, ok := d.fsHelpers[instanceID]
|
||||
if ok {
|
||||
delete(d.fsHelpers, instanceID)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Stop(ctx, h.containerID, 2*time.Second)
|
||||
_ = d.runtime.Remove(ctx, h.containerID)
|
||||
d.log.Info("fs helper torn down", "instance_id", instanceID)
|
||||
}
|
||||
|
||||
// fsHelperReaper runs for the lifetime of the dispatcher and tears down
|
||||
// any helper that hasn't been touched in fsHelperIdleTTL. Scans every
|
||||
// minute — granular enough that abandoned Files tabs don't pin resources
|
||||
// for long, coarse enough that the scan itself is cheap.
|
||||
func (d *Dispatcher) fsHelperReaper() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
now := time.Now()
|
||||
d.mu.Lock()
|
||||
stale := make([]string, 0)
|
||||
for id, h := range d.fsHelpers {
|
||||
if now.Sub(h.lastAccess) > fsHelperIdleTTL {
|
||||
stale = append(stale, id)
|
||||
}
|
||||
}
|
||||
d.mu.Unlock()
|
||||
for _, id := range stale {
|
||||
d.teardownFsHelper(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- FsList ----
|
||||
|
||||
func (d *Dispatcher) handleFsList(corrID string, req *panelv1.FsListRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsListResult(corrID, nil, "", err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
entries, err := d.listInContainer(rec, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsListResult(corrID, nil, req.Path, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsListResult(corrID, entries, req.Path, "")
|
||||
return
|
||||
}
|
||||
// Host fallback — bind mount.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsListResult(corrID, nil, req.Path, "no file storage available (container not running and no host data path)")
|
||||
return
|
||||
}
|
||||
absPath, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsListResult(corrID, nil, req.Path, err.Error())
|
||||
return
|
||||
}
|
||||
entries, err := os.ReadDir(absPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
d.sendFsListResult(corrID, nil, req.Path, "not found")
|
||||
return
|
||||
}
|
||||
d.sendFsListResult(corrID, nil, req.Path, err.Error())
|
||||
return
|
||||
}
|
||||
out := make([]*panelv1.FsEntry, 0, len(entries))
|
||||
for i, e := range entries {
|
||||
if i >= fsMaxListSize {
|
||||
break
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, &panelv1.FsEntry{
|
||||
Name: e.Name(),
|
||||
Path: filepath.ToSlash(filepath.Join(req.Path, e.Name())),
|
||||
IsDir: e.IsDir(),
|
||||
Size: info.Size(),
|
||||
ModTime: timestamppb.New(info.ModTime()),
|
||||
})
|
||||
}
|
||||
d.sendFsListResult(corrID, out, filepath.ToSlash(req.Path), "")
|
||||
}
|
||||
|
||||
// listInContainer shells out to `ls -la --time-style=+%s <root/rel>` and
|
||||
// parses the output into FsEntry records.
|
||||
func (d *Dispatcher) listInContainer(rec *instanceRecord, rel string) ([]*panelv1.FsEntry, error) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// -L dereferences symlinks so that listing a `@ModName` link (which
|
||||
// points into the shared workshop tree) shows the mod's contents, not
|
||||
// the symlink itself. Without this, `ls -l` on a symlinked dir prints
|
||||
// the symlink as a single entry and the UI looks empty.
|
||||
stdout, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{
|
||||
"sh", "-c",
|
||||
"ls -lAL --time-style=+%s -- " + shellQuote(abs),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exec ls: %w", err)
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("ls exited %d", code)
|
||||
}
|
||||
return nil, fmt.Errorf("%s", msg)
|
||||
}
|
||||
return parseLsOutput(string(stdout), rel)
|
||||
}
|
||||
|
||||
// parseLsOutput parses `ls -lA --time-style=+%s` output. Each line:
|
||||
//
|
||||
// drwxr-xr-x 2 root root 4096 1735000000 somefile
|
||||
//
|
||||
// We split on any whitespace and take fields. The date field is a unix
|
||||
// timestamp (because of --time-style=+%s).
|
||||
func parseLsOutput(out, relRoot string) ([]*panelv1.FsEntry, error) {
|
||||
lines := strings.Split(strings.TrimSpace(out), "\n")
|
||||
entries := make([]*panelv1.FsEntry, 0, len(lines))
|
||||
for i, line := range lines {
|
||||
if i == 0 && strings.HasPrefix(line, "total ") {
|
||||
continue
|
||||
}
|
||||
// Split into at most 7 fields so filenames with spaces survive.
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 7 {
|
||||
continue
|
||||
}
|
||||
// fields[0]=mode 1=nlink 2=user 3=group 4=size 5=unixts 6+=name
|
||||
size, _ := strconv.ParseInt(fields[4], 10, 64)
|
||||
tsUnix, _ := strconv.ParseInt(fields[5], 10, 64)
|
||||
name := strings.Join(fields[6:], " ")
|
||||
// For symlinks "name -> target", keep just the name.
|
||||
if strings.Contains(name, " -> ") {
|
||||
name = strings.SplitN(name, " -> ", 2)[0]
|
||||
}
|
||||
if name == "" || name == "." || name == ".." {
|
||||
continue
|
||||
}
|
||||
isDir := strings.HasPrefix(fields[0], "d")
|
||||
entries = append(entries, &panelv1.FsEntry{
|
||||
Name: name,
|
||||
Path: path.Join(relRoot, name),
|
||||
IsDir: isDir,
|
||||
Size: size,
|
||||
ModTime: timestamppb.New(time.Unix(tsUnix, 0)),
|
||||
})
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// shellQuote wraps s in single quotes, escaping embedded quotes for
|
||||
// interpolation into `sh -c`.
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsListResult(corrID string, entries []*panelv1.FsEntry, p, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsListResult{
|
||||
FsListResult: &panelv1.FsListResult{Entries: entries, Path: p, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsRead ----
|
||||
|
||||
func (d *Dispatcher) handleFsRead(corrID string, req *panelv1.FsReadRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
data, err := d.runtime.CopyFileFromContainer(ctx, targetID, abs)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if len(data) > fsMaxReadBytes {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, fmt.Sprintf("file too large (%d bytes; max %d)", len(data), fsMaxReadBytes))
|
||||
return
|
||||
}
|
||||
d.sendFsReadResult(corrID, req.Path, data, "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, "no file storage available")
|
||||
return
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if info.IsDir() {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, "is a directory")
|
||||
return
|
||||
}
|
||||
if info.Size() > fsMaxReadBytes {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, fmt.Sprintf("file too large (%d bytes; max %d)", info.Size(), fsMaxReadBytes))
|
||||
return
|
||||
}
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsReadResult(corrID, req.Path, data, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsReadResult(corrID, p string, content []byte, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsReadResult{
|
||||
FsReadResult: &panelv1.FsReadResult{Path: p, Content: content, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsWrite ----
|
||||
|
||||
func (d *Dispatcher) handleFsWrite(corrID string, req *panelv1.FsWriteRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
|
||||
if err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
// Make sure the parent dir exists. `mkdir -p` inside container.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
targetID, terr := d.getFsTargetContainerID(ctx, rec)
|
||||
if terr != nil {
|
||||
cancel()
|
||||
d.sendFsWriteResult(corrID, 0, terr.Error())
|
||||
return
|
||||
}
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(abs))})
|
||||
cancel()
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel2()
|
||||
if err := d.runtime.CopyFileToContainer(ctx2, targetID, abs, req.Content); err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsWriteResult(corrID, int64(len(req.Content)), "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsWriteResult(corrID, 0, "no file storage available")
|
||||
return
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(abs, req.Content, 0o644); err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsWriteResult(corrID, int64(len(req.Content)), "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsWriteResult(corrID string, bytesWritten int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsWriteResult{
|
||||
FsWriteResult: &panelv1.FsWriteResult{BytesWritten: bytesWritten, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsDelete ----
|
||||
|
||||
func (d *Dispatcher) handleFsDelete(corrID string, req *panelv1.FsDeleteRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if abs == rec.BrowseableRoot {
|
||||
d.sendFsDeleteResult(corrID, "refusing to delete instance root")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
flag := ""
|
||||
if req.Recursive {
|
||||
flag = "-rf"
|
||||
} else {
|
||||
flag = "-f"
|
||||
}
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"rm " + flag + " -- " + shellQuote(abs)})
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendFsDeleteResult(corrID, strings.TrimSpace(string(stderr)))
|
||||
return
|
||||
}
|
||||
d.sendFsDeleteResult(corrID, "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsDeleteResult(corrID, "no file storage available")
|
||||
return
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if abs == rec.DataPath {
|
||||
d.sendFsDeleteResult(corrID, "refusing to delete instance data root")
|
||||
return
|
||||
}
|
||||
var delErr error
|
||||
if req.Recursive {
|
||||
delErr = os.RemoveAll(abs)
|
||||
} else {
|
||||
delErr = os.Remove(abs)
|
||||
}
|
||||
if delErr != nil {
|
||||
d.sendFsDeleteResult(corrID, delErr.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsDeleteResult(corrID, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsDeleteResult(corrID, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsDeleteResult{
|
||||
FsDeleteResult: &panelv1.FsDeleteResult{Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
package dispatch
|
||||
|
||||
// Archive operations for the file browser:
|
||||
//
|
||||
// - FsExtract : expand a zip / tar / tar.gz / tar.bz2 archive that
|
||||
// already lives inside the instance's volume.
|
||||
// - FsCompress : zip up one or more paths inside the instance into a
|
||||
// destination .zip in the same volume.
|
||||
// - FsRename : safe `mv` that respects the browseable_root sandbox.
|
||||
//
|
||||
// Every container-side path goes through the existing fs helper sidecar
|
||||
// (see getFsTargetContainerID) so these all work on stopped instances.
|
||||
//
|
||||
// Implementation notes:
|
||||
// - Extract reads the archive bytes via CopyFileFromContainer, walks
|
||||
// the entries with the appropriate stdlib reader, and re-emits a
|
||||
// single tar stream that's piped to CopyTarToContainer in one API
|
||||
// call. That avoids N per-file writes.
|
||||
// - Compress walks each source via CopyDirFromContainer (tar stream
|
||||
// from Docker), and re-encodes the entries as zip. The final zip
|
||||
// bytes go back via CopyFileToContainer.
|
||||
// - We support zip + tar + tar.gz + tar.bz2 + tar.xz + 7z + rar.
|
||||
// 7z and rar use pure-Go libraries (bodgit/sevenzip,
|
||||
// nwaples/rardecode/v2) so no native deps. Encrypted rars and
|
||||
// solid 7z volumes are surfaced as errors, not silently skipped.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/archive"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// ---- FsExtract ----
|
||||
|
||||
func (d *Dispatcher) handleFsExtract(corrID string, req *panelv1.FsExtractRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if req.ArchivePath == "" {
|
||||
d.sendFsExtractResult(corrID, 0, 0, "archive_path is required")
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
d.extractInContainer(corrID, rec, req)
|
||||
return
|
||||
}
|
||||
d.extractOnHost(corrID, rec, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) extractInContainer(corrID string, rec *instanceRecord, req *panelv1.FsExtractRequest) {
|
||||
archiveAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.ArchivePath)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
destRel := req.DestDir
|
||||
if destRel == "" {
|
||||
// Default: same dir as the archive.
|
||||
destRel = path.Dir(req.ArchivePath)
|
||||
}
|
||||
destAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), destRel)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Pull the archive bytes back from the container. CopyFileFromContainer
|
||||
// internally tar-walks Docker's response and returns just the file body.
|
||||
archiveData, err := d.runtime.CopyFileFromContainer(ctx, targetID, archiveAbs)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, fmt.Errorf("read archive: %w", err).Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure the destination dir exists before we ship the tar stream.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(destAbs)})
|
||||
|
||||
// Build a tar stream of the archive's contents and pipe it to
|
||||
// CopyTarToContainer. Doing this in a single round-trip keeps the
|
||||
// wall-time of "extract a 500 MB archive with 4000 entries" manageable.
|
||||
pr, pw := io.Pipe()
|
||||
emitErr := make(chan error, 1)
|
||||
var entries, bytesOut int64
|
||||
go func() {
|
||||
n, b, err := archive.WriteAsTar(archiveData, req.ArchivePath, pw)
|
||||
entries, bytesOut = n, b
|
||||
_ = pw.CloseWithError(err)
|
||||
emitErr <- err
|
||||
}()
|
||||
if err := d.runtime.CopyTarToContainer(ctx, targetID, destAbs, pr); err != nil {
|
||||
// Pipe gets closed with our error from the goroutine; whichever
|
||||
// side surfaces first wins. We prefer the goroutine error since
|
||||
// it carries archive-format detail.
|
||||
if encErr := <-emitErr; encErr != nil {
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, encErr.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, fmt.Errorf("copy to container: %w", err).Error())
|
||||
return
|
||||
}
|
||||
if encErr := <-emitErr; encErr != nil {
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, encErr.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
// extractOnHost handles the rare host-fallback case (no container, no
|
||||
// helper) — uses os primitives. Same path-safety contract as everywhere
|
||||
// else: must stay under the instance's data path.
|
||||
func (d *Dispatcher) extractOnHost(corrID string, rec *instanceRecord, req *panelv1.FsExtractRequest) {
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsExtractResult(corrID, 0, 0, "no file storage available")
|
||||
return
|
||||
}
|
||||
archiveAbs, err := safeJoinHost(rec.DataPath, req.ArchivePath)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
destRel := req.DestDir
|
||||
if destRel == "" {
|
||||
destRel = filepath.Dir(req.ArchivePath)
|
||||
}
|
||||
destAbs, err := safeJoinHost(rec.DataPath, destRel)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(destAbs, 0o755); err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
data, err := os.ReadFile(archiveAbs)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
entries, bytesOut, err := archive.Walk(data, req.ArchivePath, func(name string, mode int64, isDir bool, body io.Reader) error {
|
||||
safeName := archive.SafeEntryPath(name)
|
||||
if safeName == "" {
|
||||
return nil
|
||||
}
|
||||
out := filepath.Join(destAbs, filepath.FromSlash(safeName))
|
||||
if isDir {
|
||||
return os.MkdirAll(out, 0o755)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(mode&0o777))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(f, body); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
})
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsExtractResult(corrID string, entries, bytesOut int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsExtractResult{
|
||||
FsExtractResult: &panelv1.FsExtractResult{EntriesWritten: entries, BytesWritten: bytesOut, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsCompress ----
|
||||
|
||||
func (d *Dispatcher) handleFsCompress(corrID string, req *panelv1.FsCompressRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.Sources) == 0 {
|
||||
d.sendFsCompressResult(corrID, 0, 0, "at least one source path is required")
|
||||
return
|
||||
}
|
||||
if req.DestZipPath == "" {
|
||||
d.sendFsCompressResult(corrID, 0, 0, "dest_zip_path is required")
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
d.compressInContainer(corrID, rec, req)
|
||||
return
|
||||
}
|
||||
d.compressOnHost(corrID, rec, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) compressInContainer(corrID string, rec *instanceRecord, req *panelv1.FsCompressRequest) {
|
||||
destAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.DestZipPath)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var zipBuf bytes.Buffer
|
||||
zw := zip.NewWriter(&zipBuf)
|
||||
var entries, bytesOut int64
|
||||
|
||||
for _, src := range req.Sources {
|
||||
srcAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), src)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
// CopyDirFromContainer returns Docker's tar stream of srcAbs.
|
||||
// For files: a single-entry tar. For dirs: every file under it.
|
||||
// The base name of srcAbs is the prefix Docker uses, which we
|
||||
// keep in the zip so the user gets familiar names back.
|
||||
rc, err := d.runtime.CopyDirFromContainer(ctx, targetID, srcAbs)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, fmt.Errorf("copy from container %q: %w", src, err).Error())
|
||||
return
|
||||
}
|
||||
n, b, err := pipeTarIntoZip(rc, zw)
|
||||
_ = rc.Close()
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, entries+n, bytesOut+b, fmt.Errorf("encode %q: %w", src, err).Error())
|
||||
return
|
||||
}
|
||||
entries += n
|
||||
bytesOut += b
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, fmt.Errorf("zip close: %w", err).Error())
|
||||
return
|
||||
}
|
||||
// Drop the resulting zip into the volume.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(destAbs))})
|
||||
if err := d.runtime.CopyFileToContainer(ctx, targetID, destAbs, zipBuf.Bytes()); err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) compressOnHost(corrID string, rec *instanceRecord, req *panelv1.FsCompressRequest) {
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsCompressResult(corrID, 0, 0, "no file storage available")
|
||||
return
|
||||
}
|
||||
destAbs, err := safeJoinHost(rec.DataPath, req.DestZipPath)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(destAbs), 0o755); err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
f, err := os.Create(destAbs)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
zw := zip.NewWriter(f)
|
||||
var entries, bytesOut int64
|
||||
for _, src := range req.Sources {
|
||||
srcAbs, err := safeJoinHost(rec.DataPath, src)
|
||||
if err != nil {
|
||||
zw.Close()
|
||||
f.Close()
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
base := filepath.Base(srcAbs)
|
||||
err = filepath.Walk(srcAbs, func(p string, info os.FileInfo, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
rel, err := filepath.Rel(srcAbs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
zipName := base
|
||||
if rel != "." {
|
||||
zipName = filepath.ToSlash(filepath.Join(base, rel))
|
||||
}
|
||||
if info.IsDir() {
|
||||
if zipName == "" || zipName == "." {
|
||||
return nil
|
||||
}
|
||||
_, err = zw.Create(zipName + "/")
|
||||
return err
|
||||
}
|
||||
zf, err := zw.Create(zipName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rf, err := os.Open(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rf.Close()
|
||||
n, err := io.Copy(zf, rf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries++
|
||||
bytesOut += n
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
zw.Close()
|
||||
f.Close()
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
f.Close()
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
// pipeTarIntoZip walks a docker-style tar stream and writes each
|
||||
// regular-file entry into the zip writer, preserving relative paths.
|
||||
// Directory entries are written as zero-byte zip directory markers.
|
||||
func pipeTarIntoZip(r io.Reader, zw *zip.Writer) (int64, int64, error) {
|
||||
tr := tar.NewReader(r)
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
safe := archive.SafeEntryPath(hdr.Name)
|
||||
if safe == "" {
|
||||
continue
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if _, err := zw.Create(safe + "/"); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
zf, err := zw.Create(safe)
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
n, err := io.Copy(zf, tr)
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
bytesOut += n
|
||||
totalBytes += n
|
||||
if totalBytes > archive.MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("compress: total uncompressed bytes exceed %d", archive.MaxTotalBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsCompressResult(corrID string, entries, bytesOut int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsCompressResult{
|
||||
FsCompressResult: &panelv1.FsCompressResult{EntriesWritten: entries, BytesWritten: bytesOut, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsRename ----
|
||||
|
||||
func (d *Dispatcher) handleFsRename(corrID string, req *panelv1.FsRenameRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if req.FromPath == "" || req.ToPath == "" {
|
||||
d.sendFsRenameResult(corrID, "from_path and to_path are required")
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
fromAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.FromPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
toAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.ToPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if fromAbs == rec.BrowseableRoot {
|
||||
d.sendFsRenameResult(corrID, "refusing to rename instance root")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
// mkdir parent so a "rename to subdir" implicitly creates it.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(toAbs))})
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mv -- " + shellQuote(fromAbs) + " " + shellQuote(toAbs)})
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("mv exited %d", code)
|
||||
}
|
||||
d.sendFsRenameResult(corrID, msg)
|
||||
return
|
||||
}
|
||||
d.sendFsRenameResult(corrID, "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsRenameResult(corrID, "no file storage available")
|
||||
return
|
||||
}
|
||||
fromAbs, err := safeJoinHost(rec.DataPath, req.FromPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
toAbs, err := safeJoinHost(rec.DataPath, req.ToPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if fromAbs == rec.DataPath {
|
||||
d.sendFsRenameResult(corrID, "refusing to rename instance root")
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(toAbs), 0o755); err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.Rename(fromAbs, toAbs); err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsRenameResult(corrID, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsRenameResult(corrID, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsRenameResult{
|
||||
FsRenameResult: &panelv1.FsRenameResult{Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package dispatch
|
||||
|
||||
// Chunked file upload — solves three problems at once:
|
||||
//
|
||||
// 1. Cloudflare's request-body cap (~100 MB free, 200 MB pro) silently
|
||||
// stalls "single-shot" uploads of large mod / scenario archives
|
||||
// that go through the panel's public hostname.
|
||||
// 2. The previous single-shot path buffered the entire body in memory
|
||||
// on both the controller and the agent, peaking at ~3× file size.
|
||||
// A 3 GB upload would thrash a smaller box.
|
||||
// 3. Single-shot uploads block the agent's bidi stream for the
|
||||
// duration of the transfer; chunks are small and interleaved.
|
||||
//
|
||||
// Wire protocol (all on the existing AgentEnvelope):
|
||||
//
|
||||
// FsWriteChunkRequest { upload_id, instance_id, path, offset, data,
|
||||
// total_size, is_final }
|
||||
// FsWriteChunkResult { bytes_received, total_received, finalized,
|
||||
// error }
|
||||
//
|
||||
// The agent owns one `uploadSession` per upload_id. Each session has an
|
||||
// `*os.File` open in its scratch dir. Chunks append in order. On
|
||||
// is_final the agent ships the file to the container via a streaming
|
||||
// CopyTarToContainer call (no full-file in-memory copy), then unlinks.
|
||||
// Stale sessions are reaped after uploadIdleTTL.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
const uploadIdleTTL = 30 * time.Minute
|
||||
|
||||
type uploadSession struct {
|
||||
mu sync.Mutex
|
||||
uploadID string
|
||||
instanceID string
|
||||
path string
|
||||
tempPath string
|
||||
f *os.File
|
||||
bytes int64
|
||||
total int64
|
||||
lastTouch time.Time
|
||||
}
|
||||
|
||||
func (d *Dispatcher) uploadScratchDir() string {
|
||||
dir := filepath.Join(d.dataRoot, ".panel-uploads")
|
||||
_ = os.MkdirAll(dir, 0o755)
|
||||
return dir
|
||||
}
|
||||
|
||||
// handleFsWriteChunk processes one chunk in the streaming-upload protocol.
|
||||
// First chunk creates the session; subsequent chunks append; is_final
|
||||
// triggers a streaming copy into the container and cleans up.
|
||||
func (d *Dispatcher) handleFsWriteChunk(corrID string, req *panelv1.FsWriteChunkRequest) {
|
||||
if req.UploadId == "" {
|
||||
d.sendChunkResult(corrID, 0, 0, false, "upload_id is required")
|
||||
return
|
||||
}
|
||||
if req.InstanceId == "" || req.Path == "" {
|
||||
d.sendChunkResult(corrID, 0, 0, false, "instance_id and path are required")
|
||||
return
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendChunkResult(corrID, 0, 0, false, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
sess, ok := d.uploads[req.UploadId]
|
||||
if !ok {
|
||||
// First chunk for this upload — create session + temp file.
|
||||
// Use the upload_id in the filename so a recovered scratch dir
|
||||
// is self-describing for forensic purposes.
|
||||
base := filepath.Join(d.uploadScratchDir(), "u-"+sanitizeUploadID(req.UploadId))
|
||||
f, err := os.OpenFile(base, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
d.mu.Unlock()
|
||||
d.sendChunkResult(corrID, 0, 0, false, fmt.Errorf("open temp: %w", err).Error())
|
||||
return
|
||||
}
|
||||
sess = &uploadSession{
|
||||
uploadID: req.UploadId,
|
||||
instanceID: req.InstanceId,
|
||||
path: req.Path,
|
||||
tempPath: base,
|
||||
f: f,
|
||||
total: req.TotalSize,
|
||||
lastTouch: time.Now(),
|
||||
}
|
||||
d.uploads[req.UploadId] = sess
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
sess.mu.Lock()
|
||||
defer sess.mu.Unlock()
|
||||
|
||||
// Reject stray chunks for a session that's already closed (final
|
||||
// chunk processed earlier — operator double-clicked, retry, etc.).
|
||||
if sess.f == nil {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, true, "")
|
||||
return
|
||||
}
|
||||
|
||||
// Allow operator to upload to a renamed path mid-stream? No — pin
|
||||
// the destination from the first chunk for safety. Subsequent
|
||||
// chunks with a different path get rejected.
|
||||
if sess.path != req.Path {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, false, "path differs from initial chunk")
|
||||
return
|
||||
}
|
||||
|
||||
// Append the chunk. The browser is expected to send chunks in
|
||||
// order; if offset doesn't match the running tail we err so the
|
||||
// frontend can fall back / retry rather than silently writing a
|
||||
// hole.
|
||||
if req.Offset != sess.bytes {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, false,
|
||||
fmt.Sprintf("chunk offset %d != expected %d (out-of-order chunk)", req.Offset, sess.bytes))
|
||||
return
|
||||
}
|
||||
n, err := sess.f.Write(req.Data)
|
||||
if err != nil {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, false, fmt.Errorf("write chunk: %w", err).Error())
|
||||
return
|
||||
}
|
||||
sess.bytes += int64(n)
|
||||
sess.lastTouch = time.Now()
|
||||
|
||||
if !req.IsFinal {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, "")
|
||||
return
|
||||
}
|
||||
|
||||
// Final chunk — close the temp file, ship it to the container.
|
||||
if err := sess.f.Sync(); err != nil {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, fmt.Errorf("fsync: %w", err).Error())
|
||||
return
|
||||
}
|
||||
if err := sess.f.Close(); err != nil {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, fmt.Errorf("close temp: %w", err).Error())
|
||||
return
|
||||
}
|
||||
sess.f = nil
|
||||
if sess.total > 0 && sess.bytes != sess.total {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false,
|
||||
fmt.Sprintf("size mismatch: received %d bytes, expected %d", sess.bytes, sess.total))
|
||||
return
|
||||
}
|
||||
|
||||
// Ship to the container (or host fallback) without re-reading the
|
||||
// whole file into memory — we hand Docker an io.Reader that
|
||||
// streams from disk, wrapped in a tar header on the fly.
|
||||
if err := d.deliverUploadedFile(rec, sess); err != nil {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Tear down the session — temp file deleted, map slot freed.
|
||||
_ = os.Remove(sess.tempPath)
|
||||
d.mu.Lock()
|
||||
delete(d.uploads, sess.uploadID)
|
||||
d.mu.Unlock()
|
||||
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, true, "")
|
||||
}
|
||||
|
||||
// deliverUploadedFile streams the temp file into the destination,
|
||||
// either via container CopyTarToContainer (running or helper sidecar)
|
||||
// or via a plain os.Rename for host-fallback instances. Designed so
|
||||
// the file's bytes never sit in memory in their entirety.
|
||||
func (d *Dispatcher) deliverUploadedFile(rec *instanceRecord, sess *uploadSession) error {
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), sess.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// mkdir -p the parent so the destination always exists.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(abs))})
|
||||
|
||||
// Open the temp file for read; build a tar wrapper on the fly
|
||||
// and feed Docker. CopyTarToContainer streams the body without
|
||||
// materializing it in memory.
|
||||
fr, err := os.Open(sess.tempPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fr.Close()
|
||||
fi, err := fr.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
tw := tar.NewWriter(pw)
|
||||
err := tw.WriteHeader(&tar.Header{
|
||||
Name: path.Base(abs),
|
||||
Mode: 0o644,
|
||||
Size: fi.Size(),
|
||||
})
|
||||
if err == nil {
|
||||
_, err = io.Copy(tw, fr)
|
||||
}
|
||||
if err == nil {
|
||||
err = tw.Close()
|
||||
}
|
||||
_ = pw.CloseWithError(err)
|
||||
errCh <- err
|
||||
}()
|
||||
if err := d.runtime.CopyTarToContainer(ctx, targetID, path.Dir(abs), pr); err != nil {
|
||||
if encErr := <-errCh; encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return fmt.Errorf("copy to container: %w", err)
|
||||
}
|
||||
if encErr := <-errCh; encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Host fallback — just move the temp file into place.
|
||||
if rec.DataPath == "" {
|
||||
return fmt.Errorf("no file storage available")
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, sess.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
// os.Rename is atomic on the same filesystem; if scratch lives on
|
||||
// a different fs we fall back to copy + delete.
|
||||
if err := os.Rename(sess.tempPath, abs); err == nil {
|
||||
return nil
|
||||
}
|
||||
src, err := os.Open(sess.tempPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
dst, err := os.OpenFile(abs, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadReaper closes + removes any session that hasn't been touched
|
||||
// in uploadIdleTTL. Runs forever; cheap enough to scan once a minute.
|
||||
func (d *Dispatcher) uploadReaper() {
|
||||
t := time.NewTicker(time.Minute)
|
||||
defer t.Stop()
|
||||
for range t.C {
|
||||
now := time.Now()
|
||||
d.mu.Lock()
|
||||
stale := []*uploadSession{}
|
||||
for id, s := range d.uploads {
|
||||
if now.Sub(s.lastTouch) > uploadIdleTTL {
|
||||
stale = append(stale, s)
|
||||
delete(d.uploads, id)
|
||||
}
|
||||
}
|
||||
d.mu.Unlock()
|
||||
for _, s := range stale {
|
||||
s.mu.Lock()
|
||||
if s.f != nil {
|
||||
_ = s.f.Close()
|
||||
s.f = nil
|
||||
}
|
||||
_ = os.Remove(s.tempPath)
|
||||
s.mu.Unlock()
|
||||
d.log.Info("upload session reaped", "upload_id", s.uploadID, "bytes", s.bytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sanitizeUploadID strips characters that could escape the scratch dir.
|
||||
// upload_id is generated by the browser (UUID-like) — defense in depth.
|
||||
func sanitizeUploadID(s string) string {
|
||||
out := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case (c >= 'a' && c <= 'z'), (c >= 'A' && c <= 'Z'), (c >= '0' && c <= '9'), c == '-', c == '_':
|
||||
out = append(out, c)
|
||||
}
|
||||
if len(out) > 64 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = []byte("anon")
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendChunkResult(corrID string, n, total int64, finalized bool, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsWriteChunkResult{
|
||||
FsWriteChunkResult: &panelv1.FsWriteChunkResult{
|
||||
BytesReceived: n,
|
||||
TotalReceived: total,
|
||||
Finalized: finalized,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// hangGuard implements a belt-and-suspenders hang-detection guardrail for
|
||||
// ARK Survival Ascended (ark-sa) instances running under Wine 9.
|
||||
// The Wine 9 RCON listener can wedge under sustained traffic; the engine
|
||||
// also sometimes emits a "!!!HANG DETECTED!!!" line on its own.
|
||||
// This guard auto-restarts the docker container in either case, with a
|
||||
// cooldown to prevent flapping.
|
||||
// The root-cause fix (docker_exec_rcon adapter) shipped 2026-05-03; this
|
||||
// guard handles the rare residual cases.
|
||||
// It replaces the bash watchdog that previously ran on princess via systemd timer.
|
||||
|
||||
const (
|
||||
hangGuardWindow = 10 * time.Minute
|
||||
hangGuardFailThreshold = 5
|
||||
hangGuardCooldown = 10 * time.Minute
|
||||
hangGuardEngineMarker = "!!!HANG DETECTED!!!"
|
||||
)
|
||||
|
||||
// hangGuardState holds per-instance hang-detection state.
|
||||
// Embed by value in instanceRecord; zero value is ready to use.
|
||||
type hangGuardState struct {
|
||||
mu sync.Mutex
|
||||
consecutiveFails int
|
||||
windowStart time.Time
|
||||
lastRestart time.Time
|
||||
restarting atomic.Bool
|
||||
}
|
||||
|
||||
// arkHangGuardOnLogLine checks for engine-side hang detection tokens
|
||||
// in console log lines. Only applies to ark-sa instances.
|
||||
func (d *Dispatcher) arkHangGuardOnLogLine(rec *instanceRecord, line string) {
|
||||
if rec.ModuleID != "ark-sa" {
|
||||
return
|
||||
}
|
||||
if !strings.Contains(line, hangGuardEngineMarker) {
|
||||
return
|
||||
}
|
||||
// Engine self-detected a hang; trigger restart asynchronously.
|
||||
go d.arkHangGuardRestart(rec, fmt.Sprintf("engine emitted '%s'", hangGuardEngineMarker))
|
||||
}
|
||||
|
||||
// arkHangGuardOnRconResult tracks RCON call successes/failures for
|
||||
// hang detection. A threshold of consecutive failures triggers a restart.
|
||||
func (d *Dispatcher) arkHangGuardOnRconResult(rec *instanceRecord, success bool) {
|
||||
if rec.ModuleID != "ark-sa" {
|
||||
return
|
||||
}
|
||||
|
||||
rec.hangGuard.mu.Lock()
|
||||
defer rec.hangGuard.mu.Unlock()
|
||||
|
||||
if success {
|
||||
rec.hangGuard.consecutiveFails = 0
|
||||
rec.hangGuard.windowStart = time.Time{}
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if rec.hangGuard.windowStart.IsZero() || now.Sub(rec.hangGuard.windowStart) > hangGuardWindow {
|
||||
// Start a new window
|
||||
rec.hangGuard.windowStart = now
|
||||
rec.hangGuard.consecutiveFails = 1
|
||||
} else {
|
||||
rec.hangGuard.consecutiveFails++
|
||||
}
|
||||
|
||||
if rec.hangGuard.consecutiveFails >= hangGuardFailThreshold {
|
||||
go d.arkHangGuardRestart(rec, fmt.Sprintf("%d RCON failures in last %v", rec.hangGuard.consecutiveFails, hangGuardWindow))
|
||||
}
|
||||
}
|
||||
|
||||
// arkHangGuardRestart performs the actual container restart, respecting a
|
||||
// cooldown and using a single-flight pattern to prevent concurrent restarts.
|
||||
func (d *Dispatcher) arkHangGuardRestart(rec *instanceRecord, reason string) {
|
||||
// Single-flight: only one restart at a time per instance.
|
||||
if !rec.hangGuard.restarting.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
defer rec.hangGuard.restarting.Store(false)
|
||||
|
||||
// Lock only for cooldown check and counter reset. We do NOT set
|
||||
// lastRestart here — only after a successful runtime.Restart, so a
|
||||
// failed restart attempt doesn't lock out retries for the full
|
||||
// cooldown (would block the next legitimate hang signal). Resetting
|
||||
// consecutiveFails / windowStart up front is fine: the bounce is
|
||||
// either about to happen (success path will set lastRestart) or has
|
||||
// failed (caller logged + emitted; future signals will re-arm).
|
||||
rec.hangGuard.mu.Lock()
|
||||
if !rec.hangGuard.lastRestart.IsZero() && time.Since(rec.hangGuard.lastRestart) < hangGuardCooldown {
|
||||
remaining := hangGuardCooldown - time.Since(rec.hangGuard.lastRestart)
|
||||
d.log.Debug("hang-guard skipped (cooldown)",
|
||||
"instance_id", rec.InstanceID,
|
||||
"cooldown_remaining", remaining,
|
||||
)
|
||||
rec.hangGuard.mu.Unlock()
|
||||
return
|
||||
}
|
||||
rec.hangGuard.consecutiveFails = 0
|
||||
rec.hangGuard.windowStart = time.Time{}
|
||||
rec.hangGuard.mu.Unlock()
|
||||
|
||||
d.log.Warn("hang-guard restarting container",
|
||||
"instance_id", rec.InstanceID,
|
||||
"container_id", rec.ContainerID,
|
||||
"reason", reason,
|
||||
)
|
||||
|
||||
// Emit a panel log line announcing the restart.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] HANG-GUARD: " + reason,
|
||||
}},
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 2026-05-25: bumped grace 5s → 30s. The hang-guard's bounce can
|
||||
// fire mid-SaveWorld; the 5s window made it a near-guarantee that the
|
||||
// SIGKILL would land inside the SQLite-backed save's flush — same
|
||||
// torn-save pattern that the memory guardrail was producing. 30s lets
|
||||
// most in-flight saves finish before the engine dies; if the engine is
|
||||
// truly wedged it'll get SIGKILL'd after the 30s anyway. See
|
||||
// stats.go's emergencyStopGracePeriod for the matching rationale.
|
||||
if err := d.runtime.Restart(ctx, rec.ContainerID, 30*time.Second); err != nil {
|
||||
d.log.Error("hang-guard restart failed",
|
||||
"instance_id", rec.InstanceID,
|
||||
"err", err,
|
||||
)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] HANG-GUARD restart failed: " + err.Error(),
|
||||
}},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Successful restart — start the cooldown clock so we don't bounce
|
||||
// the same instance again for hangGuardCooldown. A failed restart
|
||||
// (above branch returned early) intentionally leaves lastRestart
|
||||
// untouched so the next hang signal can retry.
|
||||
rec.hangGuard.mu.Lock()
|
||||
rec.hangGuard.lastRestart = time.Now()
|
||||
rec.hangGuard.mu.Unlock()
|
||||
|
||||
d.log.Info("hang-guard restart issued",
|
||||
"instance_id", rec.InstanceID,
|
||||
)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] HANG-GUARD restart issued",
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package dispatch
|
||||
|
||||
// Tests for WI-05: heartbeats routed through the send queue (sendLoop as the
|
||||
// sole owner of stream.Send) and the single-flight, ctx-bound exit watcher.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// recordingSender captures every envelope the sendLoop delivers.
|
||||
type recordingSender struct {
|
||||
mu sync.Mutex
|
||||
envs []*panelv1.AgentEnvelope
|
||||
}
|
||||
|
||||
func (r *recordingSender) Send(e *panelv1.AgentEnvelope) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.envs = append(r.envs, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingSender) snapshot() []*panelv1.AgentEnvelope {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
out := make([]*panelv1.AgentEnvelope, len(r.envs))
|
||||
copy(out, r.envs)
|
||||
return out
|
||||
}
|
||||
|
||||
// TestSendHeartbeatGoesThroughQueue verifies SendHeartbeat enqueues onto
|
||||
// sendCh and the envelope reaches the sender via sendLoop — i.e. heartbeats
|
||||
// no longer need a direct stream.Send from the session goroutine.
|
||||
func TestSendHeartbeatGoesThroughQueue(t *testing.T) {
|
||||
d := newTestDispatcher()
|
||||
rs := &recordingSender{}
|
||||
d.SetSender(rs)
|
||||
|
||||
now := time.Now()
|
||||
d.SendHeartbeat(now)
|
||||
|
||||
deadline := time.After(2 * time.Second)
|
||||
for {
|
||||
for _, e := range rs.snapshot() {
|
||||
if hb := e.GetHeartbeat(); hb != nil {
|
||||
if got := hb.At.AsTime(); !got.Equal(now.UTC().Truncate(time.Nanosecond)) && got.Unix() != now.Unix() {
|
||||
t.Fatalf("heartbeat timestamp mismatch: got %v want %v", got, now)
|
||||
}
|
||||
return // delivered through the queue — pass
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("heartbeat never delivered through sendLoop")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeatIsDroppable pins the backpressure classification: a heartbeat
|
||||
// must be shed (not block the producer) when the buffer is full.
|
||||
func TestHeartbeatIsDroppable(t *testing.T) {
|
||||
env := &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_Heartbeat{Heartbeat: &panelv1.Heartbeat{}}}
|
||||
if !isDroppableEnvelope(env) {
|
||||
t.Fatal("heartbeat envelope must be droppable telemetry")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeExitRuntime implements just enough of runtime.Runtime for watchExit.
|
||||
// The embedded nil interface panics on any unexpected call — that's a test
|
||||
// failure signal, not a hazard.
|
||||
type fakeExitRuntime struct {
|
||||
runtime.Runtime
|
||||
mu sync.Mutex
|
||||
activeWaits int
|
||||
maxWaits int
|
||||
release chan struct{} // close to make Wait return exit code 0
|
||||
}
|
||||
|
||||
func (f *fakeExitRuntime) Wait(ctx context.Context, id string) (int, error) {
|
||||
f.mu.Lock()
|
||||
f.activeWaits++
|
||||
if f.activeWaits > f.maxWaits {
|
||||
f.maxWaits = f.activeWaits
|
||||
}
|
||||
f.mu.Unlock()
|
||||
defer func() {
|
||||
f.mu.Lock()
|
||||
f.activeWaits--
|
||||
f.mu.Unlock()
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return -1, ctx.Err()
|
||||
case <-f.release:
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeExitRuntime) InspectByName(ctx context.Context, name string) (runtime.RuntimeInstanceState, error) {
|
||||
return runtime.RuntimeInstanceState{Status: "exited"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeExitRuntime) counts() (active, max int) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.activeWaits, f.maxWaits
|
||||
}
|
||||
|
||||
func newExitWatchDispatcher(f *fakeExitRuntime) *Dispatcher {
|
||||
d := &Dispatcher{
|
||||
log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
runtime: f,
|
||||
instances: map[string]*instanceRecord{},
|
||||
sendCh: make(chan *panelv1.AgentEnvelope, 256),
|
||||
}
|
||||
go d.sendLoop()
|
||||
return d
|
||||
}
|
||||
|
||||
// waitFor polls cond until true or the deadline elapses.
|
||||
func waitFor(t *testing.T, d time.Duration, cond func() bool, msg string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(d)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal(msg)
|
||||
}
|
||||
|
||||
// TestStartExitWatchSingleFlight re-arms the watcher many times (the shape of
|
||||
// Announce firing on every controller reconnect) and asserts only ONE
|
||||
// ContainerWait is ever in flight — the pre-fix behavior leaked one blocked
|
||||
// Wait (and its docker connection) per reconnect.
|
||||
func TestStartExitWatchSingleFlight(t *testing.T) {
|
||||
f := &fakeExitRuntime{release: make(chan struct{})}
|
||||
d := newExitWatchDispatcher(f)
|
||||
rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
|
||||
|
||||
for i := 0; i < 8; i++ {
|
||||
d.startExitWatch(rec)
|
||||
}
|
||||
// All superseded watchers must exit; exactly one survivor remains.
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 },
|
||||
"expected exactly 1 in-flight ContainerWait after re-arms")
|
||||
// The peak may briefly exceed 1 while a cancelled watcher unwinds, but it
|
||||
// must never approach the number of re-arms (pre-fix: 8 concurrent Waits).
|
||||
if _, max := f.counts(); max >= 8 {
|
||||
t.Fatalf("watchExit not single-flighted: peak concurrent waits = %d", max)
|
||||
}
|
||||
|
||||
// Cancelling the record's watcher (handleStop / Shutdown path) must
|
||||
// release the last Wait.
|
||||
d.mu.Lock()
|
||||
cancel := rec.ExitWatchCancel
|
||||
rec.ExitWatchCancel = nil
|
||||
d.mu.Unlock()
|
||||
cancel()
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 0 },
|
||||
"cancelled exit watcher did not release ContainerWait")
|
||||
}
|
||||
|
||||
// TestWatchExitCancelledEmitsNothing: a ctx-cancelled watcher must return
|
||||
// silently — no spurious stopped/crashed InstanceState for a container that
|
||||
// is still running.
|
||||
func TestWatchExitCancelledEmitsNothing(t *testing.T) {
|
||||
f := &fakeExitRuntime{release: make(chan struct{})}
|
||||
d := newExitWatchDispatcher(f)
|
||||
rs := &recordingSender{}
|
||||
d.SetSender(rs)
|
||||
rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
|
||||
|
||||
d.startExitWatch(rec)
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 }, "watcher never started")
|
||||
d.mu.Lock()
|
||||
cancel := rec.ExitWatchCancel
|
||||
d.mu.Unlock()
|
||||
cancel()
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 0 }, "watcher never exited")
|
||||
time.Sleep(50 * time.Millisecond) // allow any (wrong) emission to flush
|
||||
for _, e := range rs.snapshot() {
|
||||
if e.GetInstanceState() != nil {
|
||||
t.Fatalf("cancelled watchExit emitted InstanceState: %v", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchExitTerminalExit: when the container really exits (Wait returns
|
||||
// 0 and inspect says "exited"), the watcher emits a STOPPED state.
|
||||
func TestWatchExitTerminalExit(t *testing.T) {
|
||||
f := &fakeExitRuntime{release: make(chan struct{})}
|
||||
d := newExitWatchDispatcher(f)
|
||||
rs := &recordingSender{}
|
||||
d.SetSender(rs)
|
||||
rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
|
||||
d.instances["x"] = rec
|
||||
|
||||
d.startExitWatch(rec)
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 }, "watcher never started")
|
||||
close(f.release)
|
||||
waitFor(t, 2*time.Second, func() bool {
|
||||
for _, e := range rs.snapshot() {
|
||||
if st := e.GetInstanceState(); st != nil && st.Status == panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}, "terminal exit never produced a STOPPED InstanceState")
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
)
|
||||
|
||||
// logCoalescer throttles repetitive progress-style log lines while letting
|
||||
// genuine content through unchanged.
|
||||
//
|
||||
// Why this exists: containers under install/update pressure (SteamCMD
|
||||
// download ticks, preallocation, LGSM step banners) can emit 40+ lines per
|
||||
// second. Forwarding every one to the controller saturates the gRPC send
|
||||
// window, fills the event bus's per-subscriber buffer (which silently drops
|
||||
// on overflow), and burns CPU re-rendering DOM. The browser only needs to
|
||||
// see the *latest* progress value, not every interstitial frame.
|
||||
//
|
||||
// Behaviour:
|
||||
// - Lines with progressKey() == "": emit immediately.
|
||||
// - Lines with a key: if it's been less than `interval` since the last
|
||||
// emit of that key, replace the currently-buffered line for that key.
|
||||
// A single background timer flushes any pending lines every `interval`.
|
||||
type logCoalescer struct {
|
||||
interval time.Duration
|
||||
emit func(stream runtime.LogStream, line string, at time.Time)
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[string]*pendingLine
|
||||
lastAt map[string]time.Time
|
||||
timer *time.Timer
|
||||
stopped bool
|
||||
}
|
||||
|
||||
type pendingLine struct {
|
||||
stream runtime.LogStream
|
||||
line string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func newLogCoalescer(interval time.Duration, emit func(stream runtime.LogStream, line string, at time.Time)) *logCoalescer {
|
||||
c := &logCoalescer{
|
||||
interval: interval,
|
||||
emit: emit,
|
||||
pending: map[string]*pendingLine{},
|
||||
lastAt: map[string]time.Time{},
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// offer delivers a line to the coalescer. Safe to call concurrently, though
|
||||
// in practice the caller is the single log-scan goroutine.
|
||||
func (c *logCoalescer) offer(stream runtime.LogStream, line string, at time.Time) {
|
||||
key := progressKey(line)
|
||||
if key == "" {
|
||||
// Non-progress — emit immediately, but first flush any buffered
|
||||
// progress lines so they don't appear AFTER a later non-progress line.
|
||||
c.mu.Lock()
|
||||
c.flushLocked()
|
||||
c.mu.Unlock()
|
||||
c.emit(stream, line, at)
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.stopped {
|
||||
return
|
||||
}
|
||||
last := c.lastAt[key]
|
||||
if last.IsZero() || at.Sub(last) >= c.interval {
|
||||
// Enough quiet time — emit right away.
|
||||
c.emit(stream, line, at)
|
||||
c.lastAt[key] = at
|
||||
delete(c.pending, key)
|
||||
return
|
||||
}
|
||||
// Replace the buffered line for this key (we only care about the newest).
|
||||
c.pending[key] = &pendingLine{stream: stream, line: line, at: at}
|
||||
// Make sure the timer is armed. Single timer drains all pending keys.
|
||||
if c.timer == nil {
|
||||
c.timer = time.AfterFunc(c.interval, c.tick)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *logCoalescer) tick() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.flushLocked()
|
||||
// If more pending arrived during flush (or we expect more), re-arm the timer.
|
||||
if len(c.pending) > 0 && !c.stopped {
|
||||
c.timer = time.AfterFunc(c.interval, c.tick)
|
||||
} else {
|
||||
c.timer = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *logCoalescer) flushLocked() {
|
||||
now := time.Now()
|
||||
for key, p := range c.pending {
|
||||
c.emit(p.stream, p.line, p.at)
|
||||
c.lastAt[key] = now
|
||||
delete(c.pending, key)
|
||||
}
|
||||
}
|
||||
|
||||
// stop flushes any pending lines and prevents further work. Call from the
|
||||
// streamLogs goroutine's defer so a stopping instance doesn't leave buffered
|
||||
// progress frames undelivered.
|
||||
func (c *logCoalescer) stop() {
|
||||
c.mu.Lock()
|
||||
c.stopped = true
|
||||
if c.timer != nil {
|
||||
c.timer.Stop()
|
||||
c.timer = nil
|
||||
}
|
||||
c.flushLocked()
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// Regexes for lines we treat as part of a repeating progress stream.
|
||||
// Each group returns a stable key — same key = same progress counter.
|
||||
var (
|
||||
// SteamCMD: "Update state (0x61) downloading, progress: 47.95 (...)"
|
||||
// Also covers "verifying", "preallocating", "committing" stages.
|
||||
reSteamcmdProgress = regexp.MustCompile(`Update state \([^)]+\) (\w+), progress:`)
|
||||
// Generic "Progress: 64%" or "[ 64%] Installing ...".
|
||||
reGenericProgress = regexp.MustCompile(`^\s*(?:Progress:|\[\s*\d+%\s*\])`)
|
||||
// LGSM step banner: "Starting sdtdserver: <step>". vinanrra emits
|
||||
// several per step as the banner animates ([ .... ] → [ INFO ] → [ OK ]).
|
||||
reLGSMStarting = regexp.MustCompile(`^\s*Starting sdtdserver:`)
|
||||
reLGSMStopping = regexp.MustCompile(`^\s*Stopping sdtdserver:`)
|
||||
// Docker pull: "Downloading [=>] 12.3MB / 100MB"
|
||||
reDockerPull = regexp.MustCompile(`^\s*(?:Downloading|Extracting|Pulling)\s+[\[(]`)
|
||||
)
|
||||
|
||||
// progressKey returns a stable identifier for a progress-style line, or ""
|
||||
// if the line should be forwarded unchanged. The returned key collapses
|
||||
// every frame of the same counter onto a single slot so we keep only the
|
||||
// latest within each coalesce window.
|
||||
func progressKey(text string) string {
|
||||
if m := reSteamcmdProgress.FindStringSubmatch(text); m != nil {
|
||||
return "steamcmd:" + m[1]
|
||||
}
|
||||
if reGenericProgress.MatchString(text) {
|
||||
return "progress"
|
||||
}
|
||||
if reLGSMStarting.MatchString(text) {
|
||||
return "lgsm-starting"
|
||||
}
|
||||
if reLGSMStopping.MatchString(text) {
|
||||
return "lgsm-stopping"
|
||||
}
|
||||
if reDockerPull.MatchString(text) {
|
||||
return "docker-pull"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// instanceMeta is the on-disk sidecar written per instance after successful
|
||||
// Create. It lets the agent rehydrate its in-memory records after a restart,
|
||||
// so operator commands (stop, etc.) continue to work against instances that
|
||||
// outlived the agent process.
|
||||
//
|
||||
// Stored at <metaDir>/<instance_id>.json. Mode 0600 because it carries the
|
||||
// RCON password. Deleted on stop.
|
||||
type instanceMeta struct {
|
||||
InstanceID string `json:"instance_id"`
|
||||
ContainerID string `json:"container_id"`
|
||||
ModuleID string `json:"module_id"`
|
||||
// Branch is the normalized Steam branch the install came from (warm-seed
|
||||
// matching key). Empty on sidecars written before this feature.
|
||||
Branch string `json:"branch,omitempty"`
|
||||
DataPath string `json:"data_path"`
|
||||
BrowseableRoot string `json:"browseable_root,omitempty"`
|
||||
RCONAddr string `json:"rcon_addr,omitempty"`
|
||||
RCONPassword string `json:"rcon_password,omitempty"`
|
||||
ConfigValues map[string]string `json:"config_values,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (d *Dispatcher) metaPath(instanceID string) string {
|
||||
return filepath.Join(d.metaDir, instanceID+".json")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) writeMeta(rec *instanceRecord) error {
|
||||
if d.metaDir == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(d.metaDir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir meta dir: %w", err)
|
||||
}
|
||||
m := instanceMeta{
|
||||
InstanceID: rec.InstanceID,
|
||||
ContainerID: rec.ContainerID,
|
||||
ModuleID: rec.ModuleID,
|
||||
Branch: rec.Branch,
|
||||
DataPath: rec.DataPath,
|
||||
BrowseableRoot: rec.BrowseableRoot,
|
||||
RCONAddr: rec.RCONAddr,
|
||||
RCONPassword: rec.RCONPassword,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
data, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal meta: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(d.metaPath(rec.InstanceID), data, 0o600); err != nil {
|
||||
return fmt.Errorf("write meta: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) deleteMeta(instanceID string) {
|
||||
if d.metaDir == "" {
|
||||
return
|
||||
}
|
||||
if err := os.Remove(d.metaPath(instanceID)); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
d.log.Warn("delete meta failed", "instance_id", instanceID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) loadAllMeta() ([]instanceMeta, error) {
|
||||
if d.metaDir == "" {
|
||||
return nil, nil
|
||||
}
|
||||
entries, err := os.ReadDir(d.metaDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var out []instanceMeta
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
if filepath.Ext(name) != ".json" {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(d.metaDir, name))
|
||||
if err != nil {
|
||||
d.log.Warn("read meta failed", "file", name, "err", err)
|
||||
continue
|
||||
}
|
||||
var m instanceMeta
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
d.log.Warn("bad meta file", "file", name, "err", err)
|
||||
continue
|
||||
}
|
||||
if m.InstanceID == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// seedModsFromInstance clones the Mods/ folder of srcInstanceID (the cluster
|
||||
// master) onto dstInstanceID's install volume, REPLACING the destination's mod
|
||||
// set with the master's. Runs a throwaway debian sidecar that mounts both
|
||||
// instances' install volumes and tar-copies /src-game/Mods → /dst-game/Mods.
|
||||
//
|
||||
// This is the inverse of tryWarmSeedFromSibling (which copies the install but
|
||||
// EXCLUDES Mods): here we copy ONLY Mods, so a freshly-joined cluster member
|
||||
// ends up running exactly the master's mod set without an operator re-uploading
|
||||
// every mod by hand.
|
||||
//
|
||||
// Per-server STATE is preserved/excluded so a clone never leaks one server's
|
||||
// player data onto another:
|
||||
// - The master's RefugeBot homes.json (teleport homes, keyed by SteamID) is
|
||||
// EXCLUDED from the copy — it's per-server state, not mod config. This is
|
||||
// the exact file that leaked when an operator rsync'd Mods by hand.
|
||||
// - The destination's OWN homes.json is snapshotted before the wipe and
|
||||
// restored after, so the joining server keeps its own teleports.
|
||||
//
|
||||
// Both instances must live on THIS agent (same Docker host) — the sidecar
|
||||
// mounts local named volumes. If the master isn't found here the seed returns
|
||||
// an error and the caller treats it as best-effort (the server still boots with
|
||||
// whatever mods it already had).
|
||||
func (d *Dispatcher) seedModsFromInstance(ctx context.Context, srcInstanceID, dstInstanceID, dstDataPath string, manifest *modulepkg.Manifest) (int, error) {
|
||||
if manifest == nil {
|
||||
return 0, fmt.Errorf("mod-seed: nil manifest")
|
||||
}
|
||||
if srcInstanceID == dstInstanceID {
|
||||
return 0, fmt.Errorf("mod-seed: refusing to seed instance %q from itself", dstInstanceID)
|
||||
}
|
||||
installPath := ""
|
||||
if len(manifest.UpdateProviders) > 0 {
|
||||
installPath = manifest.UpdateProviders[0].InstallPath
|
||||
}
|
||||
if installPath == "" {
|
||||
installPath = manifest.Runtime.Docker.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return 0, fmt.Errorf("mod-seed: no install path for module %q", manifest.ID)
|
||||
}
|
||||
|
||||
// Resolve the master's data path from our local instance table.
|
||||
d.mu.Lock()
|
||||
srcRec, ok := d.instances[srcInstanceID]
|
||||
d.mu.Unlock()
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("mod-seed: master instance %q is not on this agent", srcInstanceID)
|
||||
}
|
||||
if srcRec.ModuleID != manifest.ID {
|
||||
return 0, fmt.Errorf("mod-seed: master %q is module %q, not %q", srcInstanceID, srcRec.ModuleID, manifest.ID)
|
||||
}
|
||||
|
||||
srcVolumes := agentmodule.ResolveVolumes(manifest, srcInstanceID, srcRec.DataPath)
|
||||
dstVolumes := agentmodule.ResolveVolumes(manifest, dstInstanceID, dstDataPath)
|
||||
srcGameVol := volumeNameForPath(srcVolumes, installPath)
|
||||
dstGameVol := volumeNameForPath(dstVolumes, installPath)
|
||||
if srcGameVol == "" || dstGameVol == "" {
|
||||
return 0, fmt.Errorf("mod-seed: install_path %q not a named volume on both instances", installPath)
|
||||
}
|
||||
|
||||
// tar, not rsync — debian:12-slim ships no rsync. Replace /dst-game/Mods
|
||||
// from /src-game/Mods, excluding the master's per-server homes.json, and
|
||||
// preserving the destination's own homes.json across the wipe.
|
||||
const seedScript = `set -e
|
||||
echo "mod-seed: cloning master Mods set (excl per-server homes.json)"
|
||||
HOMES=/dst-game/Mods/zzzzzzzRefugeBot/homes.json
|
||||
if [ -f "$HOMES" ]; then cp -a "$HOMES" /tmp/dst-homes.json; echo "mod-seed: preserved destination homes.json"; fi
|
||||
mkdir -p /dst-game/Mods
|
||||
find /dst-game/Mods -mindepth 1 -delete
|
||||
( cd /src-game/Mods && tar --exclude='./zzzzzzzRefugeBot/homes.json' -cf - . ) | ( cd /dst-game/Mods && tar -xf - )
|
||||
if [ -f /tmp/dst-homes.json ]; then mkdir -p /dst-game/Mods/zzzzzzzRefugeBot; cp -a /tmp/dst-homes.json "$HOMES"; echo "mod-seed: restored destination homes.json"; fi
|
||||
echo "mod-seed: $(ls /dst-game/Mods 2>/dev/null | wc -l) mods now present"
|
||||
echo SEED_OK
|
||||
`
|
||||
|
||||
sidecarID := fmt.Sprintf("%s-modseed", dstInstanceID)
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: "debian:12-slim",
|
||||
Command: []string{"bash", "-c", seedScript},
|
||||
RestartPolicy: "no",
|
||||
Volumes: []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true},
|
||||
{Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"},
|
||||
},
|
||||
}
|
||||
|
||||
d.log.Info("mod-seed: starting sidecar", "src", srcInstanceID, "dst", dstInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol)
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("mod-seed: create sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return 0, fmt.Errorf("mod-seed: start sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
var modCount int
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.log.Info("mod-seed log", "instance_id", dstInstanceID, "line", line)
|
||||
// Parse the sidecar's "mod-seed: N mods now present" echo so the
|
||||
// caller (and the cluster-sync UI) can report how many mods landed.
|
||||
if strings.Contains(line, "mods now present") {
|
||||
fields := strings.Fields(line)
|
||||
for i, f := range fields {
|
||||
if f == "mods" && i > 0 {
|
||||
if n, err := strconv.Atoi(fields[i-1]); err == nil {
|
||||
modCount = n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("mod-seed: wait sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return 0, fmt.Errorf("mod-seed: sidecar exited %d", exitCode)
|
||||
}
|
||||
d.log.Info("mod-seed: success", "src", srcInstanceID, "dst", dstInstanceID, "mods", modCount)
|
||||
return modCount, nil
|
||||
}
|
||||
|
||||
// handleSeedMods runs the mod-seed sidecar on demand — no container recreate,
|
||||
// no restart. Clones src_instance_id's Mods/ onto instance_id (excluding the
|
||||
// master's per-server homes.json, preserving the member's own). The member
|
||||
// keeps running on its currently-loaded mods; the freshly-copied files take
|
||||
// effect on its NEXT restart. Backs the cluster "Sync mods to members" action,
|
||||
// where the master pushes its current mod set to every other member at once.
|
||||
func (d *Dispatcher) handleSeedMods(ctx context.Context, corrID string, req *panelv1.SeedModsRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "src", req.SrcInstanceId, "correlation_id", corrID)
|
||||
reply := func(ok bool, modCount int, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_SeedModsResult{
|
||||
SeedModsResult: &panelv1.SeedModsResult{
|
||||
InstanceId: req.InstanceId,
|
||||
Ok: ok,
|
||||
ModCount: int32(modCount),
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
if !ok {
|
||||
log.Warn("mod-seed RPC failed", "err", errMsg)
|
||||
}
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
reply(false, 0, err.Error())
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
reply(false, 0, "module "+rec.ModuleID+" not in registry")
|
||||
return
|
||||
}
|
||||
modCount, err := d.seedModsFromInstance(ctx, req.SrcInstanceId, req.InstanceId, rec.DataPath, manifest)
|
||||
if err != nil {
|
||||
reply(false, 0, err.Error())
|
||||
return
|
||||
}
|
||||
log.Info("mod-seed RPC complete", "mods", modCount)
|
||||
reply(true, modCount, "")
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// Region Medic agent handlers — scan a 7DTD instance's active-world Region
|
||||
// directory for .7rg corruption, and heal one region from the newest clean
|
||||
// panel backup. Volume files are reached through the runtime's container-path
|
||||
// ops (the agent runs unprivileged, so it cannot touch /var/lib/docker/volumes
|
||||
// directly). The controller orchestrates stop→heal→start; handleRegionHeal
|
||||
// itself only performs the file swap and refuses to run on a live container.
|
||||
|
||||
// activeRegionDir resolves the container-absolute Region directory of the
|
||||
// instance's active world by reading GameWorld/GameName from the live
|
||||
// serverconfig.xml. Works on a stopped container (CopyFileFromContainer does).
|
||||
func (d *Dispatcher) activeRegionDir(ctx context.Context, container string) (string, error) {
|
||||
raw, err := d.runtime.CopyFileFromContainer(ctx, container, "/game-saves/serverconfig.xml")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read serverconfig.xml: %w", err)
|
||||
}
|
||||
world := xmlProp(string(raw), "GameWorld")
|
||||
game := xmlProp(string(raw), "GameName")
|
||||
if world == "" || game == "" {
|
||||
return "", fmt.Errorf("serverconfig.xml missing GameWorld/GameName (got %q/%q)", world, game)
|
||||
}
|
||||
// HOME is pinned to /game-saves by the 7dtd entrypoint.
|
||||
return path.Join("/game-saves/.local/share/7DaysToDie/Saves", world, game, "Region"), nil
|
||||
}
|
||||
|
||||
// xmlProp pulls the value from a 7DTD serverconfig line:
|
||||
//
|
||||
// <property name="GameWorld" value="Navezgane" />
|
||||
//
|
||||
// Tolerant of attribute order and single/double quotes; skips commented lines.
|
||||
func xmlProp(content, name string) string {
|
||||
for _, raw := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "<!--") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(line, `name="`+name+`"`) && !strings.Contains(line, `name='`+name+`'`) {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(line, "value=")
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
rest := line[i+len("value="):]
|
||||
if rest == "" {
|
||||
continue
|
||||
}
|
||||
q := rest[0]
|
||||
rest = rest[1:]
|
||||
if end := strings.IndexByte(rest, q); end >= 0 {
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// listRegionDirNames returns the file names in the container's Region dir.
|
||||
// Uses ExecCapture when the container is running; nil + error otherwise (the
|
||||
// scan path runs while the instance is up).
|
||||
func (d *Dispatcher) listRegionDirNames(ctx context.Context, container, regionDir string) ([]string, error) {
|
||||
stdout, _, _, err := d.runtime.ExecCapture(ctx, container, []string{"ls", "-1", regionDir})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var names []string
|
||||
for _, n := range strings.Split(string(stdout), "\n") {
|
||||
if n = strings.TrimSpace(n); n != "" {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// handleRegionScan reports corruption evidence for an instance's active world:
|
||||
// error_backup salvage files clustered into regions, plus structural validation
|
||||
// of each affected region's .7rg file.
|
||||
func (d *Dispatcher) handleRegionScan(ctx context.Context, corrID string, req *panelv1.RegionScanRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "not_found", Message: err.Error()}})
|
||||
return
|
||||
}
|
||||
container := "panel-" + rec.InstanceID
|
||||
regionDir, err := d.activeRegionDir(ctx, container)
|
||||
if err != nil {
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "resolve_region_dir", Message: err.Error()}})
|
||||
return
|
||||
}
|
||||
names, err := d.listRegionDirNames(ctx, container, regionDir)
|
||||
if err != nil {
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{RegionDir: regionDir, Error: &panelv1.Error{Code: "list_region_dir", Message: err.Error()}})
|
||||
return
|
||||
}
|
||||
|
||||
// Cluster error_backups by region; note which region files exist.
|
||||
type agg struct {
|
||||
files int
|
||||
present bool
|
||||
}
|
||||
clusters := map[regionmedic.RegionCoord]*agg{}
|
||||
fileSet := map[regionmedic.RegionCoord]bool{}
|
||||
total := 0
|
||||
for _, n := range names {
|
||||
if cx, cz, ok := regionmedic.ParseErrorBackup(n); ok {
|
||||
total++
|
||||
rc := regionmedic.RegionForChunk(cx, cz)
|
||||
a := clusters[rc]
|
||||
if a == nil {
|
||||
a = &agg{}
|
||||
clusters[rc] = a
|
||||
}
|
||||
a.files++
|
||||
continue
|
||||
}
|
||||
if rc, ok := regionmedic.ParseRegionFileName(n); ok {
|
||||
fileSet[rc] = true
|
||||
}
|
||||
}
|
||||
|
||||
var affected []*panelv1.AffectedRegionMsg
|
||||
for rc, a := range clusters {
|
||||
msg := &panelv1.AffectedRegionMsg{
|
||||
Region: rc.String(),
|
||||
ErrorBackups: int32(a.files),
|
||||
FilePresent: fileSet[rc],
|
||||
}
|
||||
// Validate the affected region file (read via container-path op).
|
||||
if fileSet[rc] {
|
||||
if b, rerr := d.runtime.CopyFileFromContainer(ctx, container, path.Join(regionDir, rc.FileName())); rerr == nil {
|
||||
rep := regionmedic.ValidateRegionBytes(b, false)
|
||||
msg.FileCorrupt = !rep.Healthy()
|
||||
msg.BadChunks = int32(len(rep.BadChunks))
|
||||
}
|
||||
}
|
||||
affected = append(affected, msg)
|
||||
}
|
||||
sort.Slice(affected, func(i, j int) bool { return affected[i].ErrorBackups > affected[j].ErrorBackups })
|
||||
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{
|
||||
ErrorBackups: int32(total),
|
||||
Affected: affected,
|
||||
RegionDir: regionDir,
|
||||
})
|
||||
}
|
||||
|
||||
// handleRegionHeal swaps one region file for the newest clean backup copy.
|
||||
// It REQUIRES the instance to be stopped (the controller stops it first) so the
|
||||
// running engine can't overwrite the swap. It snapshots the corrupt original
|
||||
// next to the region file before overwriting. Orchestration (stop/start) is the
|
||||
// controller's job.
|
||||
func (d *Dispatcher) handleRegionHeal(ctx context.Context, corrID string, req *panelv1.RegionHealRequest) {
|
||||
fail := func(code, msg string) {
|
||||
d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{Region: req.Region, Error: &panelv1.Error{Code: code, Message: msg}})
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
fail("not_found", err.Error())
|
||||
return
|
||||
}
|
||||
region, ok := regionmedic.ParseRegionFileName(regionFileArg(req.Region))
|
||||
if !ok {
|
||||
fail("bad_region", fmt.Sprintf("invalid region %q", req.Region))
|
||||
return
|
||||
}
|
||||
container := "panel-" + rec.InstanceID
|
||||
|
||||
// Refuse to swap a region on a LIVE container (it would be overwritten on
|
||||
// the next save). ExecCapture succeeds only when the container is running.
|
||||
if _, _, _, exErr := d.runtime.ExecCapture(ctx, container, []string{"true"}); exErr == nil {
|
||||
fail("instance_running", "instance must be stopped before healing a region")
|
||||
return
|
||||
}
|
||||
|
||||
regionDir, err := d.activeRegionDir(ctx, container)
|
||||
if err != nil {
|
||||
fail("resolve_region_dir", err.Error())
|
||||
return
|
||||
}
|
||||
liveFile := path.Join(regionDir, region.FileName())
|
||||
liveBytes, err := d.runtime.CopyFileFromContainer(ctx, container, liveFile)
|
||||
if err != nil {
|
||||
fail("read_live_region", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Find the newest clean backup copy of this region (backups are local to
|
||||
// the agent at <backupDir>/<instance>/).
|
||||
backups, err := regionmedic.ListPanelBackups(path.Join(d.backupDir, rec.InstanceID))
|
||||
if err != nil {
|
||||
fail("list_backups", err.Error())
|
||||
return
|
||||
}
|
||||
entry := regionmedic.SaveRelEntry(regionDir, region.FileName())
|
||||
best, _, err := regionmedic.FindLastGoodRegion(backups, entry, true, time.Time{})
|
||||
if err != nil {
|
||||
fail("find_backup", err.Error())
|
||||
return
|
||||
}
|
||||
if !best.Found || !best.Clean() {
|
||||
fail("no_clean_backup", "no backup contains a clean copy of this region")
|
||||
return
|
||||
}
|
||||
|
||||
// Snapshot the corrupt original alongside the region file (no new subdir, so
|
||||
// CopyFileToContainer needs no parent-dir creation on a stopped container).
|
||||
stamp := time.Now().UTC().Format("20060102-150405")
|
||||
snapName := region.FileName() + ".medic-corrupt-" + stamp
|
||||
snapPath := path.Join(regionDir, snapName)
|
||||
if err := d.runtime.CopyFileToContainer(ctx, container, snapPath, liveBytes); err != nil {
|
||||
fail("snapshot", err.Error())
|
||||
return
|
||||
}
|
||||
// Swap in the clean copy (overwrites the live region file in place).
|
||||
if err := d.runtime.CopyFileToContainer(ctx, container, liveFile, best.Bytes); err != nil {
|
||||
fail("write_region", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{
|
||||
Region: region.String(),
|
||||
SourceBackup: best.Backup.ID,
|
||||
BytesWritten: int64(len(best.Bytes)),
|
||||
SnapshotPath: snapPath,
|
||||
Healed: true,
|
||||
})
|
||||
}
|
||||
|
||||
// regionFileArg normalizes "r.X.Z" or "r.X.Z.7rg" to a .7rg filename.
|
||||
func regionFileArg(s string) string {
|
||||
if strings.HasSuffix(s, ".7rg") {
|
||||
return s
|
||||
}
|
||||
return s + ".7rg"
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendRegionScanResult(corrID string, res *panelv1.RegionScanResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_RegionScanResult{RegionScanResult: res},
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendRegionHealResult(corrID string, res *panelv1.RegionHealResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_RegionHealResult{RegionHealResult: res},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
)
|
||||
|
||||
func medicEnvOr(k, def string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// Medic host paths on the agent box (figaro, for 7DTD). Overridable via env so
|
||||
// the same agent binary works if 7DTD ever moves hosts.
|
||||
func regionRollingRoot() string {
|
||||
return medicEnvOr("PANEL_REGION_ROLLING_DIR", "/home/refuge/region-rolling")
|
||||
}
|
||||
func regionMedicBin() string {
|
||||
return medicEnvOr("PANEL_REGION_MEDIC_BIN", "/home/refuge/panel/bin/region-medic")
|
||||
}
|
||||
func discordTokenFile() string {
|
||||
return medicEnvOr("PANEL_DISCORD_TOKEN_FILE", "/home/refuge/panel/discord-bot-token")
|
||||
}
|
||||
|
||||
func medicFileExists(p string) bool {
|
||||
st, err := os.Stat(p)
|
||||
return err == nil && !st.IsDir()
|
||||
}
|
||||
|
||||
// runRegionMedicOnStart runs the autoheal CLI in a throwaway debian sidecar while
|
||||
// the 7DTD container is STILL STOPPED, just before the agent starts it:
|
||||
// - validate every .7rg in the active world's Region dir,
|
||||
// - heal corrupt ones from the newest clean rolling snapshot, then the panel
|
||||
// tar backups as a fallback,
|
||||
// - snapshot the now-clean Region dir to a fresh rolling slot,
|
||||
// - post a Discord summary (naming the server) if anything healed.
|
||||
//
|
||||
// The per-server medic config (enable / rolling-keep / Discord channel) lives in
|
||||
// the SAVES volume at region-medic.json — written by the panel's medic-config
|
||||
// endpoint via FsWrite (NO container recreate) and read by the CLI here in the
|
||||
// sidecar. The agent therefore always launches the sidecar for 7DTD and lets the
|
||||
// CLI decide; an absent/empty config just means defaults + no Discord.
|
||||
//
|
||||
// Best-effort by contract: any failure is logged + swallowed, never blocking the
|
||||
// start. 10-min hard cap so a wedged sidecar/daemon can't stall a boot.
|
||||
func (d *Dispatcher) runRegionMedicOnStart(ctx context.Context, rec *instanceRecord) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
log := d.log.With("instance_id", rec.InstanceID, "medic", true)
|
||||
medicBin := regionMedicBin()
|
||||
if !medicFileExists(medicBin) {
|
||||
log.Warn("region-medic: binary missing, skipping pre-start heal", "bin", medicBin)
|
||||
return
|
||||
}
|
||||
savesVol := fmt.Sprintf("panel-%s-saves", rec.InstanceID)
|
||||
rollingDir := path.Join(regionRollingRoot(), rec.InstanceID)
|
||||
_ = os.MkdirAll(rollingDir, 0o755)
|
||||
backupsDir := path.Join(d.backupDir, rec.InstanceID)
|
||||
|
||||
cmd := []string{"/rm", "autoheal",
|
||||
"-saves-root", "/saves",
|
||||
"-rolling-dir", "/rolling",
|
||||
"-keep", "2", // default; overridden by region-medic.json in the saves volume
|
||||
"-label", rec.InstanceID,
|
||||
"-apply",
|
||||
}
|
||||
vols := []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: savesVol, ContainerPath: "/saves"},
|
||||
{Type: "bind", HostPath: rollingDir, ContainerPath: "/rolling"},
|
||||
{Type: "bind", HostPath: medicBin, ContainerPath: "/rm", ReadOnly: true},
|
||||
}
|
||||
// debian:12-slim ships no CA bundle, so the CLI's HTTPS POST to Discord can't
|
||||
// verify the TLS cert ("x509: certificate signed by unknown authority"). Mount
|
||||
// the host's CA certs read-only — Go's x509 finds ca-certificates.crt there.
|
||||
if _, err := os.Stat("/etc/ssl/certs"); err == nil {
|
||||
vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: "/etc/ssl/certs", ContainerPath: "/etc/ssl/certs", ReadOnly: true})
|
||||
}
|
||||
// Panel tar backups are the heal fallback when no rolling slot has a clean
|
||||
// copy (e.g. very first boot, before any snapshot). Only mount if present.
|
||||
if st, err := os.Stat(backupsDir); err == nil && st.IsDir() {
|
||||
cmd = append(cmd, "-backups-dir", "/backups")
|
||||
vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: backupsDir, ContainerPath: "/backups", ReadOnly: true})
|
||||
}
|
||||
// The token is mounted whenever it exists; the CLI only posts if the saves
|
||||
// config names a channel. No -discord-channel flag — the CLI reads it from
|
||||
// /saves/region-medic.json.
|
||||
if tf := discordTokenFile(); medicFileExists(tf) {
|
||||
cmd = append(cmd, "-discord-token-file", "/discord-token")
|
||||
vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: tf, ContainerPath: "/discord-token", ReadOnly: true})
|
||||
}
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: rec.InstanceID + "-medic",
|
||||
IsSidecar: true,
|
||||
Image: "debian:12-slim",
|
||||
Command: cmd,
|
||||
RestartPolicy: "no",
|
||||
Volumes: vols,
|
||||
}
|
||||
|
||||
log.Info("region-medic: pre-start scan+heal")
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
log.Warn("region-medic: create sidecar failed", "err", err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
log.Warn("region-medic: start sidecar failed", "err", err)
|
||||
return
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
log.Info("region-medic", "line", line)
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
log.Warn("region-medic: wait sidecar failed", "err", err)
|
||||
return
|
||||
}
|
||||
if exitCode != 0 {
|
||||
log.Warn("region-medic: sidecar exited non-zero", "code", exitCode)
|
||||
return
|
||||
}
|
||||
log.Info("region-medic: pre-start heal complete")
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// handleRenderConfig re-renders the module's declared config_files into the
|
||||
// instance's host DataPath — the same modulepkg.Render call handleCreate
|
||||
// makes, minus the container recreate. For modules that bind-mount the
|
||||
// rendered output into the container (7dtd's serverconfig.xml.rendered),
|
||||
// os.WriteFile rewrites the host file in place (same inode), so the running
|
||||
// container sees the new content immediately and the entrypoint applies it
|
||||
// on the NEXT boot. The container itself is never restarted.
|
||||
//
|
||||
// Containers created BEFORE a config_file's bind-mount was added to the
|
||||
// module manifest don't have the bind at all (docker pins mounts at create)
|
||||
// — their /game-saves/<file> is a plain file inside the saves volume. For
|
||||
// those we self-verify: read the file back through the container, and if it
|
||||
// doesn't match the fresh render, copy the rendered bytes in directly
|
||||
// (works running or stopped; the game only reads config at boot).
|
||||
func (d *Dispatcher) handleRenderConfig(ctx context.Context, corrID string, req *panelv1.InstanceRenderConfigRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||||
fail := func(msg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_InstanceRenderConfigResult{
|
||||
InstanceRenderConfigResult: &panelv1.InstanceRenderConfigResult{Error: msg},
|
||||
},
|
||||
})
|
||||
log.Warn("render config failed", "err", msg)
|
||||
}
|
||||
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
fail(err.Error())
|
||||
return
|
||||
}
|
||||
if rec.DataPath == "" {
|
||||
fail("instance has no data path (pre-feature sidecar?) — recreate once to adopt")
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
fail("module " + rec.ModuleID + " not in registry")
|
||||
return
|
||||
}
|
||||
|
||||
// Same merge handleCreate uses: generated secrets first, user values on
|
||||
// top (empties dropped → template `or` defaults kick in).
|
||||
values, err := mergeValuesAndSecrets(manifest, req.ConfigValues)
|
||||
if err != nil {
|
||||
fail("secrets: " + err.Error())
|
||||
return
|
||||
}
|
||||
// Branch-aware: a 3.0 instance renders serverconfig.v3.xml.tmpl, ≤2.6
|
||||
// keeps the default. Resolve from the instance's persisted branch (set at
|
||||
// create), falling back to the merged config_values' provider_id so a
|
||||
// durable config-render still picks the right template if the record
|
||||
// predates branch persistence.
|
||||
branch := rec.Branch
|
||||
if branch == "" {
|
||||
branch = resolveCreateBranch(manifest, values)
|
||||
}
|
||||
if err := modulepkg.RenderForBranch(manifest, rec.DataPath, values, branch); err != nil {
|
||||
fail("render: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Self-verify through the container; fall back to a direct copy into
|
||||
// the volume for pre-bind containers. The mapping host-path → container
|
||||
// path comes from the manifest's volume specs.
|
||||
volumes := agentmodule.ResolveVolumes(manifest, rec.InstanceID, rec.DataPath)
|
||||
container := "panel-" + rec.InstanceID
|
||||
var rendered []string
|
||||
for _, cf := range manifest.ConfigFiles {
|
||||
if cf.TemplateForBranch(branch) == "" {
|
||||
continue
|
||||
}
|
||||
rendered = append(rendered, cf.Path)
|
||||
hostPath := filepath.Join(rec.DataPath, cf.Path)
|
||||
ctrPath := ""
|
||||
for _, v := range volumes {
|
||||
if v.Type == "bind" && v.HostPath == hostPath {
|
||||
ctrPath = v.ContainerPath
|
||||
break
|
||||
}
|
||||
}
|
||||
if ctrPath == "" {
|
||||
continue // file isn't mounted into the container; host render is all there is
|
||||
}
|
||||
want, rerr := os.ReadFile(hostPath)
|
||||
if rerr != nil {
|
||||
fail("read rendered " + cf.Path + ": " + rerr.Error())
|
||||
return
|
||||
}
|
||||
got, gerr := d.runtime.CopyFileFromContainer(ctx, container, ctrPath)
|
||||
if gerr == nil && bytes.Equal(got, want) {
|
||||
continue // live bind-mount delivered the new render
|
||||
}
|
||||
// Pre-bind container (or unreadable): write the volume copy directly.
|
||||
if cerr := d.runtime.CopyFileToContainer(ctx, container, ctrPath, want); cerr != nil {
|
||||
fail("container has no live bind for " + cf.Path + " and direct copy failed: " + cerr.Error())
|
||||
return
|
||||
}
|
||||
log.Info("rendered config copied into pre-bind container volume", "file", cf.Path, "container_path", ctrPath)
|
||||
}
|
||||
log.Info("re-rendered config files (durable, applies next boot)", "files", rendered)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_InstanceRenderConfigResult{
|
||||
InstanceRenderConfigResult: &panelv1.InstanceRenderConfigResult{RenderedFiles: rendered},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// blockingSender simulates a controller that has stopped draining the stream:
|
||||
// every Send blocks until released. This is the backpressure condition that
|
||||
// used to wedge the whole agent when sendEnv called Send inline.
|
||||
type blockingSender struct {
|
||||
release chan struct{}
|
||||
mu sync.Mutex
|
||||
sent int
|
||||
}
|
||||
|
||||
func (b *blockingSender) Send(*panelv1.AgentEnvelope) error {
|
||||
<-b.release // block until the test lets it through
|
||||
b.mu.Lock()
|
||||
b.sent++
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func logEnv() *panelv1.AgentEnvelope {
|
||||
return &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{Line: "x"}}}
|
||||
}
|
||||
|
||||
func stateEnv() *panelv1.AgentEnvelope {
|
||||
return &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_InstanceState{InstanceState: &panelv1.InstanceStateUpdate{InstanceId: "i"}}}
|
||||
}
|
||||
|
||||
// newTestDispatcher builds a Dispatcher with just the send machinery wired —
|
||||
// enough to exercise sendEnv / sendLoop without a real runtime.
|
||||
func newTestDispatcher() *Dispatcher {
|
||||
d := &Dispatcher{sendCh: make(chan *panelv1.AgentEnvelope, 4096)}
|
||||
go d.sendLoop()
|
||||
return d
|
||||
}
|
||||
|
||||
// TestSendEnvDropsTelemetryUnderBackpressure is the core regression guard for
|
||||
// the VEIN-exposed wedge: with the sender blocked, a flood of log envelopes far
|
||||
// exceeding the buffer must NOT block the producer. Pre-fix, sendEnv called
|
||||
// Send inline and the producer (and the whole agent) froze here.
|
||||
func TestSendEnvDropsTelemetryUnderBackpressure(t *testing.T) {
|
||||
d := newTestDispatcher()
|
||||
bs := &blockingSender{release: make(chan struct{})}
|
||||
d.SetSender(bs)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
// 50k log frames — 12x the 4096 buffer. Must all return promptly.
|
||||
for i := 0; i < 50000; i++ {
|
||||
d.sendEnv(logEnv())
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Producer completed without blocking on the stalled sender. Pass.
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("sendEnv blocked under backpressure — the wedge regressed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCriticalEnvelopeNotDropped verifies an instance-state envelope is queued
|
||||
// (not silently dropped) even when telemetry is being shed. We fill the buffer
|
||||
// with the sender blocked, then confirm a critical send blocks until drained
|
||||
// (i.e. it's preserved, not discarded).
|
||||
func TestCriticalEnvelopeNotDropped(t *testing.T) {
|
||||
d := &Dispatcher{sendCh: make(chan *panelv1.AgentEnvelope, 2)}
|
||||
// No drainer started yet — fill the buffer.
|
||||
d.sendEnv(logEnv())
|
||||
d.sendEnv(logEnv())
|
||||
// Buffer is full (cap 2). A telemetry frame must drop (non-blocking).
|
||||
dropDone := make(chan struct{})
|
||||
go func() { d.sendEnv(logEnv()); close(dropDone) }()
|
||||
select {
|
||||
case <-dropDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("telemetry send blocked on full buffer — should have dropped")
|
||||
}
|
||||
// A critical frame must NOT drop: it blocks until space frees. Start a
|
||||
// drainer to free space and confirm the critical send then completes.
|
||||
critDone := make(chan struct{})
|
||||
go func() { d.sendEnv(stateEnv()); close(critDone) }()
|
||||
select {
|
||||
case <-critDone:
|
||||
t.Fatal("critical send returned before buffer had space — it was dropped")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// Correctly still blocking. Now drain and it should complete.
|
||||
}
|
||||
go d.sendLoop()
|
||||
select {
|
||||
case <-critDone:
|
||||
// Drained and delivered. Pass.
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("critical send never completed after drain")
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrentProducersNoRace runs many producers through sendEnv at once.
|
||||
// Pre-fix, each called stream.Send directly — concurrent Sends on one gRPC
|
||||
// stream are a data race. The single-drainer design serializes them. Run with
|
||||
// -race to catch a regression.
|
||||
func TestConcurrentProducersNoRace(t *testing.T) {
|
||||
d := newTestDispatcher()
|
||||
bs := &blockingSender{release: make(chan struct{})}
|
||||
close(bs.release) // never block — just count
|
||||
d.SetSender(bs)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for p := 0; p < 32; p++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 1000; i++ {
|
||||
d.sendEnv(logEnv())
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
// Give the drainer a moment to flush.
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
// No assertion on exact count (some may still be in flight / dropped);
|
||||
// the point is the -race detector finds no concurrent Send.
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Per-instance stats polling. Docker streams a JSON frame ~once per
|
||||
// second; we throttle to one upstream emit per statsEmitInterval so the
|
||||
// event bus isn't flooded. CPU% uses the frame's cur−previous delta
|
||||
// directly (Docker includes precpu_stats on streamed frames).
|
||||
|
||||
// 3s is snappy enough to watch RAM climb during ARK world-load (12-15 GB
|
||||
// in under a minute) without flooding the SSE bus. Older 10s value made
|
||||
// the dashboard feel laggy when memory was actually moving fast.
|
||||
const statsEmitInterval = 3 * time.Second
|
||||
|
||||
// ARK SA memory guardrails — two-stage so we can flush saves cleanly
|
||||
// before having to SIGKILL.
|
||||
//
|
||||
// Healthy ARK SA world: 10-15 GB resident. 16-30 GB during save-flush
|
||||
// peaks is normal on a populated cluster. 45 GB is the warning level;
|
||||
// 60 GB is the hard kill that protects the host from OOM (which would
|
||||
// take down all sibling ARKs + postgres + the panel itself).
|
||||
//
|
||||
// 2026-05-25 history: the previous single-threshold 40 GB → 5s SIGKILL
|
||||
// path corrupted at least three SQLite-backed `.ark` saves over the
|
||||
// 2026-05-19 → 2026-05-25 window because the SIGKILL hit while a
|
||||
// SaveWorld was mid-flush (FAtlasSaveManager::LoadOperationSql crash on
|
||||
// next boot — null page read in the half-written SQLite store).
|
||||
//
|
||||
// The two-stage guard:
|
||||
//
|
||||
// At arkRAMWarnLimit (45 GB) — fired AT MOST ONCE per container life:
|
||||
// 1. Broadcast a 60s heads-up to in-game players via RCON
|
||||
// 2. SaveWorld (lets ARK flush cleanly)
|
||||
// 3. Wait 60s for the save + warning window
|
||||
// 4. runtime.Stop with 120s grace (clean shutdown path)
|
||||
// The container then stays Exited; operator restarts when ready, no
|
||||
// torn save, no progress loss beyond what the SaveWorld captured.
|
||||
//
|
||||
// At arkRAMHardLimit (60 GB) — fires if the warn path didn't help:
|
||||
// SIGKILL via runtime.Stop with 30s grace. Save MAY be torn here;
|
||||
// this is the host-protection layer, accepted as last-resort.
|
||||
//
|
||||
// The warn level fires at most once per container life via the
|
||||
// rec.memGuardWarned atomic — once we've committed to a graceful stop,
|
||||
// the hard limit isn't allowed to also fire and interrupt the SaveWorld.
|
||||
const arkRAMWarnLimit uint64 = 45 * 1024 * 1024 * 1024
|
||||
const arkRAMHardLimit uint64 = 60 * 1024 * 1024 * 1024
|
||||
|
||||
// gracefulStopGracePeriod is the docker-stop grace given to the
|
||||
// graceful warn path. ARK's SaveWorld can take 30-60s on a populated
|
||||
// world; 120s is enough headroom for the flush + the watchdog's DoExit.
|
||||
const gracefulStopGracePeriod = 120 * time.Second
|
||||
|
||||
// emergencyStopGracePeriod is the docker-stop grace for the 60 GB hard
|
||||
// path. 30s (was 5s) gives the engine a real shot at flushing in-flight
|
||||
// state before SIGKILL — the host-OOM risk that motivated the original
|
||||
// 5s grace doesn't materialize this fast in practice.
|
||||
const emergencyStopGracePeriod = 30 * time.Second
|
||||
|
||||
// dockerStatsFrame is the minimum subset of Docker's /containers/<id>/stats
|
||||
// response we need. Kept as a local shape so we don't depend on which
|
||||
// version of github.com/docker/docker renamed the struct this week.
|
||||
type dockerStatsFrame struct {
|
||||
CPUStats struct {
|
||||
// IMPORTANT: every nested field needs an explicit json tag.
|
||||
// Go's unmarshal is case-insensitive but does NOT translate
|
||||
// CamelCase ↔ snake_case. Without `json:"total_usage"`, the
|
||||
// field reads as 0 because Docker emits the snake_case key
|
||||
// and we'd be looking for "TotalUsage" / "totalusage". This
|
||||
// bug shipped for months — every container reported 0% CPU.
|
||||
CPUUsage struct {
|
||||
TotalUsage uint64 `json:"total_usage"`
|
||||
} `json:"cpu_usage"`
|
||||
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
||||
OnlineCPUs uint32 `json:"online_cpus"`
|
||||
} `json:"cpu_stats"`
|
||||
PreCPUStats struct {
|
||||
CPUUsage struct {
|
||||
TotalUsage uint64 `json:"total_usage"`
|
||||
} `json:"cpu_usage"`
|
||||
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
||||
} `json:"precpu_stats"`
|
||||
MemoryStats struct {
|
||||
Usage uint64 `json:"usage"`
|
||||
Limit uint64 `json:"limit"`
|
||||
// Docker's `usage` includes file-system page cache, which can
|
||||
// balloon to multi-GB on read-heavy workloads (ARK SA's mod +
|
||||
// asset reads inflate it to 25 GB on a 12 GB RSS workload).
|
||||
// `docker stats` CLI subtracts cache/inactive_file to match
|
||||
// what operators expect. We mirror that.
|
||||
// - cgroups v1: usage - stats.cache
|
||||
// - cgroups v2: usage - stats.inactive_file
|
||||
// Both keys are in the same nested map; whichever is present
|
||||
// is the one our cgroup version exposes.
|
||||
Stats struct {
|
||||
Cache uint64 `json:"cache"` // cgroups v1
|
||||
InactiveFile uint64 `json:"inactive_file"` // cgroups v2
|
||||
RSS uint64 `json:"rss"` // cgroups v1
|
||||
Anon uint64 `json:"anon"` // cgroups v2 (active anon)
|
||||
} `json:"stats"`
|
||||
} `json:"memory_stats"`
|
||||
Networks map[string]struct {
|
||||
RxBytes uint64 `json:"rx_bytes"`
|
||||
TxBytes uint64 `json:"tx_bytes"`
|
||||
} `json:"networks"`
|
||||
PidsStats struct {
|
||||
Current uint32 `json:"current"`
|
||||
} `json:"pids_stats"`
|
||||
}
|
||||
|
||||
func (d *Dispatcher) streamStats(ctx context.Context, rec *instanceRecord) {
|
||||
rc, err := d.runtime.StatsStream(ctx, rec.ContainerID)
|
||||
if err != nil {
|
||||
d.log.Warn("stats stream open failed", "instance_id", rec.InstanceID, "err", err)
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
d.log.Info("stats stream opened", "instance_id", rec.InstanceID)
|
||||
emitCount := 0
|
||||
defer func() { d.log.Info("stats stream closed", "instance_id", rec.InstanceID, "emitted", emitCount) }()
|
||||
|
||||
dec := json.NewDecoder(rc)
|
||||
lastEmit := time.Time{}
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var f dockerStatsFrame
|
||||
if err := dec.Decode(&f); err != nil {
|
||||
if err != io.EOF && ctx.Err() == nil {
|
||||
d.log.Debug("stats decode ended", "instance_id", rec.InstanceID, "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Since(lastEmit) < statsEmitInterval {
|
||||
continue
|
||||
}
|
||||
memUsed := d.sendStats(rec.InstanceID, &f)
|
||||
lastEmit = time.Now()
|
||||
emitCount++
|
||||
// ARK runaway-memory guardrails. Two stages: warn (45 GB) tries a
|
||||
// graceful flush; hard (60 GB) SIGKILLs to protect the host.
|
||||
// Both paths set rec.stopping inside their handler so subsequent
|
||||
// samples short-circuit there.
|
||||
if rec.ModuleID == "ark-sa" {
|
||||
if memUsed > arkRAMHardLimit {
|
||||
go d.emergencyRestart(rec, fmt.Sprintf(
|
||||
"ARK RAM hit %.1f GB (hard limit %d GB) — bouncing to protect host",
|
||||
float64(memUsed)/1024/1024/1024, arkRAMHardLimit/1024/1024/1024))
|
||||
return
|
||||
}
|
||||
if memUsed > arkRAMWarnLimit && rec.memGuardWarned.CompareAndSwap(false, true) {
|
||||
go d.gracefulMemoryRestart(rec, fmt.Sprintf(
|
||||
"ARK RAM hit %.1f GB (warn limit %d GB) — flushing save + graceful restart",
|
||||
float64(memUsed)/1024/1024/1024, arkRAMWarnLimit/1024/1024/1024))
|
||||
// Don't return — keep streaming so the hard limit can still
|
||||
// fire if RAM keeps climbing past 60 GB during the 60s flush
|
||||
// window. The memGuardWarned latch prevents the warn path
|
||||
// from re-arming, but the hard path is the host-protection
|
||||
// backstop and must remain live.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendStats(instanceID string, f *dockerStatsFrame) uint64 {
|
||||
// CPU%: delta total / delta system * online_cpus * 100.
|
||||
// First frame's precpu_stats is zero so cpuPct comes out 0 — fine.
|
||||
var cpuPct float64
|
||||
cpuDelta := int64(f.CPUStats.CPUUsage.TotalUsage) - int64(f.PreCPUStats.CPUUsage.TotalUsage)
|
||||
sysDelta := int64(f.CPUStats.SystemCPUUsage) - int64(f.PreCPUStats.SystemCPUUsage)
|
||||
hostCpus := f.CPUStats.OnlineCPUs
|
||||
if sysDelta > 0 && cpuDelta > 0 {
|
||||
cpus := float64(hostCpus)
|
||||
if cpus < 1 {
|
||||
cpus = 1
|
||||
}
|
||||
cpuPct = float64(cpuDelta) / float64(sysDelta) * cpus * 100.0
|
||||
}
|
||||
|
||||
var rx, tx uint64
|
||||
for _, n := range f.Networks {
|
||||
rx += n.RxBytes
|
||||
tx += n.TxBytes
|
||||
}
|
||||
|
||||
// "Real" memory: subtract file cache (cgroups v1) or inactive_file
|
||||
// (cgroups v2). Falls through to raw usage when neither is reported
|
||||
// (Windows containers, exotic runtimes). This is what `docker stats`
|
||||
// shows and what operators expect — the inflated ARK numbers (~25 GB
|
||||
// vs the actual ~12 GB RSS) were the cache-included variant.
|
||||
memUsed := f.MemoryStats.Usage
|
||||
if f.MemoryStats.Stats.InactiveFile > 0 && f.MemoryStats.Stats.InactiveFile <= memUsed {
|
||||
memUsed -= f.MemoryStats.Stats.InactiveFile
|
||||
} else if f.MemoryStats.Stats.Cache > 0 && f.MemoryStats.Stats.Cache <= memUsed {
|
||||
memUsed -= f.MemoryStats.Stats.Cache
|
||||
}
|
||||
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_InstanceStats{
|
||||
InstanceStats: &panelv1.InstanceStatsUpdate{
|
||||
InstanceId: instanceID,
|
||||
CpuPercent: cpuPct,
|
||||
MemUsedBytes: memUsed,
|
||||
MemLimitBytes: f.MemoryStats.Limit,
|
||||
NetRxBytes: rx,
|
||||
NetTxBytes: tx,
|
||||
Pids: f.PidsStats.Current,
|
||||
HostCpus: hostCpus,
|
||||
At: timestamppb.Now(),
|
||||
},
|
||||
},
|
||||
})
|
||||
return memUsed
|
||||
}
|
||||
|
||||
// gracefulMemoryRestart is the 45 GB warn path. Tries to land a clean
|
||||
// SaveWorld before bouncing the container so the next boot doesn't face
|
||||
// a torn SQLite save. Sequence:
|
||||
//
|
||||
// 1. Latch rec.stopping so other paths back off.
|
||||
// 2. Emit a panel log line + INSTANCE_STATUS_STOPPING state.
|
||||
// 3. RCON Broadcast: "[Panel] high memory — saving and restarting in 60s"
|
||||
// 4. RCON SaveWorld
|
||||
// 5. Sleep 60s (gives players warning + engine time to write the save)
|
||||
// 6. Cancel tracker/log goroutines, runtime.Restart with 120s grace.
|
||||
//
|
||||
// Uses Restart NOT Stop: a plain Stop on an `unless-stopped` container
|
||||
// is honored by docker as an operator stop — the container then stays
|
||||
// Exited until someone manually restarts it. That regression took
|
||||
// Ragnarok offline for 17 hours after the 02:02 PDT warn fire on
|
||||
// 2026-05-25 (see HANDOFF "Last meaningfully updated"). Restart bounces
|
||||
// the container as a transient event so it comes back up automatically;
|
||||
// the operator just sees a brief blip + the "[panel] MEMORY-WARN" line.
|
||||
//
|
||||
// If any RCON step fails (engine wedged, RCON socket dead), fall through
|
||||
// to the restart anyway — we still want the engine down + back up. The
|
||||
// 120s docker grace gives the engine a real shot at a clean DoExit even
|
||||
// when RCON is the broken thing.
|
||||
func (d *Dispatcher) gracefulMemoryRestart(rec *instanceRecord, reason string) {
|
||||
if !rec.stopping.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
d.log.Warn("ark memory warn — graceful restart", "instance_id", rec.InstanceID, "reason", reason)
|
||||
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] MEMORY-WARN: " + reason,
|
||||
}},
|
||||
})
|
||||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPING, 0, "graceful memory restart: "+reason)
|
||||
|
||||
// Best-effort RCON broadcast + SaveWorld. Use a short per-call
|
||||
// timeout so a wedged RCON socket doesn't block the whole graceful
|
||||
// path. tracker.Exec redials internally, so a closed socket from
|
||||
// idle-disconnect won't burn the attempt.
|
||||
if rec.Tracker != nil {
|
||||
bctx, bcancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if _, err := rec.Tracker.Exec(bctx, "Broadcast [Panel] High memory detected. Server saving and restarting in 60 seconds."); err != nil {
|
||||
d.log.Warn("rcon broadcast failed during graceful memory restart", "instance_id", rec.InstanceID, "err", err)
|
||||
}
|
||||
bcancel()
|
||||
|
||||
sctx, scancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if _, err := rec.Tracker.Exec(sctx, "SaveWorld"); err != nil {
|
||||
d.log.Warn("rcon SaveWorld failed during graceful memory restart", "instance_id", rec.InstanceID, "err", err)
|
||||
}
|
||||
scancel()
|
||||
}
|
||||
|
||||
// Wait the warning window — players see the broadcast, engine
|
||||
// finishes writing the save. Sleeping in the goroutine is fine;
|
||||
// the stats loop continues to monitor for the 60 GB hard limit
|
||||
// in case RAM keeps climbing during the wait.
|
||||
time.Sleep(60 * time.Second)
|
||||
|
||||
// Cancel ancillary goroutines so they don't fight the bounce.
|
||||
// They'll be re-armed by watchExit when the restart settles.
|
||||
d.mu.Lock()
|
||||
if rec.TrackerCancel != nil {
|
||||
rec.TrackerCancel()
|
||||
rec.TrackerCancel = nil
|
||||
}
|
||||
if rec.LogCancel != nil {
|
||||
rec.LogCancel()
|
||||
rec.LogCancel = nil
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gracefulStopGracePeriod+30*time.Second)
|
||||
defer cancel()
|
||||
if err := d.runtime.Restart(ctx, rec.ContainerID, gracefulStopGracePeriod); err != nil {
|
||||
d.log.Error("graceful memory restart runtime failure", "instance_id", rec.InstanceID, "err", err)
|
||||
// Restart failed — surface to operator so they know to investigate.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] MEMORY-WARN restart failed: " + err.Error(),
|
||||
}},
|
||||
})
|
||||
}
|
||||
// Clear both the stopping and memGuardWarned latches so future
|
||||
// memory-warn fires can re-arm on this container's next life.
|
||||
rec.stopping.Store(false)
|
||||
rec.memGuardWarned.Store(false)
|
||||
}
|
||||
|
||||
// emergencyRestart bounces a runaway container outside the normal
|
||||
// handleStop path. Difference from handleStop: 30s grace (not 60s),
|
||||
// runs even when no operator request is in flight, and emits a clear
|
||||
// log line so the dashboard's Console tab shows WHY this happened.
|
||||
//
|
||||
// Uses Restart NOT Stop — see gracefulMemoryRestart for the rationale.
|
||||
// A plain Stop on `unless-stopped` containers parks them Exited until
|
||||
// manually started; the panel operator has no way to know they need to
|
||||
// click Start. Bouncing the container keeps the service available, lets
|
||||
// the engine recover into a fresh process with no leak state, and the
|
||||
// "[panel] AUTO-KILL" log line is preserved for after-the-fact diagnosis.
|
||||
// If the underlying leak is persistent, RAM will climb again and the
|
||||
// guard will fire again — that's the operator's signal to investigate.
|
||||
//
|
||||
// 2026-05-25: bumped from 5s → 30s grace. The 5s window guaranteed
|
||||
// SIGKILL mid-save when this fired during a SaveWorld, and that's
|
||||
// what was producing the FAtlasSaveManager corruption pattern. 30s
|
||||
// is enough for the engine to finish a save in flight without giving
|
||||
// up host-OOM protection.
|
||||
//
|
||||
// Idempotent via rec.stopping: the stats loop fires this from a goroutine
|
||||
// every 3s while RAM is over the limit, but the second+ invocations
|
||||
// short-circuit at the stopping check so we don't pile up Stop calls
|
||||
// against a container that's already heading down.
|
||||
func (d *Dispatcher) emergencyRestart(rec *instanceRecord, reason string) {
|
||||
if !rec.stopping.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
d.log.Warn("emergency restart", "instance_id", rec.InstanceID, "reason", reason)
|
||||
|
||||
// Surface to operator via log line + state update.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] AUTO-KILL: " + reason,
|
||||
}},
|
||||
})
|
||||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPING, 0, "auto-kill: "+reason)
|
||||
|
||||
// Cancel ancillary goroutines so they don't fight the bounce.
|
||||
// They'll be re-armed by watchExit when the restart settles.
|
||||
d.mu.Lock()
|
||||
if rec.TrackerCancel != nil {
|
||||
rec.TrackerCancel()
|
||||
rec.TrackerCancel = nil
|
||||
}
|
||||
if rec.LogCancel != nil {
|
||||
rec.LogCancel()
|
||||
rec.LogCancel = nil
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), emergencyStopGracePeriod+30*time.Second)
|
||||
defer cancel()
|
||||
if err := d.runtime.Restart(ctx, rec.ContainerID, emergencyStopGracePeriod); err != nil {
|
||||
d.log.Error("emergency restart runtime failure", "instance_id", rec.InstanceID, "err", err)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] AUTO-KILL restart failed: " + err.Error(),
|
||||
}},
|
||||
})
|
||||
}
|
||||
// Clear both latches so future signals can re-arm on the bounced container.
|
||||
rec.stopping.Store(false)
|
||||
rec.memGuardWarned.Store(false)
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/updater"
|
||||
"github.com/dbledeez/panel/pkg/steamvdf"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Update-available check (WI-14). CHECK-ONLY: reads the installed
|
||||
// buildid from the instance's Steam appmanifest ACF and the latest
|
||||
// buildid for its branch via `steamcmd +app_info_print` — it never
|
||||
// starts an update or touches the install.
|
||||
|
||||
// appInfoCacheTTL is how long a fetched branch map is reused before a
|
||||
// fresh steamcmd run. app_info runs cost a sidecar spin-up (~10-30 s),
|
||||
// so checks across many instances of the same game share one fetch.
|
||||
const appInfoCacheTTL = 15 * time.Minute
|
||||
|
||||
type appInfoCacheEntry struct {
|
||||
branches map[string]string
|
||||
fetchedAt time.Time
|
||||
}
|
||||
|
||||
var (
|
||||
appInfoCacheMu sync.Mutex
|
||||
appInfoCache = map[string]appInfoCacheEntry{} // key: app_id
|
||||
// appInfoFetchMu single-flights concurrent fetches per process; a
|
||||
// coarse lock is fine — checks are rare and operator-initiated.
|
||||
appInfoFetchMu sync.Mutex
|
||||
)
|
||||
|
||||
func (d *Dispatcher) handleUpdateCheck(corrID string, req *panelv1.UpdateCheckRequest) {
|
||||
fail := func(msg string) {
|
||||
d.sendUpdateCheckResult(corrID, &panelv1.UpdateCheckResult{
|
||||
InstanceId: req.InstanceId, Error: msg,
|
||||
})
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
fail(err.Error())
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
fail("module not in registry")
|
||||
return
|
||||
}
|
||||
// Same provider selection as handleUpdate: by id when supplied, else
|
||||
// first in the manifest.
|
||||
var spec *modulepkg_UpdateProvider
|
||||
for i := range manifest.UpdateProviders {
|
||||
p := &manifest.UpdateProviders[i]
|
||||
if req.ProviderId == "" || p.ID == req.ProviderId {
|
||||
spec = p
|
||||
break
|
||||
}
|
||||
}
|
||||
if spec == nil {
|
||||
fail(fmt.Sprintf("no provider %q on module %s", req.ProviderId, rec.ModuleID))
|
||||
return
|
||||
}
|
||||
if spec.Kind != "steamcmd" {
|
||||
fail(fmt.Sprintf("update check supports steamcmd providers only (provider %q is %q)", spec.ID, spec.Kind))
|
||||
return
|
||||
}
|
||||
if spec.AppID == "" {
|
||||
fail("provider has no app_id")
|
||||
return
|
||||
}
|
||||
|
||||
// The check spins a steamcmd sidecar (tens of seconds) — never block
|
||||
// the dispatch read loop.
|
||||
go func() {
|
||||
branch := spec.Beta
|
||||
branchKey := branch
|
||||
if branchKey == "" {
|
||||
branchKey = "public"
|
||||
}
|
||||
res := &panelv1.UpdateCheckResult{
|
||||
InstanceId: req.InstanceId,
|
||||
AppId: spec.AppID,
|
||||
Branch: branchKey,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// Installed side: appmanifest ACF under the provider's install
|
||||
// root. Prefer betakey/buildid straight from the manifest file.
|
||||
if acf, rerr := d.readInstalledACF(ctx, rec, spec.InstallPath, spec.AppID); rerr != nil {
|
||||
res.Error = "read appmanifest: " + rerr.Error()
|
||||
} else if m, perr := steamvdf.ParseAppManifest(string(acf)); perr != nil {
|
||||
res.Error = "parse appmanifest: " + perr.Error()
|
||||
} else {
|
||||
res.InstalledBuildid = m.BuildID
|
||||
if m.BetaKey != "" {
|
||||
// Trust the on-disk truth over the manifest's declared
|
||||
// beta — this is exactly the "bounced between branches"
|
||||
// case the check exists to expose.
|
||||
res.Branch = m.BetaKey
|
||||
branchKey = m.BetaKey
|
||||
}
|
||||
}
|
||||
|
||||
// Latest side: branch map via steamcmd app_info (15-min cache).
|
||||
branches, cached, ferr := d.latestBranches(ctx, rec.InstanceID, spec.AppID, req.Refresh)
|
||||
if ferr != nil {
|
||||
if res.Error != "" {
|
||||
res.Error += "; "
|
||||
}
|
||||
res.Error += "fetch latest: " + ferr.Error()
|
||||
} else {
|
||||
res.Cached = cached
|
||||
if bid, ok := branches[branchKey]; ok {
|
||||
res.LatestBuildid = bid
|
||||
} else if res.Error == "" {
|
||||
res.Error = fmt.Sprintf("branch %q not in Steam branches map (%d branches known)", branchKey, len(branches))
|
||||
}
|
||||
}
|
||||
|
||||
res.UpdateAvailable = res.InstalledBuildid != "" && res.LatestBuildid != "" &&
|
||||
res.InstalledBuildid != res.LatestBuildid
|
||||
d.log.Info("update check",
|
||||
"instance_id", req.InstanceId, "app_id", spec.AppID, "branch", branchKey,
|
||||
"installed", res.InstalledBuildid, "latest", res.LatestBuildid,
|
||||
"update_available", res.UpdateAvailable, "cached", res.Cached, "err", res.Error)
|
||||
d.sendUpdateCheckResult(corrID, res)
|
||||
}()
|
||||
}
|
||||
|
||||
// latestBranches returns the branch→buildid map for appID, from cache
|
||||
// when fresh (unless refresh), otherwise via a steamcmd sidecar run.
|
||||
func (d *Dispatcher) latestBranches(ctx context.Context, instanceID, appID string, refresh bool) (map[string]string, bool, error) {
|
||||
if !refresh {
|
||||
appInfoCacheMu.Lock()
|
||||
e, ok := appInfoCache[appID]
|
||||
appInfoCacheMu.Unlock()
|
||||
if ok && time.Since(e.fetchedAt) < appInfoCacheTTL {
|
||||
return e.branches, true, nil
|
||||
}
|
||||
}
|
||||
appInfoFetchMu.Lock()
|
||||
defer appInfoFetchMu.Unlock()
|
||||
// Re-check under the fetch lock — a concurrent check may have just
|
||||
// filled the cache while we waited.
|
||||
if !refresh {
|
||||
appInfoCacheMu.Lock()
|
||||
e, ok := appInfoCache[appID]
|
||||
appInfoCacheMu.Unlock()
|
||||
if ok && time.Since(e.fetchedAt) < appInfoCacheTTL {
|
||||
return e.branches, true, nil
|
||||
}
|
||||
}
|
||||
branches, err := updater.FetchAppInfoBranches(ctx, d.runtime, instanceID, appID, func(line string) {
|
||||
d.log.Debug("app_info", "instance_id", instanceID, "line", line)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
appInfoCacheMu.Lock()
|
||||
appInfoCache[appID] = appInfoCacheEntry{branches: branches, fetchedAt: time.Now()}
|
||||
appInfoCacheMu.Unlock()
|
||||
return branches, false, nil
|
||||
}
|
||||
|
||||
// readInstalledACF reads steamapps/appmanifest_<appID>.acf under the
|
||||
// provider's install root, using the same container-first/host-fallback
|
||||
// mechanics as the FsRead RPC (fs helper sidecar works on stopped
|
||||
// instances too).
|
||||
func (d *Dispatcher) readInstalledACF(ctx context.Context, rec *instanceRecord, installPath, appID string) ([]byte, error) {
|
||||
rel := path.Join("steamapps", fmt.Sprintf("appmanifest_%s.acf", appID))
|
||||
if d.useContainerOps(rec) {
|
||||
root := installPath
|
||||
if root == "" {
|
||||
root = rec.BrowseableRoot
|
||||
}
|
||||
abs, err := safeJoinAny(root, append(rootPaths(rec), root), path.Join(root, rel))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.runtime.CopyFileFromContainer(ctx, targetID, abs)
|
||||
}
|
||||
if rec.DataPath == "" {
|
||||
return nil, fmt.Errorf("no file storage available")
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return os.ReadFile(abs)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendUpdateCheckResult(corrID string, res *panelv1.UpdateCheckResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_UpdateCheckResult{UpdateCheckResult: res},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/updater"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Use a type alias so the file's handler signatures don't bleed the
|
||||
// modulepkg import into their exposed type names.
|
||||
type modulepkg_UpdateProvider = modulepkg.UpdateProvider
|
||||
|
||||
// registerUpdater stores the cancel func for the in-flight updater of
|
||||
// an instance. Returns (slot, true) when registration succeeds, or
|
||||
// (nil, false) when another updater is already running for the same
|
||||
// instance.
|
||||
//
|
||||
// We refuse instead of preempting because preempting races the docker
|
||||
// daemon's name cleanup: cancelling the prior context returns instantly,
|
||||
// but the prior steamcmd sidecar (e.g. panel-<id>-steamcmd-1) takes a
|
||||
// few seconds to actually exit and be removed. A new updater that
|
||||
// charges in immediately hits a "name already in use" Docker error and,
|
||||
// worse, leaves SteamCMD's staging directory mid-finalize (state 0x602)
|
||||
// — files stranded under <install>/steamapps/downloading/<appid>/.
|
||||
//
|
||||
// Concrete failure mode that prompted this guard:
|
||||
// - panelctl create palworld-test → auto-triggers first update.
|
||||
// - Operator (impatiently) runs panelctl update palworld-test 13s later.
|
||||
// - Old behavior cancelled the auto-update mid-validate, then the new
|
||||
// run failed name-conflict, then SteamCMD looped on exit-8 / 0x602.
|
||||
//
|
||||
// The delete path uses cancelUpdater (different entry), so this guard
|
||||
// doesn't block teardown.
|
||||
func (d *Dispatcher) registerUpdater(instanceID string, cancel context.CancelFunc) (*updaterSlot, bool) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if existing := d.updaters[instanceID]; existing != nil {
|
||||
return nil, false
|
||||
}
|
||||
slot := &updaterSlot{cancel: cancel}
|
||||
d.updaters[instanceID] = slot
|
||||
return slot, true
|
||||
}
|
||||
|
||||
// unregisterUpdater clears the registration ONLY IF the current entry
|
||||
// is still our slot — pointer compare guards against a later updater
|
||||
// having taken over.
|
||||
func (d *Dispatcher) unregisterUpdater(instanceID string, slot *updaterSlot) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.updaters[instanceID] == slot {
|
||||
delete(d.updaters, instanceID)
|
||||
}
|
||||
}
|
||||
|
||||
// cancelUpdater stops any in-flight updater for this instance and
|
||||
// removes the registration. Called from the delete path so the sidecar
|
||||
// gets torn down before we try to remove the volume.
|
||||
func (d *Dispatcher) cancelUpdater(instanceID string) {
|
||||
d.mu.Lock()
|
||||
slot := d.updaters[instanceID]
|
||||
delete(d.updaters, instanceID)
|
||||
d.mu.Unlock()
|
||||
if slot != nil && slot.cancel != nil {
|
||||
slot.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// Update handler. Controller sends ControllerEnvelope_Update; agent:
|
||||
//
|
||||
// 1. Looks up the instance + module manifest.
|
||||
// 2. Picks the update provider by id (or defaults to the first one).
|
||||
// 3. Emits a synchronous UpdateResult{accepted=true} so the panel can
|
||||
// un-gray the button and show "update started".
|
||||
// 4. Spawns a goroutine that runs the provider, streaming each progress
|
||||
// line as a LogLine{stream="update"}. Final sentinel line
|
||||
// "update:ok" or "update:error:<msg>" tells the UI we're done.
|
||||
//
|
||||
// Progress is not correlation-routed because the UI subscribes to the
|
||||
// normal event bus and picks up LogLines directly.
|
||||
|
||||
func (d *Dispatcher) handleUpdate(corrID string, req *panelv1.UpdateRequest) {
|
||||
d.handleUpdateWithHook(corrID, req, nil)
|
||||
}
|
||||
|
||||
// handleUpdateWithHook is the internal form that optionally fires onComplete
|
||||
// once the provider run finishes (success or failure). Used by the auto-
|
||||
// update-on-create path to chain a Start after the first-time download and
|
||||
// to clear UI state if the update errors out. The normal controller-driven
|
||||
// Update flow passes nil so the hook stays opt-in.
|
||||
func (d *Dispatcher) handleUpdateWithHook(corrID string, req *panelv1.UpdateRequest, onComplete func(err error)) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendUpdateResult(corrID, false, err.Error(), "", "")
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
d.sendUpdateResult(corrID, false, "module not in registry", "", "")
|
||||
return
|
||||
}
|
||||
|
||||
// Special pseudo-provider "_basemods": short-circuits to the third-
|
||||
// party base-mod installer (Allocs + PrismaCore) instead of running
|
||||
// the module's normal updater. Reuses the Update RPC plumbing so the
|
||||
// UI gets the same "updating" pulse + error surface as a real update.
|
||||
// Reasoning for the underscore prefix: real provider ids never start
|
||||
// with one (manifest schema convention), so collisions are impossible.
|
||||
if req.ProviderId == "_basemods" {
|
||||
d.sendUpdateResult(corrID, true, "", "_basemods", "basemods")
|
||||
d.log.Info("install-base-mods: kicked", "instance_id", req.InstanceId)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
err := d.installBaseMods(ctx, req.InstanceId, manifest, rec.DataPath)
|
||||
if err != nil {
|
||||
d.log.Error("install-base-mods: failed", "instance_id", req.InstanceId, "err", err)
|
||||
} else {
|
||||
d.log.Info("install-base-mods: done", "instance_id", req.InstanceId)
|
||||
}
|
||||
if onComplete != nil {
|
||||
onComplete(err)
|
||||
}
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
||||
// Pick provider: by id if supplied, else first in manifest.
|
||||
var spec *modulepkg_UpdateProvider
|
||||
for i := range manifest.UpdateProviders {
|
||||
p := &manifest.UpdateProviders[i]
|
||||
if req.ProviderId == "" || p.ID == req.ProviderId {
|
||||
spec = providerAdapter(p)
|
||||
break
|
||||
}
|
||||
}
|
||||
if spec == nil {
|
||||
d.sendUpdateResult(corrID, false, fmt.Sprintf("no provider %q on module %s", req.ProviderId, rec.ModuleID), "", "")
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := updater.Get(spec.Kind)
|
||||
if err != nil {
|
||||
d.sendUpdateResult(corrID, false, err.Error(), spec.ID, spec.Kind)
|
||||
return
|
||||
}
|
||||
|
||||
// Reject if another updater is already running on this instance —
|
||||
// see registerUpdater for why we refuse instead of preempting.
|
||||
//
|
||||
// Timeout bumped 30 → 90 min (2026-05-04) — Steam CDN can rate-limit
|
||||
// anonymous-login pulls down to 1-3 MB/s, and a 17 GB game (7DTD,
|
||||
// ARK SA) at 5 MB/s = 56 min, easily over 30. Killing the sidecar
|
||||
// at 30 min leaves a partial install + a confused operator. 90 min
|
||||
// covers the realistic worst case with margin; truly hung steamcmd
|
||||
// still gets killed eventually so we don't burn an updater slot
|
||||
// forever.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Minute)
|
||||
slot, ok := d.registerUpdater(req.InstanceId, cancel)
|
||||
if !ok {
|
||||
cancel()
|
||||
d.sendUpdateResult(corrID, false, "another update is already in progress for this instance", spec.ID, spec.Kind)
|
||||
d.log.Info("update rejected: already running", "instance_id", req.InstanceId, "provider_id", spec.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// Accept synchronously — run async.
|
||||
d.sendUpdateResult(corrID, true, "", spec.ID, spec.Kind)
|
||||
d.log.Info("update started", "instance_id", req.InstanceId, "provider_id", spec.ID, "kind", spec.Kind)
|
||||
|
||||
// Surface the in-flight update as a server-authoritative status.
|
||||
// Without this, the panel's status pill stayed at whatever it was
|
||||
// before the update (often "stopped" or worse, "crashed" if the
|
||||
// previous run had hard-exited) — operators saw "crashed" while
|
||||
// SteamCMD was actually downloading several GB. Emitting UPDATING
|
||||
// here gives every dashboard viewer the same accurate "installing"
|
||||
// pulse, regardless of which client kicked off the update.
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, "updating:"+spec.Kind)
|
||||
|
||||
go func() {
|
||||
defer cancel()
|
||||
defer d.unregisterUpdater(req.InstanceId, slot)
|
||||
sink := func(line string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "update",
|
||||
At: timestamppb.Now(),
|
||||
Line: line,
|
||||
}},
|
||||
})
|
||||
}
|
||||
sink(fmt.Sprintf("---- update started: provider=%s kind=%s ----", spec.ID, spec.Kind))
|
||||
uc := &updater.Context{
|
||||
InstanceID: rec.InstanceID,
|
||||
Manifest: manifest,
|
||||
ContainerID: rec.ContainerID,
|
||||
BrowseableRoot: rec.BrowseableRoot,
|
||||
DataPath: rec.DataPath,
|
||||
Runtime: d.runtime,
|
||||
Log: sink,
|
||||
// Steam login (if any) is forwarded by the controller for
|
||||
// modules that set `requires_steam_login`. Fields are empty
|
||||
// for anonymous-login apps — steamcmd.go falls back to that.
|
||||
SteamUsername: req.SteamUsername,
|
||||
SteamPassword: req.SteamPassword,
|
||||
}
|
||||
// ctx + cancel + slot were registered before the goroutine launched
|
||||
// (see registerUpdater above). The defer at the top of the goroutine
|
||||
// handles cleanup; the dispatcher's delete path can still abort us
|
||||
// via cancelUpdater.
|
||||
if err := prov.Update(ctx, uc, spec); err != nil {
|
||||
sink(fmt.Sprintf("update:error: %s", err.Error()))
|
||||
d.log.Warn("update failed", "instance_id", req.InstanceId, "err", err)
|
||||
// Clear the UPDATING pill — surface the failure as CRASHED with
|
||||
// a structured detail so operators see "install_failed: …" in
|
||||
// the UI rather than the pill being stuck on UPDATING forever.
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, -1, "install_failed: "+err.Error())
|
||||
if onComplete != nil {
|
||||
onComplete(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
sink("update:ok")
|
||||
d.log.Info("update complete", "instance_id", req.InstanceId, "provider_id", spec.ID)
|
||||
// Update succeeded — drop UPDATING back to STOPPED. handleStart's
|
||||
// own state emission supersedes if an auto-start follows. Without
|
||||
// this, a manually-triggered update left the pill on UPDATING
|
||||
// indefinitely until the next start/stop cycle.
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "installed")
|
||||
if onComplete != nil {
|
||||
onComplete(nil)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendUpdateResult(corrID string, accepted bool, errMsg, providerID, providerKind string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_UpdateResult{
|
||||
UpdateResult: &panelv1.UpdateResult{
|
||||
Accepted: accepted,
|
||||
Error: errMsg,
|
||||
ProviderId: providerID,
|
||||
ProviderKind: providerKind,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// providerAdapter is a pass-through that pins us to the modulepkg type
|
||||
// without needing to add another import alias at the top of this file.
|
||||
func providerAdapter(p *modulepkg_UpdateProvider) *modulepkg_UpdateProvider { return p }
|
||||
@@ -0,0 +1,331 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// tryWarmSeedFromSibling looks for another instance of the same module
|
||||
// already on this agent with a populated install volume; if found, copies
|
||||
// that volume verbatim into the new instance's install volume via a
|
||||
// throwaway debian sidecar. Saves operators 30+ minutes of redundant
|
||||
// SteamCMD downloads when standing up a second instance of a game.
|
||||
//
|
||||
// Returns (true, nil) on a successful seed — caller should skip the
|
||||
// updater and treat the install as already-complete.
|
||||
// Returns (false, nil) when there's no candidate sibling — caller falls
|
||||
// through to the normal updater.
|
||||
// Returns (false, err) on operational failure of the seed itself —
|
||||
// caller should also fall through (we'd rather over-download than leave
|
||||
// the user without a working install).
|
||||
//
|
||||
// Symlink handling: install volumes typically contain symlinks back into
|
||||
// the per-instance saves volume (e.g. /game/serverconfig.xml ->
|
||||
// /game-saves/serverconfig.xml on 7DTD, set up by the entrypoint at first
|
||||
// boot). Those symlinks would be DANGLING in the new instance because
|
||||
// the new saves volume is empty. We strip broken symlinks after copy
|
||||
// (`find -xtype l -delete`) so the entrypoint sees a clean state and
|
||||
// recreates them on its first boot.
|
||||
//
|
||||
// Saves bootstrap: if the sibling's saves volume contains a
|
||||
// serverconfig.xml-style file at a known location, we copy it to the
|
||||
// new saves volume so the entrypoint has a starting config (operator
|
||||
// can edit afterwards). Otherwise the entrypoint falls back to the
|
||||
// game's shipped default. Done per-module by the seedSavesPaths list.
|
||||
func (d *Dispatcher) tryWarmSeedFromSibling(ctx context.Context, newInstanceID string, manifest *modulepkg.Manifest, newDataPath string) (bool, error) {
|
||||
if manifest == nil || len(manifest.UpdateProviders) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
provider := manifest.UpdateProviders[0]
|
||||
installPath := provider.InstallPath
|
||||
if installPath == "" {
|
||||
installPath = manifest.Runtime.Docker.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Find a sibling: same agent, same module, NOT this instance, with a
|
||||
// container that exists (so we know the volume was at least populated
|
||||
// once). Prefer a stopped or running instance over a half-baked one.
|
||||
//
|
||||
// VERSION SAFETY: only reuse an install from a sibling on the SAME Steam
|
||||
// branch as this new instance. Copying e.g. a 3.0 (latest_experimental)
|
||||
// install into a server the operator created as 2.6 — or vice versa —
|
||||
// produces a binary/world mismatch that corrupts the save. The new
|
||||
// instance's record was tagged with its branch at Create (resolveCreateBranch),
|
||||
// before this runs. Siblings predating the feature have an empty branch and
|
||||
// are treated as the module's DEFAULT branch (branchOrDefault) — for 7DTD
|
||||
// that's v2.6, so the existing 2.6 fleet is still a valid seed source for a
|
||||
// new 2.6 server.
|
||||
d.mu.Lock()
|
||||
wantBranch := branchOrDefault("", manifest)
|
||||
if nr, ok := d.instances[newInstanceID]; ok {
|
||||
wantBranch = branchOrDefault(nr.Branch, manifest)
|
||||
}
|
||||
var srcID, srcDataPath string
|
||||
for id, rec := range d.instances {
|
||||
if id == newInstanceID {
|
||||
continue
|
||||
}
|
||||
if rec.ModuleID != manifest.ID {
|
||||
continue
|
||||
}
|
||||
if branchOrDefault(rec.Branch, manifest) != wantBranch {
|
||||
continue // different game version — not a safe seed source
|
||||
}
|
||||
// Best-effort: if we have multiple on this branch, the first one wins.
|
||||
// Could be improved later by picking the most-recently-updated.
|
||||
srcID = id
|
||||
srcDataPath = rec.DataPath
|
||||
break
|
||||
}
|
||||
d.mu.Unlock()
|
||||
if srcID == "" {
|
||||
d.log.Debug("warm-seed: no same-branch sibling instance found", "module", manifest.ID, "instance", newInstanceID, "want_branch", wantBranch)
|
||||
return false, nil
|
||||
}
|
||||
d.log.Info("warm-seed: matched sibling on branch", "module", manifest.ID, "instance", newInstanceID, "src", srcID, "branch", wantBranch)
|
||||
|
||||
srcVolumes := agentmodule.ResolveVolumes(manifest, srcID, srcDataPath)
|
||||
dstVolumes := agentmodule.ResolveVolumes(manifest, newInstanceID, newDataPath)
|
||||
|
||||
srcGameVol := volumeNameForPath(srcVolumes, installPath)
|
||||
dstGameVol := volumeNameForPath(dstVolumes, installPath)
|
||||
if srcGameVol == "" || dstGameVol == "" {
|
||||
return false, fmt.Errorf("warm-seed: install_path %q not in module volumes", installPath)
|
||||
}
|
||||
|
||||
// Find a "saves-style" sibling volume to bootstrap from. Heuristic:
|
||||
// any non-install named volume whose container path contains "save"
|
||||
// or "data". Most modules have one. If none, we still seed the install
|
||||
// volume; the entrypoint may still need to bootstrap saves itself.
|
||||
srcSavesVol, srcSavesPath := findSavesVolume(srcVolumes, installPath)
|
||||
dstSavesVol, _ := findSavesVolume(dstVolumes, installPath)
|
||||
|
||||
// Build the seed sidecar. debian:12-slim is small, has bash + find +
|
||||
// cp + mkdir, and is already cached on every panel-agent host (the
|
||||
// fs-helper sidecar uses it).
|
||||
// Why a tar pipe instead of cp -a: we want fine-grained exclusions for
|
||||
// per-instance state that shouldn't be inherited from the sibling.
|
||||
//
|
||||
// Mods/ split:
|
||||
// BASE mods (TFP_*, 0_TFP_*, Allocs_*, PrismaCore, xMarkers) are
|
||||
// load-bearing — TFP_Harmony is the runtime patcher, the server
|
||||
// won't even boot without it; Allocs is the live map renderer the
|
||||
// panel maps tab depends on; PrismaCore is the staff command surface
|
||||
// refugebotserver expects. We KEEP these.
|
||||
//
|
||||
// CUSTOM mods (AGF-VP-*, RefugeBot, AsylumRoboticInbox, anything
|
||||
// else) are server-specific — operators usually want a clean slate
|
||||
// on a new server so the panel's mod manager / their own upload
|
||||
// decides what's installed. We DROP these.
|
||||
//
|
||||
// steamapps/userdata, steamapps/downloading: per-Steam-account state
|
||||
// + half-fetched chunks. Always excluded.
|
||||
//
|
||||
// Approach: tar everything EXCEPT Mods + downloading + userdata, then
|
||||
// selectively copy back the base-mod entries from the sibling.
|
||||
// The Mods/ split + base-mod completeness gate below are 7DTD-SPECIFIC:
|
||||
// TFP_Harmony/Allocs/PrismaCore are load-bearing 7DTD server mods, and the
|
||||
// "missing base mods → exit 42" check re-triggers a full SteamCMD validate
|
||||
// to refetch them. Every OTHER module (palworld, soulmask, valheim, …) has
|
||||
// no Mods/ dir at all, so running this block against them ALWAYS hit the
|
||||
// completeness gate and exit-42'd — turning a valid sibling seed into a
|
||||
// spurious "incomplete sibling" failure. For non-7DTD modules we copy the
|
||||
// whole install verbatim (minus per-instance Steam state) and skip the gate.
|
||||
//
|
||||
// Root cause of the 2026-07-10 gamehost outage: Palworld + Soulmask checkouts
|
||||
// warm-seeded from a sibling, hit this 7DTD gate, exit-42'd, and fell through
|
||||
// to a fresh download — which was then canceled by a racing env-config
|
||||
// recreate (fixed separately in the controller). See changelog 2026-07-10.
|
||||
seedScript := `set -e
|
||||
echo "warm-seed: copying install volume contents (excl. Mods/, steamapps/userdata, steamapps/downloading)"
|
||||
find /dst-game -mindepth 1 -delete
|
||||
( cd /src-game && tar --exclude='./Mods' --exclude='./steamapps/userdata' --exclude='./steamapps/downloading' -cf - . ) | ( cd /dst-game && tar -xf - )
|
||||
`
|
||||
if manifest.ID == "7dtd" {
|
||||
seedScript += `mkdir -p /dst-game/Mods
|
||||
echo "warm-seed: copying BASE mods (TFP_*, Allocs_*, PrismaCore, Xample_MarkersMod) — load-bearing for the server"
|
||||
for mod in /src-game/Mods/TFP_* /src-game/Mods/0_TFP_* /src-game/Mods/Allocs_* /src-game/Mods/PrismaCore /src-game/Mods/Xample_* /src-game/Mods/*Markers* /src-game/Mods/xMarkers /src-game/Mods/xmarkers; do
|
||||
case "$mod" in *"*"*) continue ;; esac
|
||||
if [ -e "$mod" ]; then
|
||||
cp -a "$mod" /dst-game/Mods/
|
||||
echo " + $(basename $mod)"
|
||||
fi
|
||||
done
|
||||
`
|
||||
}
|
||||
seedScript += `echo "warm-seed: stripping dangling symlinks (saves-volume references)"
|
||||
find /dst-game -xtype l -delete
|
||||
`
|
||||
if manifest.ID == "7dtd" {
|
||||
seedScript += `# Completeness check (7DTD only) — fail the warmseed if any LOAD-BEARING base
|
||||
# mod is missing on dst. The caller falls back to a full steamcmd validate which
|
||||
# will re-fetch the TFP/xMarkers mods from the dedicated-server depot.
|
||||
missing=""
|
||||
for required in 0_TFP_Harmony TFP_WebServer; do
|
||||
[ -d "/dst-game/Mods/$required" ] || missing="$missing $required"
|
||||
done
|
||||
if [ -n "$missing" ]; then
|
||||
echo "warm-seed: incomplete sibling — missing base mods:$missing"
|
||||
exit 42
|
||||
fi
|
||||
`
|
||||
}
|
||||
if srcSavesVol != "" && dstSavesVol != "" {
|
||||
// Copy ONLY the rendered config file (and the .local/share parent
|
||||
// for HOME-based games). Don't copy the world data — the user wants
|
||||
// a fresh save on this new instance, not the sibling's.
|
||||
seedScript += `echo "warm-seed: bootstrapping saves volume from sibling"
|
||||
[ -s /src-saves/serverconfig.xml ] && cp /src-saves/serverconfig.xml /dst-saves/serverconfig.xml || true
|
||||
mkdir -p /dst-saves/.local/share 2>/dev/null || true
|
||||
# 7DTD HOME tree — entrypoint mkdir's the deeper path on boot, but creating
|
||||
# the parent here lets the install symlink resolve to a valid empty dir
|
||||
# even before the first start.
|
||||
for d in /src-saves/.local/share/7DaysToDie /src-saves/.local/share/Empyrion; do
|
||||
if [ -d "$d" ]; then
|
||||
mkdir -p "/dst-saves/$(echo $d | sed 's|/src-saves/||')"
|
||||
fi
|
||||
done
|
||||
`
|
||||
}
|
||||
seedScript += `echo SEED_OK`
|
||||
|
||||
sidecarID := fmt.Sprintf("%s-warmseed", newInstanceID)
|
||||
volumes := []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true},
|
||||
{Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"},
|
||||
}
|
||||
if srcSavesVol != "" && dstSavesVol != "" {
|
||||
volumes = append(volumes,
|
||||
runtime.VolumeSpec{Type: "volume", VolumeName: srcSavesVol, ContainerPath: "/src-saves", ReadOnly: true},
|
||||
runtime.VolumeSpec{Type: "volume", VolumeName: dstSavesVol, ContainerPath: "/dst-saves"},
|
||||
)
|
||||
}
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: "debian:12-slim",
|
||||
Command: []string{"bash", "-c", seedScript},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
|
||||
// Surface what's happening — without this the card just shows a static
|
||||
// "installing" pulse with no console output during a multi-GB sibling copy,
|
||||
// which looks hung.
|
||||
d.sendInstanceState(newInstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, fmt.Sprintf("pulling game files from existing install (%s) — no SteamCMD download needed", srcID))
|
||||
d.log.Info("warm-seed: starting sidecar", "src", srcID, "dst", newInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol, "saves_seed", srcSavesPath != "")
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("warm-seed: create sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return false, fmt.Errorf("warm-seed: start sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.log.Info("warm-seed log", "instance_id", newInstanceID, "line", line)
|
||||
// Mirror the sidecar's friendly progress echoes ("warm-seed: copying
|
||||
// install volume…", "+ <mod>") onto the instance console so the
|
||||
// operator can watch the "pulling from existing install" progress.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: newInstanceID, Stream: "install", At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("warm-seed: wait sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return false, fmt.Errorf("warm-seed: sidecar exited %d", exitCode)
|
||||
}
|
||||
d.log.Info("warm-seed: success", "src", srcID, "dst", newInstanceID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// volumeNameForPath returns the named-volume name whose container path
|
||||
// matches `path` exactly, or "" if no match. Bind mounts are skipped —
|
||||
// we only seed named volumes (host path is operator-managed).
|
||||
func volumeNameForPath(volumes []runtime.VolumeSpec, path string) string {
|
||||
for _, v := range volumes {
|
||||
if v.Type == "volume" && v.VolumeName != "" && v.ContainerPath == path {
|
||||
return v.VolumeName
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// findSavesVolume returns the (volumeName, containerPath) of the first
|
||||
// named-volume whose container path looks like a saves/data dir, ignoring
|
||||
// the install volume. Used to bootstrap the new instance's saves with a
|
||||
// rendered config from the sibling.
|
||||
func findSavesVolume(volumes []runtime.VolumeSpec, installPath string) (string, string) {
|
||||
for _, v := range volumes {
|
||||
if v.Type != "volume" || v.VolumeName == "" {
|
||||
continue
|
||||
}
|
||||
if v.ContainerPath == installPath {
|
||||
continue
|
||||
}
|
||||
lower := strings.ToLower(v.ContainerPath)
|
||||
if strings.Contains(lower, "save") || strings.Contains(lower, "data") {
|
||||
return v.VolumeName, v.ContainerPath
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// normalizeBranch collapses a provider `beta` value into a stable warm-seed
|
||||
// matching key: the empty beta (the moving public/stable branch) becomes
|
||||
// "public"; any explicit beta (e.g. "v2.6", "latest_experimental") is returned
|
||||
// trimmed as-is. Two installs are warm-seed compatible iff their normalized
|
||||
// branches are equal.
|
||||
func normalizeBranch(beta string) string {
|
||||
b := strings.TrimSpace(beta)
|
||||
if b == "" {
|
||||
return "public"
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// branchOrDefault returns the normalized branch for a record's Branch value,
|
||||
// treating an empty value (instances created before branch-tagging existed) as
|
||||
// the module's DEFAULT branch — UpdateProviders[0].Beta normalized. For 7DTD
|
||||
// the default is v2.6, so the pre-existing 2.6 fleet remains a valid warm-seed
|
||||
// source for a new 2.6 server. Returns "public" when the manifest has no
|
||||
// providers.
|
||||
func branchOrDefault(branch string, m *modulepkg.Manifest) string {
|
||||
if strings.TrimSpace(branch) != "" {
|
||||
return normalizeBranch(branch)
|
||||
}
|
||||
if m != nil && len(m.UpdateProviders) > 0 {
|
||||
return normalizeBranch(m.UpdateProviders[0].Beta)
|
||||
}
|
||||
return "public"
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Package module (agent-side) resolves a manifest + InstanceCreate RPC
|
||||
// into a runtime.InstanceSpec ready for the Docker / host runtime to launch.
|
||||
package module
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// ResolveDocker produces an InstanceSpec for Docker-mode launch of the given
|
||||
// module manifest and RPC create request. It substitutes $DATA_PATH in volume
|
||||
// targets and maps declared ports against the request's port overrides.
|
||||
func ResolveDocker(manifest *modulepkg.Manifest, req *panelv1.InstanceCreate) (runtime.InstanceSpec, error) {
|
||||
if manifest == nil {
|
||||
return runtime.InstanceSpec{}, errors.New("manifest is nil")
|
||||
}
|
||||
if manifest.Runtime.Docker == nil {
|
||||
return runtime.InstanceSpec{}, fmt.Errorf("module %q has no docker runtime", manifest.ID)
|
||||
}
|
||||
if req.DataPath == "" {
|
||||
return runtime.InstanceSpec{}, errors.New("req.data_path is required")
|
||||
}
|
||||
|
||||
docker := manifest.Runtime.Docker
|
||||
|
||||
env := map[string]string{}
|
||||
for k, v := range docker.Env {
|
||||
env[k] = expand(v, req.DataPath)
|
||||
}
|
||||
// Per-instance config_values layer: if the operator (or controller
|
||||
// feature code) set a value for an env key the module already
|
||||
// declares, use that value instead of the manifest default. Only
|
||||
// keys that ALREADY exist in docker.Env get overridden — we don't
|
||||
// let config_values smuggle in unrelated env vars. Used today by
|
||||
// the ARK cluster feature to override CLUSTER_ID per-instance.
|
||||
for k, v := range req.ConfigValues {
|
||||
if _, ok := env[k]; ok {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
volumes := make([]runtime.VolumeSpec, 0, len(docker.Volumes))
|
||||
for _, v := range docker.Volumes {
|
||||
vs := runtime.VolumeSpec{
|
||||
ContainerPath: v.Container,
|
||||
ReadOnly: v.ReadOnly,
|
||||
}
|
||||
kind := v.Type
|
||||
if kind == "" {
|
||||
// Infer from which field was populated.
|
||||
if v.Name != "" {
|
||||
kind = "volume"
|
||||
} else {
|
||||
kind = "bind"
|
||||
}
|
||||
}
|
||||
switch kind {
|
||||
case "volume":
|
||||
vs.Type = "volume"
|
||||
vs.VolumeName = expandInstance(v.Name, req.InstanceId)
|
||||
default:
|
||||
vs.Type = "bind"
|
||||
vs.HostPath = expand(v.Target, req.DataPath)
|
||||
}
|
||||
// Apply per-instance mount override if the controller asked for
|
||||
// one by container path. An absolute path (Unix `/...` or
|
||||
// Windows `C:\...`) is a bind mount host path; anything else is
|
||||
// a Docker named volume. Used by the ARK cluster feature to
|
||||
// swap the default per-instance cluster volume for a shared
|
||||
// host-visible `arkcluster/<id>/` bind mount. `$AGENT_DATA_ROOT`
|
||||
// has already been expanded by handleCreate.
|
||||
if ov, ok := req.MountOverrides[v.Container]; ok && ov != "" {
|
||||
if filepath.IsAbs(ov) {
|
||||
vs.Type = "bind"
|
||||
vs.HostPath = ov
|
||||
vs.VolumeName = ""
|
||||
} else {
|
||||
vs.Type = "volume"
|
||||
vs.VolumeName = ov
|
||||
vs.HostPath = ""
|
||||
}
|
||||
}
|
||||
volumes = append(volumes, vs)
|
||||
}
|
||||
|
||||
portOverrides := indexPortOverrides(req.Ports)
|
||||
ports := make([]runtime.PortSpec, 0, len(manifest.Ports))
|
||||
for _, p := range manifest.Ports {
|
||||
containerPort := uint16(p.Default)
|
||||
hostPort := containerPort
|
||||
if ov, ok := portOverrides[p.Name]; ok {
|
||||
if ov.ContainerPort != 0 {
|
||||
containerPort = uint16(ov.ContainerPort)
|
||||
}
|
||||
if ov.HostPort != 0 {
|
||||
hostPort = uint16(ov.HostPort)
|
||||
}
|
||||
}
|
||||
// Internal ports (e.g. 7DTD's telnet RCON on 8081) must still be
|
||||
// reachable from the Target agent — it runs on the same host — so we
|
||||
// bind them to 127.0.0.1 instead of 0.0.0.0. Public ports bind to
|
||||
// all interfaces (HostIP left empty).
|
||||
hostIP := ""
|
||||
if p.Internal {
|
||||
hostIP = "127.0.0.1"
|
||||
}
|
||||
ports = append(ports, runtime.PortSpec{
|
||||
ContainerPort: containerPort,
|
||||
HostPort: hostPort,
|
||||
Proto: strings.ToLower(p.Proto),
|
||||
HostIP: hostIP,
|
||||
})
|
||||
}
|
||||
|
||||
// Preserve nil when the manifest doesn't override — passing []string{} to
|
||||
// Docker ContainerCreate replaces the image's default entrypoint/cmd with
|
||||
// empty, which errors out as "no command specified".
|
||||
var entrypoint, command []string
|
||||
if len(docker.Entrypoint) > 0 {
|
||||
entrypoint = append([]string{}, docker.Entrypoint...)
|
||||
}
|
||||
if len(docker.Command) > 0 {
|
||||
command = append([]string{}, docker.Command...)
|
||||
}
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: req.InstanceId,
|
||||
Image: docker.Image,
|
||||
Entrypoint: entrypoint,
|
||||
Command: command,
|
||||
Env: env,
|
||||
User: docker.User,
|
||||
Volumes: volumes,
|
||||
Ports: ports,
|
||||
NetworkMode: docker.NetworkMode,
|
||||
BuildContext: manifest.Dir, // module's directory holds the Dockerfile
|
||||
}
|
||||
// Auto-enable OpenStdin whenever the module's RCON adapter is stdio —
|
||||
// it's the one feature the adapter needs from the runtime, and making
|
||||
// it implicit saves a second knob every stdio-based module would have
|
||||
// to remember to set.
|
||||
if manifest.RCON != nil && manifest.RCON.Adapter == "stdio" {
|
||||
spec.OpenStdin = true
|
||||
}
|
||||
if lim := req.Limits; lim != nil {
|
||||
spec.Resources = runtime.ResourceLimits{
|
||||
CPUShares: lim.CpuShares,
|
||||
MemoryBytes: lim.MemBytes,
|
||||
}
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
// expand substitutes $DATA_PATH in the given string.
|
||||
func expand(s, dataPath string) string {
|
||||
return strings.ReplaceAll(s, "$DATA_PATH", dataPath)
|
||||
}
|
||||
|
||||
// expandInstance substitutes $INSTANCE_ID in the given string. Used for
|
||||
// named-volume templating so each instance gets its own volume.
|
||||
func expandInstance(s, instanceID string) string {
|
||||
return strings.ReplaceAll(s, "$INSTANCE_ID", instanceID)
|
||||
}
|
||||
|
||||
// ResolveVolumes translates manifest volumes into runtime.VolumeSpec form
|
||||
// for a specific instance. Exposed so sidecar launchers (SteamCMD updater,
|
||||
// future backup-runner, etc.) can mount the same volumes the main
|
||||
// container uses, putting files where the main container expects them.
|
||||
func ResolveVolumes(manifest *modulepkg.Manifest, instanceID, dataPath string) []runtime.VolumeSpec {
|
||||
if manifest == nil || manifest.Runtime.Docker == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]runtime.VolumeSpec, 0, len(manifest.Runtime.Docker.Volumes))
|
||||
for _, v := range manifest.Runtime.Docker.Volumes {
|
||||
vs := runtime.VolumeSpec{
|
||||
ContainerPath: v.Container,
|
||||
ReadOnly: v.ReadOnly,
|
||||
}
|
||||
kind := v.Type
|
||||
if kind == "" {
|
||||
if v.Name != "" {
|
||||
kind = "volume"
|
||||
} else {
|
||||
kind = "bind"
|
||||
}
|
||||
}
|
||||
switch kind {
|
||||
case "volume":
|
||||
vs.Type = "volume"
|
||||
vs.VolumeName = expandInstance(v.Name, instanceID)
|
||||
default:
|
||||
vs.Type = "bind"
|
||||
vs.HostPath = expand(v.Target, dataPath)
|
||||
}
|
||||
out = append(out, vs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// indexPortOverrides builds a by-name lookup of PortMap entries from the RPC.
|
||||
func indexPortOverrides(ports []*panelv1.PortMap) map[string]*panelv1.PortMap {
|
||||
out := map[string]*panelv1.PortMap{}
|
||||
for _, p := range ports {
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
out[p.Name] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BattlEyeDialer speaks the BattlEye RCON protocol used by DayZ, Arma 2/3,
|
||||
// Squad, Rising Storm 2, and a handful of other BE-protected games.
|
||||
// Protocol reference: https://www.battleye.com/downloads/BERConProtocol.txt
|
||||
//
|
||||
// Wire format (UDP, all frames start with "BE" + CRC32 + 0xFF + type + payload):
|
||||
//
|
||||
// 0x42 'B'
|
||||
// 0x45 'E'
|
||||
// uint32 crc32 — little-endian CRC32 of bytes from offset 6 onward
|
||||
// 0xFF — packet-header sentinel
|
||||
// uint8 type — 0x00 login, 0x01 command, 0x02 server message
|
||||
// payload — varies by type
|
||||
//
|
||||
// Auth (type 0x00): client sends its admin password; server replies with
|
||||
// a single-byte 0x01 = success, 0x00 = failure.
|
||||
//
|
||||
// Commands (type 0x01): client sends a 1-byte sequence id + the command
|
||||
// text; server echoes the same seq id with the response text.
|
||||
//
|
||||
// Server-pushed messages (type 0x02): the server streams chat/join/leave
|
||||
// events. Client MUST ack each one by echoing the seq id with an empty
|
||||
// payload or the server will disconnect.
|
||||
type BattlEyeDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (BattlEyeDialer) Name() string { return "be_rcon" }
|
||||
|
||||
const (
|
||||
bePacketHeader = 0xFF
|
||||
bePacketLogin = 0x00
|
||||
bePacketCmd = 0x01
|
||||
bePacketServer = 0x02
|
||||
)
|
||||
|
||||
// Dial performs the login handshake and returns a ready-to-use BE RCON client.
|
||||
func (BattlEyeDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("be rcon dial: host and port are required")
|
||||
}
|
||||
connectTimeout := opts.ConnectTimeout
|
||||
if connectTimeout == 0 {
|
||||
connectTimeout = 5 * time.Second
|
||||
}
|
||||
addr := &net.UDPAddr{IP: net.ParseIP(opts.Host), Port: opts.Port}
|
||||
if addr.IP == nil {
|
||||
// Allow hostnames too.
|
||||
resolved, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", opts.Host, opts.Port))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve %s: %w", opts.Host, err)
|
||||
}
|
||||
addr = resolved
|
||||
}
|
||||
conn, err := net.DialUDP("udp", nil, addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("be rcon dial %s:%d: %w", opts.Host, opts.Port, err)
|
||||
}
|
||||
|
||||
c := &beClient{conn: conn, addr: addr}
|
||||
if err := c.login(opts.Password, connectTimeout); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
// Kick off the keep-alive goroutine — BattlEye disconnects clients
|
||||
// that don't send traffic for ~45s. We ping every 30s.
|
||||
go c.keepAlive()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type beClient struct {
|
||||
conn *net.UDPConn
|
||||
addr *net.UDPAddr
|
||||
|
||||
seqMu sync.Mutex
|
||||
seq uint8 // next command sequence id
|
||||
|
||||
execMu sync.Mutex
|
||||
closed atomic.Bool
|
||||
// stopKA fires when Close is called; keep-alive loop watches it.
|
||||
stopKA chan struct{}
|
||||
}
|
||||
|
||||
// login sends the auth packet and waits for a 1-byte success response.
|
||||
func (c *beClient) login(password string, timeout time.Duration) error {
|
||||
pkt := buildBEPacket(bePacketLogin, []byte(password))
|
||||
if _, err := c.conn.Write(pkt); err != nil {
|
||||
return fmt.Errorf("be rcon login write: %w", err)
|
||||
}
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(timeout))
|
||||
buf := make([]byte, 256)
|
||||
n, err := c.conn.Read(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("be rcon login read: %w", err)
|
||||
}
|
||||
payloadType, payload, ok := parseBEPacket(buf[:n])
|
||||
if !ok {
|
||||
return errors.New("be rcon login: malformed response")
|
||||
}
|
||||
if payloadType != bePacketLogin {
|
||||
return fmt.Errorf("be rcon login: unexpected packet type 0x%02x", payloadType)
|
||||
}
|
||||
if len(payload) < 1 {
|
||||
return errors.New("be rcon login: empty response payload")
|
||||
}
|
||||
if payload[0] != 0x01 {
|
||||
return errors.New("be rcon login: bad password")
|
||||
}
|
||||
c.stopKA = make(chan struct{})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exec runs one command and returns the server's response text. BattlEye
|
||||
// doesn't multi-frame short responses; long ones arrive as multiple packets
|
||||
// with the same seq id + a 1-byte 'part_count / part_index' header. We
|
||||
// concatenate parts in order.
|
||||
func (c *beClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
if c.closed.Load() {
|
||||
return "", errors.New("be rcon: client closed")
|
||||
}
|
||||
c.execMu.Lock()
|
||||
defer c.execMu.Unlock()
|
||||
|
||||
c.seqMu.Lock()
|
||||
seq := c.seq
|
||||
c.seq++
|
||||
c.seqMu.Unlock()
|
||||
|
||||
body := make([]byte, 0, len(cmd)+1)
|
||||
body = append(body, seq)
|
||||
body = append(body, []byte(cmd)...)
|
||||
pkt := buildBEPacket(bePacketCmd, body)
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
deadline = time.Now().Add(10 * time.Second)
|
||||
}
|
||||
_ = c.conn.SetDeadline(deadline)
|
||||
defer c.conn.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
|
||||
if _, err := c.conn.Write(pkt); err != nil {
|
||||
return "", fmt.Errorf("be rcon write: %w", err)
|
||||
}
|
||||
|
||||
// Collect response parts. We loop until we've got every declared part
|
||||
// for this seq, or an idle timeout fires with partial data.
|
||||
parts := map[uint8][]byte{}
|
||||
var totalParts uint8
|
||||
for {
|
||||
buf := make([]byte, 4096)
|
||||
n, err := c.conn.Read(buf)
|
||||
if err != nil {
|
||||
if len(parts) > 0 {
|
||||
// Return whatever we got — better than nothing.
|
||||
return reassembleBEParts(parts, totalParts), nil
|
||||
}
|
||||
return "", fmt.Errorf("be rcon read: %w", err)
|
||||
}
|
||||
pType, payload, ok := parseBEPacket(buf[:n])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if pType == bePacketServer {
|
||||
// Server-pushed event (chat/join/leave). Ack it + keep reading.
|
||||
if len(payload) >= 1 {
|
||||
ack := buildBEPacket(bePacketServer, []byte{payload[0]})
|
||||
_, _ = c.conn.Write(ack)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if pType != bePacketCmd || len(payload) < 1 || payload[0] != seq {
|
||||
continue
|
||||
}
|
||||
// Multi-part response format: [seq, 0x00, total_parts, part_index, ...body...]
|
||||
if len(payload) >= 4 && payload[1] == 0x00 {
|
||||
totalParts = payload[2]
|
||||
idx := payload[3]
|
||||
parts[idx] = payload[4:]
|
||||
if uint8(len(parts)) >= totalParts {
|
||||
return reassembleBEParts(parts, totalParts), nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Single-frame response: [seq, ...body...]
|
||||
return string(payload[1:]), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Close tears down the UDP connection + stops the keep-alive loop.
|
||||
func (c *beClient) Close() error {
|
||||
if !c.closed.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
}
|
||||
if c.stopKA != nil {
|
||||
close(c.stopKA)
|
||||
}
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// keepAlive sends a zero-length command every 30s to keep BE from dropping
|
||||
// the session for inactivity. The protocol explicitly documents this.
|
||||
func (c *beClient) keepAlive() {
|
||||
tick := time.NewTicker(30 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-c.stopKA:
|
||||
return
|
||||
case <-tick.C:
|
||||
if c.closed.Load() {
|
||||
return
|
||||
}
|
||||
c.execMu.Lock()
|
||||
c.seqMu.Lock()
|
||||
seq := c.seq
|
||||
c.seq++
|
||||
c.seqMu.Unlock()
|
||||
pkt := buildBEPacket(bePacketCmd, []byte{seq})
|
||||
_ = c.conn.SetWriteDeadline(time.Now().Add(2 * time.Second))
|
||||
_, _ = c.conn.Write(pkt)
|
||||
c.execMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- wire helpers ----
|
||||
|
||||
// buildBEPacket assembles a BE RCON UDP frame: "BE" + CRC32(body) + body,
|
||||
// where body = 0xFF + type + payload.
|
||||
func buildBEPacket(pType uint8, payload []byte) []byte {
|
||||
body := make([]byte, 0, 2+len(payload))
|
||||
body = append(body, bePacketHeader, pType)
|
||||
body = append(body, payload...)
|
||||
crc := crc32.ChecksumIEEE(body)
|
||||
out := make([]byte, 6+len(body))
|
||||
out[0], out[1] = 'B', 'E'
|
||||
binary.LittleEndian.PutUint32(out[2:6], crc)
|
||||
copy(out[6:], body)
|
||||
return out
|
||||
}
|
||||
|
||||
// parseBEPacket validates a "BE..." frame and returns (type, payload, ok).
|
||||
func parseBEPacket(buf []byte) (uint8, []byte, bool) {
|
||||
if len(buf) < 7 || buf[0] != 'B' || buf[1] != 'E' {
|
||||
return 0, nil, false
|
||||
}
|
||||
crc := binary.LittleEndian.Uint32(buf[2:6])
|
||||
if crc32.ChecksumIEEE(buf[6:]) != crc {
|
||||
// Corrupt packet — accept it anyway for lenience; some mods' BE
|
||||
// extensions are known to skip CRC. Emit a debug break here
|
||||
// later if we start seeing real-world issues.
|
||||
_ = crc
|
||||
}
|
||||
if buf[6] != bePacketHeader {
|
||||
return 0, nil, false
|
||||
}
|
||||
return buf[7], buf[8:], true
|
||||
}
|
||||
|
||||
// reassembleBEParts joins multi-frame command responses in order.
|
||||
func reassembleBEParts(parts map[uint8][]byte, total uint8) string {
|
||||
var b strings.Builder
|
||||
for i := uint8(0); i < total; i++ {
|
||||
b.Write(parts[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DockerExecDialer runs per command, opening
|
||||
// a fresh socket inside the container's network namespace each time and
|
||||
// closing it on completion. Fresh process, fresh fd, fresh TCP conn — nothing
|
||||
// is held open between calls.
|
||||
//
|
||||
// Why: ARK Survival Ascended's RCON listener wedges if a polling client holds
|
||||
// a long-lived TCP session — the engine accumulates CLOSE-WAIT sockets and
|
||||
// the listener thread itself dies after enough accumulation. The wedge is
|
||||
// not recoverable without restarting the engine. Per-command exec avoids the
|
||||
// wedge entirely because nothing persists between invocations.
|
||||
//
|
||||
// Trade-off: ~50–200ms of overhead per command for docker exec + rcon-cli
|
||||
// startup. Negligible at ARK SA's polling cadence (30s+); not appropriate
|
||||
// for high-frequency command paths (none exist in panel today).
|
||||
//
|
||||
// Requirements:
|
||||
// - Container has rcon-cli on PATH (acekorneya/asa_server: yes).
|
||||
// - Container env exposes the RCON password to the in-container shell, OR
|
||||
// the agent supplies it via DialOptions.Password (we pass it in).
|
||||
// - Agent process is in the docker group (the panel-agent systemd unit
|
||||
// already declares SupplementaryGroups=docker).
|
||||
type DockerExecDialer struct{}
|
||||
|
||||
func (DockerExecDialer) Name() string { return "docker_exec_rcon" }
|
||||
|
||||
func (DockerExecDialer) Dial(_ context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.ContainerID == "" {
|
||||
return nil, errors.New("docker_exec_rcon dial: ContainerID required")
|
||||
}
|
||||
port := ""
|
||||
if opts.Port != 0 {
|
||||
port = strconv.Itoa(opts.Port)
|
||||
}
|
||||
return &dockerExecClient{
|
||||
containerID: opts.ContainerID,
|
||||
password: opts.Password,
|
||||
port: port,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type dockerExecClient struct {
|
||||
containerID string
|
||||
password string
|
||||
port string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// ansiRe matches CSI escape sequences. rcon-cli emits ANSI color resets
|
||||
// after each response (\033[0m); strip them so the panel's parsers see
|
||||
// clean text.
|
||||
var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
|
||||
|
||||
// Exec runs and returns the cleaned stdout.
|
||||
func (c *dockerExecClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
args := []string{"exec", c.containerID, "rcon-cli", "--host", "localhost"}
|
||||
if c.port != "" {
|
||||
args = append(args, "--port", c.port)
|
||||
}
|
||||
if c.password != "" {
|
||||
args = append(args, "--password", c.password)
|
||||
}
|
||||
args = append(args, cmd)
|
||||
|
||||
dockerCmd := exec.CommandContext(ctx, "docker", args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
dockerCmd.Stdout = &stdout
|
||||
dockerCmd.Stderr = &stderr
|
||||
if err := dockerCmd.Run(); err != nil {
|
||||
errMsg := strings.TrimSpace(stderr.String())
|
||||
if errMsg == "" {
|
||||
errMsg = err.Error()
|
||||
}
|
||||
return "", fmt.Errorf("docker exec rcon-cli: %s", errMsg)
|
||||
}
|
||||
out := ansiRe.ReplaceAllString(stdout.String(), "")
|
||||
return strings.TrimSpace(out), nil
|
||||
}
|
||||
|
||||
// Close is a no-op — there's nothing held open between Exec calls.
|
||||
func (c *dockerExecClient) Close() error { return nil }
|
||||
@@ -0,0 +1,81 @@
|
||||
// Package rcon provides pluggable RCON adapters. Games speak different
|
||||
// dialects — Source RCON (Counter-Strike, Minecraft with Mojang auth),
|
||||
// telnet line protocol (7 Days to Die), HTTP admin APIs (some community
|
||||
// implementations), stdin-pipe for headless servers without a network
|
||||
// surface. The Agent picks an adapter based on the module manifest's
|
||||
// `rcon.adapter` field.
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client is a connected RCON session. Exec runs one command and returns its
|
||||
// raw text response. Clients must be safe for concurrent Exec calls — the
|
||||
// poll loop and a future ad-hoc-command path both need to call through the
|
||||
// same session.
|
||||
type Client interface {
|
||||
Exec(ctx context.Context, cmd string) (string, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// DialOptions configures one RCON connection attempt.
|
||||
type DialOptions struct {
|
||||
Host string
|
||||
Port int
|
||||
Password string
|
||||
ConnectTimeout time.Duration
|
||||
// ReadIdle is how long to wait with no new bytes before treating an Exec
|
||||
// response as complete. Telnet has no packet framing, so this heuristic
|
||||
// is needed; Source RCON overrides it to 0 (ignored).
|
||||
ReadIdle time.Duration
|
||||
|
||||
// ContainerID is the Docker container name or ID for stdio/exec-based
|
||||
// adapters that don't use TCP. Ignored by telnet/source_rcon.
|
||||
ContainerID string
|
||||
// Stdio is the backend that knows how to attach to a container's
|
||||
// stdin/stdout. Provided by the Docker runtime when the adapter is
|
||||
// "stdio". Nil for TCP adapters.
|
||||
Stdio StdioBackend
|
||||
}
|
||||
|
||||
// StdioBackend attaches to a running container's combined stdin/stdout/stderr
|
||||
// stream. Needed by the stdio adapter for games whose only admin console is
|
||||
// stdin (Valheim, Terraria, Barotrauma, Enshrouded, Sons Of The Forest).
|
||||
//
|
||||
// Returned ReadWriteCloser writes go to the container's stdin; reads are the
|
||||
// demux-multiplexed stdout+stderr. Close shuts down the hijacked connection.
|
||||
type StdioBackend interface {
|
||||
AttachStdio(ctx context.Context, containerID string) (io.ReadWriteCloser, error)
|
||||
}
|
||||
|
||||
// Dialer establishes RCON connections of a specific adapter kind.
|
||||
type Dialer interface {
|
||||
Dial(ctx context.Context, opts DialOptions) (Client, error)
|
||||
Name() string
|
||||
}
|
||||
|
||||
// DialerFor returns the dialer for the given adapter id, or an error if the
|
||||
// adapter is unknown. Adapter ids come directly from the module manifest's
|
||||
// `rcon.adapter` field.
|
||||
func DialerFor(adapter string) (Dialer, error) {
|
||||
switch adapter {
|
||||
case "telnet":
|
||||
return &TelnetDialer{}, nil
|
||||
case "source_rcon":
|
||||
return &SourceDialer{}, nil
|
||||
case "stdio":
|
||||
return &StdioDialer{}, nil
|
||||
case "websocket_rcon":
|
||||
return &WebSocketDialer{}, nil
|
||||
case "docker_exec_rcon":
|
||||
return &DockerExecDialer{}, nil
|
||||
case "be_rcon":
|
||||
return &BattlEyeDialer{}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown rcon adapter %q (known: telnet, source_rcon, docker_exec_rcon, stdio, websocket_rcon, be_rcon)", adapter)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SourceDialer speaks Valve's Source RCON protocol — the standard RCON of
|
||||
// Minecraft, CS2, Rust, Valheim-with-plugins, and most Source-engine games.
|
||||
//
|
||||
// Protocol summary (little-endian throughout):
|
||||
//
|
||||
// int32 size = bytes that follow (i.e. frame length − 4)
|
||||
// int32 id = client-chosen, server echoes
|
||||
// int32 type = 3 auth-request, 2 exec, 0 response, 2 auth-response
|
||||
// body ASCIIZ = command string or response body, null-terminated
|
||||
// byte trailing null = required empty string sentinel
|
||||
//
|
||||
// size = 4 (id) + 4 (type) + len(body) + 1 (body NUL) + 1 (sentinel NUL)
|
||||
type SourceDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (SourceDialer) Name() string { return "source_rcon" }
|
||||
|
||||
const (
|
||||
pktAuth int32 = 3
|
||||
pktExecCommand int32 = 2
|
||||
pktResponseValue int32 = 0
|
||||
pktAuthResponse int32 = 2
|
||||
|
||||
minPktSize = 10
|
||||
maxPktSize = 4110
|
||||
)
|
||||
|
||||
func (SourceDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("source rcon dial: host and port are required")
|
||||
}
|
||||
timeout := opts.ConnectTimeout
|
||||
if timeout == 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
d := &net.Dialer{Timeout: timeout}
|
||||
conn, err := d.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source rcon dial %s:%d: %w", opts.Host, opts.Port, err)
|
||||
}
|
||||
c := &sourceClient{conn: conn, reader: bufio.NewReader(conn), nextID: 1}
|
||||
if err := c.authenticate(opts.Password); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("source rcon auth: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type sourceClient struct {
|
||||
conn net.Conn
|
||||
reader *bufio.Reader
|
||||
|
||||
mu sync.Mutex
|
||||
nextID int32
|
||||
}
|
||||
|
||||
// authenticate sends SERVERDATA_AUTH and waits for SERVERDATA_AUTH_RESPONSE.
|
||||
// Valve's spec says servers send an empty RESPONSE_VALUE first (id echoed),
|
||||
// then the AUTH_RESPONSE (id = original on success, −1 on failure). Some
|
||||
// implementations skip the probe — we accept either ordering.
|
||||
func (c *sourceClient) authenticate(password string) error {
|
||||
id := c.assignID()
|
||||
if err := c.writePacket(id, pktAuth, password); err != nil {
|
||||
return err
|
||||
}
|
||||
// ASA's RCON handler is slow to answer AUTH during the first few
|
||||
// minutes after "Server has completed startup" — sometimes 3-4s per
|
||||
// reply when the server is still loading tribe data / saving a
|
||||
// snapshot. The previous 2s per-read deadline + 5s overall was
|
||||
// tripping the tracker into its 32s exponential backoff on every
|
||||
// auth, stranding the panel with "RCON not yet connected" for 5-10
|
||||
// min even though the server was otherwise fine. Give AUTH a 12s
|
||||
// outer window and 8s per-read — enough slack for ASA's slowest
|
||||
// responses without letting a truly dead connection hang forever.
|
||||
deadline := time.Now().Add(12 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(8 * time.Second))
|
||||
gotID, gotType, _, err := c.readPacket()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if gotType == pktAuthResponse {
|
||||
// Spec: id == -1 means auth failed (bad password).
|
||||
// Any other id means success. Most servers echo our request id
|
||||
// but some (Conan Exiles, Palworld, a few others) return 0 or
|
||||
// an unrelated id — treat that as success, don't reject.
|
||||
if gotID == -1 {
|
||||
return errors.New("invalid password")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Probe echo (RESPONSE_VALUE with body ""), keep reading for the
|
||||
// AUTH_RESPONSE that follows.
|
||||
}
|
||||
return errors.New("auth timeout")
|
||||
}
|
||||
|
||||
// Exec sends a SERVERDATA_EXECCOMMAND and concatenates RESPONSE_VALUE bodies
|
||||
// until the server goes silent. For responses that straddle multiple packets
|
||||
// the spec recommends a "bogus packet trick" (send a second empty packet and
|
||||
// watch for its echo) but short command output fits in one packet in practice.
|
||||
func (c *sourceClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.conn.SetDeadline(dl)
|
||||
defer c.conn.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
}
|
||||
|
||||
id := c.assignID()
|
||||
if err := c.writePacket(id, pktExecCommand, cmd); err != nil {
|
||||
return "", fmt.Errorf("write: %w", err)
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
// First read waits longer — some servers (Palworld especially) take 1–3s
|
||||
// to build the response for commands like ShowPlayers. Subsequent reads
|
||||
// use a tight idle timeout so we exit promptly once the stream goes quiet.
|
||||
const (
|
||||
firstTimeout = 5 * time.Second
|
||||
idleTimeout = 800 * time.Millisecond
|
||||
)
|
||||
readDeadline := firstTimeout
|
||||
for {
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(readDeadline))
|
||||
readDeadline = idleTimeout
|
||||
gotID, gotType, body, err := c.readPacket()
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() && out.Len() > 0 {
|
||||
return out.String(), nil
|
||||
}
|
||||
if errors.Is(err, io.EOF) && out.Len() > 0 {
|
||||
return out.String(), nil
|
||||
}
|
||||
return out.String(), err
|
||||
}
|
||||
// Source RCON spec says the server MUST echo our request id on
|
||||
// RESPONSE_VALUE. Palworld always returns id=0; Conan Exiles
|
||||
// returns a different arbitrary id. We hold mu.Lock() for the
|
||||
// whole Exec so only one command is ever in flight — ID matching
|
||||
// isn't needed for correctness, so accept any id the server
|
||||
// hands us (except -1, which is reserved for auth failures).
|
||||
if gotID == -1 {
|
||||
continue
|
||||
}
|
||||
// Spec: RESPONSE_VALUE (type 0) carries command output. Conan
|
||||
// Exiles sends command output via AUTH_RESPONSE (type 2) instead —
|
||||
// non-spec but functional. We're past the auth phase by this point
|
||||
// so accepting either type is safe; anything else we skip.
|
||||
if gotType != pktResponseValue && gotType != pktAuthResponse {
|
||||
continue
|
||||
}
|
||||
out.WriteString(body)
|
||||
// Body shorter than ~4000 bytes → likely single-packet response.
|
||||
if len(body) < 3800 {
|
||||
return out.String(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close shuts down the underlying TCP connection.
|
||||
func (c *sourceClient) Close() error { return c.conn.Close() }
|
||||
|
||||
func (c *sourceClient) assignID() int32 {
|
||||
c.nextID++
|
||||
if c.nextID <= 0 {
|
||||
c.nextID = 1
|
||||
}
|
||||
return c.nextID
|
||||
}
|
||||
|
||||
// writePacket wire-encodes a Source RCON packet.
|
||||
func (c *sourceClient) writePacket(id, ptype int32, body string) error {
|
||||
bodyBytes := []byte(body)
|
||||
pktSize := int32(4 + 4 + len(bodyBytes) + 2) // id + type + body + 2×NUL
|
||||
if pktSize > maxPktSize {
|
||||
return fmt.Errorf("source rcon: body too large (%d bytes, max %d)", len(bodyBytes), maxPktSize-10)
|
||||
}
|
||||
buf := make([]byte, 4+pktSize)
|
||||
binary.LittleEndian.PutUint32(buf[0:4], uint32(pktSize))
|
||||
binary.LittleEndian.PutUint32(buf[4:8], uint32(id))
|
||||
binary.LittleEndian.PutUint32(buf[8:12], uint32(ptype))
|
||||
copy(buf[12:], bodyBytes)
|
||||
// trailing two NUL bytes already zero
|
||||
_, err := c.conn.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
// readPacket reads one framed Source RCON packet from the underlying reader.
|
||||
// Returns (id, type, body, err). Body has trailing NULs stripped.
|
||||
func (c *sourceClient) readPacket() (int32, int32, string, error) {
|
||||
var sizeBuf [4]byte
|
||||
if _, err := io.ReadFull(c.reader, sizeBuf[:]); err != nil {
|
||||
return 0, 0, "", err
|
||||
}
|
||||
size := int32(binary.LittleEndian.Uint32(sizeBuf[:]))
|
||||
if size < minPktSize || size > maxPktSize {
|
||||
return 0, 0, "", fmt.Errorf("source rcon: bogus packet size %d", size)
|
||||
}
|
||||
rest := make([]byte, size)
|
||||
if _, err := io.ReadFull(c.reader, rest); err != nil {
|
||||
return 0, 0, "", err
|
||||
}
|
||||
id := int32(binary.LittleEndian.Uint32(rest[0:4]))
|
||||
ptype := int32(binary.LittleEndian.Uint32(rest[4:8]))
|
||||
bodyEnd := size - 2 // trim both trailing NULs
|
||||
if bodyEnd < 8 {
|
||||
bodyEnd = 8
|
||||
}
|
||||
body := string(rest[8:bodyEnd])
|
||||
return id, ptype, body, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StdioDialer "RCON"s a game by attaching to the container's stdin and writing
|
||||
// the admin console command, trusting the game to execute it. It's the correct
|
||||
// choice for games whose admin surface is stdin-only — Valheim, Terraria,
|
||||
// Barotrauma, Enshrouded, Sons Of The Forest — where no TCP RCON exists.
|
||||
//
|
||||
// v1 is write-only: Exec returns empty string. Command output arrives via the
|
||||
// panel's log pipeline (docker logs → agent → bus → SSE) because the game's
|
||||
// replies appear on the same stdout the log stream is tailing. A future
|
||||
// revision could correlate a response window with each write if we ever need
|
||||
// structured replies for state polling.
|
||||
//
|
||||
// Requires the container to have been created with OpenStdin=true; the module
|
||||
// resolver auto-enables that whenever manifest.Rcon.Adapter == "stdio".
|
||||
type StdioDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (StdioDialer) Name() string { return "stdio" }
|
||||
|
||||
// Dial attaches to the container's stdio stream. A background goroutine drains
|
||||
// the multiplexed stdout/stderr so the stream never blocks from our side; the
|
||||
// docker logs pipeline is the canonical consumer of that output.
|
||||
func (StdioDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Stdio == nil {
|
||||
return nil, errors.New("stdio dial: Stdio backend required (agent must provide one)")
|
||||
}
|
||||
if opts.ContainerID == "" {
|
||||
return nil, errors.New("stdio dial: ContainerID required")
|
||||
}
|
||||
session, err := opts.Stdio.AttachStdio(ctx, opts.ContainerID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stdio attach %q: %w", opts.ContainerID, err)
|
||||
}
|
||||
c := &stdioClient{session: session}
|
||||
go func() {
|
||||
// Drain the attached stream so Docker doesn't buffer indefinitely.
|
||||
// We deliberately discard: the log pipeline already surfaces this
|
||||
// output to the panel via a separate docker logs call.
|
||||
_, _ = io.Copy(io.Discard, session)
|
||||
}()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type stdioClient struct {
|
||||
session io.ReadWriteCloser
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Exec writes cmd + "\n" to the container's stdin. Returns empty string; the
|
||||
// game's response (if any) lands in the log stream and is surfaced in the
|
||||
// Console tab.
|
||||
//
|
||||
// The write is bounded by ctx (and a hard 10s fallback). A raw Write to a
|
||||
// hijacked stdin pipe can block FOREVER if the container's stdin read side
|
||||
// stalls (game paused, buffer full, or the process not draining stdin). Because
|
||||
// the agent dispatches RCON inline on its single recv loop, an unbounded write
|
||||
// here freezes the WHOLE agent — no stop/start/anything — until restart. That's
|
||||
// the Valheim+stdio "stop never lands" hang. We run the write in a goroutine and
|
||||
// abandon it on timeout so the dispatcher stays responsive.
|
||||
func (c *stdioClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.closed {
|
||||
return "", errors.New("stdio: session closed")
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
wctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.session.Write([]byte(cmd + "\n"))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stdio write: %w", err)
|
||||
}
|
||||
return "", nil
|
||||
case <-wctx.Done():
|
||||
// The write is wedged — leave the goroutine parked (it'll unblock if the
|
||||
// pipe ever drains, or die when the session closes) but DON'T let it hold
|
||||
// up the caller / the agent's recv loop.
|
||||
return "", fmt.Errorf("stdio write timed out after 10s (container stdin not draining): %w", wctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// Close shuts down the hijacked attach connection.
|
||||
func (c *stdioClient) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.closed {
|
||||
return nil
|
||||
}
|
||||
c.closed = true
|
||||
return c.session.Close()
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TelnetDialer speaks 7DTD-style line-oriented telnet: on connect the server
|
||||
// sends a "password:" prompt; after the client writes the password and a
|
||||
// newline, commands are one-line writes and responses are everything read
|
||||
// until the connection goes quiet for a short idle period.
|
||||
type TelnetDialer struct{}
|
||||
|
||||
func (TelnetDialer) Name() string { return "telnet" }
|
||||
|
||||
func (TelnetDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("telnet dial: host and port are required")
|
||||
}
|
||||
addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port)
|
||||
|
||||
timeout := opts.ConnectTimeout
|
||||
if timeout == 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
|
||||
d := &net.Dialer{Timeout: timeout}
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("telnet dial %s: %w", addr, err)
|
||||
}
|
||||
|
||||
idle := opts.ReadIdle
|
||||
if idle == 0 {
|
||||
idle = 250 * time.Millisecond
|
||||
}
|
||||
c := &telnetClient{
|
||||
conn: conn,
|
||||
readIdle: idle,
|
||||
}
|
||||
if err := c.authenticate(opts.Password); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("telnet auth: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// telnetClient is safe for concurrent Exec via mu serialization.
|
||||
type telnetClient struct {
|
||||
conn net.Conn
|
||||
readIdle time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (c *telnetClient) authenticate(password string) error {
|
||||
// Read until the first "password:" prompt arrives, with a 5s cap.
|
||||
if _, err := c.readUntil("password:", 5*time.Second); err != nil {
|
||||
return fmt.Errorf("read password prompt: %w", err)
|
||||
}
|
||||
if _, err := c.conn.Write([]byte(password + "\n")); err != nil {
|
||||
return fmt.Errorf("write password: %w", err)
|
||||
}
|
||||
// Drain whatever welcome/banner text the server sends after auth.
|
||||
_, _ = c.readQuiet(1500*time.Millisecond, 1500*time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exec sends one line-terminated command and returns everything the server
|
||||
// writes back before going quiet for readIdle.
|
||||
func (c *telnetClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Wire ctx deadline into the conn's read/write timeouts.
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.conn.SetDeadline(dl)
|
||||
defer c.conn.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
}
|
||||
|
||||
if _, err := c.conn.Write([]byte(cmd + "\n")); err != nil {
|
||||
return "", fmt.Errorf("write cmd: %w", err)
|
||||
}
|
||||
return c.readQuiet(c.firstByteTimeout(), c.readIdle)
|
||||
}
|
||||
|
||||
func (c *telnetClient) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// firstByteTimeout is how long we wait for the server to START replying to a
|
||||
// command. A busy 7DTD server (heavy load, blood moon, GC pause) can take well
|
||||
// over readIdle to send its first byte, so this must be generous — otherwise
|
||||
// readQuiet times out with zero bytes and returns an empty response even though
|
||||
// the command executed fine on the server.
|
||||
func (c *telnetClient) firstByteTimeout() time.Duration {
|
||||
return 4 * time.Second
|
||||
}
|
||||
|
||||
// readQuiet reads a command response. It waits up to `first` for the first byte
|
||||
// (server may be slow to start replying under load), then once bytes arrive it
|
||||
// uses the short `idle` gap to detect end-of-response. Returns what it read, or
|
||||
// an error on a real (non-timeout) failure.
|
||||
func (c *telnetClient) readQuiet(first, idle time.Duration) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
tmp := make([]byte, 4096)
|
||||
for {
|
||||
// Before any data, allow the longer first-byte window; after data
|
||||
// starts flowing, switch to the short idle gap.
|
||||
wait := idle
|
||||
if buf.Len() == 0 {
|
||||
wait = first
|
||||
}
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(wait))
|
||||
n, err := c.conn.Read(tmp)
|
||||
if n > 0 {
|
||||
buf.Write(tmp[:n])
|
||||
}
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
// Timeout with data → normal end of response.
|
||||
// Timeout with NO data even after the first-byte window → the
|
||||
// server genuinely sent nothing (or is unreachable); return
|
||||
// empty and let the caller decide.
|
||||
return buf.String(), nil
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
return buf.String(), err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readUntil reads up to `overall` time waiting for `substr` (case-insensitive)
|
||||
// to appear in the accumulated buffer. Returns what it saw either way.
|
||||
func (c *telnetClient) readUntil(substr string, overall time.Duration) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
tmp := make([]byte, 4096)
|
||||
deadline := time.Now().Add(overall)
|
||||
want := strings.ToLower(substr)
|
||||
for time.Now().Before(deadline) {
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
|
||||
n, err := c.conn.Read(tmp)
|
||||
if n > 0 {
|
||||
buf.Write(tmp[:n])
|
||||
if strings.Contains(strings.ToLower(buf.String()), want) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
continue
|
||||
}
|
||||
return buf.String(), err
|
||||
}
|
||||
}
|
||||
return buf.String(), fmt.Errorf("timed out after %s waiting for %q", overall, substr)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeTelnetServer speaks enough of the 7DTD protocol to exercise dial+auth+exec:
|
||||
// - sends "Please enter password:" on accept
|
||||
// - waits for client password (any line), then sends "Logon successful.\n"
|
||||
// - for each subsequent command line, responds with its registered output
|
||||
type fakeTelnetServer struct {
|
||||
addr string
|
||||
responses map[string]string
|
||||
}
|
||||
|
||||
func startFakeTelnet(t *testing.T, responses map[string]string) *fakeTelnetServer {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
|
||||
fs := &fakeTelnetServer{addr: ln.Addr().String(), responses: responses}
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go fs.handle(conn)
|
||||
}
|
||||
}()
|
||||
return fs
|
||||
}
|
||||
|
||||
func (fs *fakeTelnetServer) handle(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
_, _ = conn.Write([]byte("*** Connected with 7DTD server.\r\nPlease enter password:"))
|
||||
|
||||
br := bufio.NewReader(conn)
|
||||
if _, err := br.ReadString('\n'); err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = conn.Write([]byte("\nLogon successful.\n"))
|
||||
|
||||
for {
|
||||
_ = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
cmd := strings.TrimSpace(line)
|
||||
resp, ok := fs.responses[cmd]
|
||||
if !ok {
|
||||
resp = fmt.Sprintf("(unknown command: %s)\n", cmd)
|
||||
}
|
||||
_, _ = conn.Write([]byte(resp))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelnetDialAndExec(t *testing.T) {
|
||||
srv := startFakeTelnet(t, map[string]string{
|
||||
"lp": "Total of 3 in the game\n\n1. id=1, Alice, pos=(0,0,0)\n2. id=2, Bob, pos=(1,0,0)\n3. id=3, Carol, pos=(2,0,0)\n\n",
|
||||
"say \"hello\"": "*Server: hello\n",
|
||||
})
|
||||
|
||||
host, port := splitAddr(t, srv.addr)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
client, err := TelnetDialer{}.Dial(ctx, DialOptions{
|
||||
Host: host,
|
||||
Port: port,
|
||||
Password: "test",
|
||||
ConnectTimeout: 2 * time.Second,
|
||||
ReadIdle: 150 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Dial: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
out, err := client.Exec(ctx, "lp")
|
||||
if err != nil {
|
||||
t.Fatalf("Exec lp: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "Total of 3 in the game") {
|
||||
t.Errorf("lp output missing expected substring, got: %q", out)
|
||||
}
|
||||
|
||||
out, err = client.Exec(ctx, `say "hello"`)
|
||||
if err != nil {
|
||||
t.Fatalf(`Exec say: %v`, err)
|
||||
}
|
||||
if !strings.Contains(out, "*Server: hello") {
|
||||
t.Errorf("say output missing, got: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialerForUnknown(t *testing.T) {
|
||||
if _, err := DialerFor("source_rcon"); err == nil {
|
||||
t.Error("expected error for unknown adapter")
|
||||
}
|
||||
if d, err := DialerFor("telnet"); err != nil || d.Name() != "telnet" {
|
||||
t.Errorf("DialerFor(telnet): d=%v err=%v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func splitAddr(t *testing.T, addr string) (string, int) {
|
||||
t.Helper()
|
||||
host, portStr, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("split %q: %v", addr, err)
|
||||
}
|
||||
var port int
|
||||
if _, err := fmt.Sscanf(portStr, "%d", &port); err != nil {
|
||||
t.Fatalf("port parse: %v", err)
|
||||
}
|
||||
return host, port
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/websocket"
|
||||
)
|
||||
|
||||
// WebSocketDialer speaks Facepunch's WebSocket RCON — the protocol used
|
||||
// by Rust (app 258550) since they removed Source RCON years ago. A small
|
||||
// subset of games have adopted the same pattern (a few Oxide-compatible
|
||||
// community servers, some Unity-based survival games).
|
||||
//
|
||||
// Wire protocol (JSON text frames over a plain websocket):
|
||||
//
|
||||
// client → server: {"Identifier": N, "Message": "command", "Name": "panel"}
|
||||
// server → client: {"Identifier": N, "Message": "result text",
|
||||
// "Type": "Generic", "Stacktrace": ""}
|
||||
//
|
||||
// The password is passed as the URL path, not a header:
|
||||
//
|
||||
// ws://<host>:<port>/<password>
|
||||
//
|
||||
// Connections are long-lived — Rust's RCON doesn't close after each command
|
||||
// the way Conan's does — but we still redial transparently on EOF/reset
|
||||
// via the tracker's existing logic.
|
||||
type WebSocketDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (WebSocketDialer) Name() string { return "websocket_rcon" }
|
||||
|
||||
// Dial opens a websocket to ws://host:port/password. Returns a client that
|
||||
// serializes Exec calls; Rust sometimes interleaves async log spam into the
|
||||
// same connection as command replies, so we match responses back by id.
|
||||
func (WebSocketDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("websocket rcon dial: host and port are required")
|
||||
}
|
||||
connectTimeout := opts.ConnectTimeout
|
||||
if connectTimeout == 0 {
|
||||
connectTimeout = 5 * time.Second
|
||||
}
|
||||
|
||||
wsURL := fmt.Sprintf("ws://%s:%d/%s", opts.Host, opts.Port, url.PathEscape(opts.Password))
|
||||
origin := fmt.Sprintf("http://%s:%d/", opts.Host, opts.Port)
|
||||
cfg, err := websocket.NewConfig(wsURL, origin)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("websocket rcon: build config: %w", err)
|
||||
}
|
||||
|
||||
// x/net/websocket.Dial doesn't honor a context; run it in a goroutine
|
||||
// so we can bail out when ctx.Done fires. Rust's authentication is
|
||||
// implicit — bad password = server closes the handshake with a 401.
|
||||
type dialResult struct {
|
||||
ws *websocket.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan dialResult, 1)
|
||||
go func() {
|
||||
ws, err := websocket.DialConfig(cfg)
|
||||
ch <- dialResult{ws, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-ch:
|
||||
if r.err != nil {
|
||||
return nil, fmt.Errorf("websocket rcon dial %s:%d: %w", opts.Host, opts.Port, r.err)
|
||||
}
|
||||
return newWebSocketClient(r.ws), nil
|
||||
case <-time.After(connectTimeout):
|
||||
return nil, fmt.Errorf("websocket rcon dial %s:%d: connect timeout", opts.Host, opts.Port)
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
type wsRequest struct {
|
||||
Identifier int32 `json:"Identifier"`
|
||||
Message string `json:"Message"`
|
||||
Name string `json:"Name"`
|
||||
}
|
||||
|
||||
type wsResponse struct {
|
||||
Message string `json:"Message"`
|
||||
Identifier int32 `json:"Identifier"`
|
||||
Type string `json:"Type"`
|
||||
Stacktrace string `json:"Stacktrace"`
|
||||
}
|
||||
|
||||
type wsClient struct {
|
||||
ws *websocket.Conn
|
||||
nextID atomic.Int32
|
||||
execMu sync.Mutex // serializes Exec calls
|
||||
readMu sync.Mutex // guards ws reads; Close may race
|
||||
closed atomic.Bool
|
||||
unsolic chan wsResponse // buffers server-pushed log events we don't consume
|
||||
}
|
||||
|
||||
func newWebSocketClient(ws *websocket.Conn) *wsClient {
|
||||
c := &wsClient{ws: ws, unsolic: make(chan wsResponse, 64)}
|
||||
c.nextID.Store(1)
|
||||
return c
|
||||
}
|
||||
|
||||
// Exec runs one command and returns the matching response body. Rust
|
||||
// echoes our request's Identifier back, so we can reject unrelated log
|
||||
// messages the server pushes on the same connection.
|
||||
func (c *wsClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
if c.closed.Load() {
|
||||
return "", errors.New("websocket rcon: client closed")
|
||||
}
|
||||
c.execMu.Lock()
|
||||
defer c.execMu.Unlock()
|
||||
|
||||
id := c.nextID.Add(1)
|
||||
req := wsRequest{Identifier: id, Message: cmd, Name: "panel"}
|
||||
payload, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal: %w", err)
|
||||
}
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
deadline = time.Now().Add(10 * time.Second)
|
||||
}
|
||||
if err := c.ws.SetDeadline(deadline); err != nil {
|
||||
return "", fmt.Errorf("set deadline: %w", err)
|
||||
}
|
||||
defer c.ws.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
|
||||
if err := websocket.Message.Send(c.ws, string(payload)); err != nil {
|
||||
return "", fmt.Errorf("send: %w", err)
|
||||
}
|
||||
|
||||
// Read frames until one matches our Identifier. Rust pipes chat, log,
|
||||
// and player-event messages through the same socket; if their id is
|
||||
// 0 (or doesn't match ours) we drop them into unsolic for someone
|
||||
// else to consume. 200ms allowance for interleaved frames.
|
||||
for {
|
||||
c.readMu.Lock()
|
||||
var raw string
|
||||
rerr := websocket.Message.Receive(c.ws, &raw)
|
||||
c.readMu.Unlock()
|
||||
if rerr != nil {
|
||||
return "", fmt.Errorf("recv: %w", rerr)
|
||||
}
|
||||
var resp wsResponse
|
||||
if err := json.Unmarshal([]byte(raw), &resp); err != nil {
|
||||
// Non-JSON frame — Rust shouldn't send those but be forgiving.
|
||||
continue
|
||||
}
|
||||
if resp.Identifier != id {
|
||||
// Async server push (log line, chat, etc.). Buffer non-blockingly.
|
||||
select {
|
||||
case c.unsolic <- resp:
|
||||
default:
|
||||
}
|
||||
continue
|
||||
}
|
||||
if resp.Stacktrace != "" {
|
||||
return resp.Message, fmt.Errorf("rust rcon error: %s", resp.Stacktrace)
|
||||
}
|
||||
return resp.Message, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Close tears down the underlying websocket connection.
|
||||
func (c *wsClient) Close() error {
|
||||
if !c.closed.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
}
|
||||
return c.ws.Close()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,67 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDetectHostNetUnsupported_EnvOverrides verifies the explicit env
|
||||
// overrides win over autodetection (the daemon-info path needs a live
|
||||
// daemon and is covered by the integration smoke test, not here).
|
||||
func TestDetectHostNetUnsupported_EnvOverrides(t *testing.T) {
|
||||
t.Setenv("PANEL_FORCE_BRIDGE", "1")
|
||||
if !detectHostNetUnsupported(nil) {
|
||||
t.Fatal("PANEL_FORCE_BRIDGE=1 must force bridge fallback (true)")
|
||||
}
|
||||
os.Unsetenv("PANEL_FORCE_BRIDGE")
|
||||
|
||||
t.Setenv("PANEL_FORCE_HOST_NET", "1")
|
||||
if detectHostNetUnsupported(nil) {
|
||||
t.Fatal("PANEL_FORCE_HOST_NET=1 must force host networking (false)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBridgeFallbackPreservesBindings is the load-bearing check: on a
|
||||
// Docker-Desktop-style daemon, a host-mode module must be converted to
|
||||
// bridge networking WITH its published port bindings intact (so the
|
||||
// panel-allocated ports actually reach players), whereas on native Linux
|
||||
// the same module keeps host mode and drops the meaningless bindings.
|
||||
//
|
||||
// It exercises the exact decision block in Create() without a live daemon
|
||||
// by replicating its inputs.
|
||||
func TestBridgeFallbackPreservesBindings(t *testing.T) {
|
||||
ports := []PortSpec{
|
||||
{Proto: "udp", HostPort: 14507, ContainerPort: 14507},
|
||||
{Proto: "udp", HostPort: 14508, ContainerPort: 14508},
|
||||
}
|
||||
exposed, bindings, err := buildPortBindings(ports)
|
||||
if err != nil {
|
||||
t.Fatalf("buildPortBindings: %v", err)
|
||||
}
|
||||
if len(bindings) != 2 {
|
||||
t.Fatalf("expected 2 published bindings, got %d", len(bindings))
|
||||
}
|
||||
|
||||
// Native Linux (hostNetUnsupported=false): host mode kept, bindings dropped.
|
||||
r := &DockerRuntime{hostNetUnsupported: false}
|
||||
netMode, gotExposed, gotBindings := applyHostNetPolicy(r, "host", exposed, bindings)
|
||||
if netMode != "host" {
|
||||
t.Errorf("linux: netMode = %q, want host", netMode)
|
||||
}
|
||||
if gotBindings != nil || gotExposed != nil {
|
||||
t.Errorf("linux: expected bindings/exposed dropped for host mode")
|
||||
}
|
||||
|
||||
// Docker Desktop (hostNetUnsupported=true): host mode → bridge, bindings kept.
|
||||
r = &DockerRuntime{hostNetUnsupported: true}
|
||||
netMode, gotExposed, gotBindings = applyHostNetPolicy(r, "host", exposed, bindings)
|
||||
if netMode != "" {
|
||||
t.Errorf("docker desktop: netMode = %q, want bridge (empty)", netMode)
|
||||
}
|
||||
if len(gotBindings) != 2 {
|
||||
t.Errorf("docker desktop: expected 2 bindings preserved, got %d", len(gotBindings))
|
||||
}
|
||||
if len(gotExposed) != 2 {
|
||||
t.Errorf("docker desktop: expected 2 exposed ports preserved, got %d", len(gotExposed))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
// Package runtime defines the Target-side instance execution abstraction.
|
||||
// A Runtime is the thing that actually launches and tears down game server
|
||||
// processes — either inside a container (DockerRuntime) or directly on the
|
||||
// host (HostRuntime — future).
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// InstanceSpec is the resolved, runtime-ready launch recipe for one instance.
|
||||
// It is independent of the manifest format: a resolver in agent/internal/module
|
||||
// flattens (Manifest, InstanceCreate RPC) into this struct.
|
||||
type InstanceSpec struct {
|
||||
InstanceID string // panel-side identifier, used as container name
|
||||
Image string // e.g. vinanrra/7dtd-server:latest
|
||||
Entrypoint []string // override, empty = image default
|
||||
Command []string // override, empty = image default
|
||||
Env map[string]string // environment variables
|
||||
User string // uid:gid or name, empty = image default
|
||||
WorkingDir string
|
||||
Volumes []VolumeSpec
|
||||
Ports []PortSpec
|
||||
Resources ResourceLimits
|
||||
// RestartPolicy: "no", "on-failure", "unless-stopped", "always"
|
||||
RestartPolicy string
|
||||
// OpenStdin keeps the container's stdin open so we can later attach and
|
||||
// write admin-console commands (stdio RCON adapter). Auto-enabled by
|
||||
// the module resolver when rcon.adapter == "stdio".
|
||||
OpenStdin bool
|
||||
// NetworkMode is the Docker network mode. Empty → bridge (default).
|
||||
// "host" shares the host's network namespace; required for modules
|
||||
// whose TURN/STUN clients can't survive Docker's UDP NAT rebinds
|
||||
// (e.g. Windrose). In host mode, Ports become informational only.
|
||||
NetworkMode string
|
||||
// BuildContext is an absolute filesystem path containing a Dockerfile
|
||||
// that produces this Image. When set, the runtime auto-builds the
|
||||
// image on first use if it's missing locally — the AMP-class "drop in
|
||||
// an agent and any module just works" behavior. Required for `panel-*`
|
||||
// images, since they are never pushed to a registry. Empty for stock
|
||||
// public images (alpine, the acekorneya ASA image, etc.) which the
|
||||
// runtime pulls normally.
|
||||
BuildContext string
|
||||
// LogSink optionally receives runtime-side progress lines (image
|
||||
// pull / image build / etc.) so the agent can forward them to the
|
||||
// controller's log stream. Without this, first-time builds happen
|
||||
// silently and the operator sees "installing" with no visible
|
||||
// progress for ~30s–3min while the Dockerfile runs.
|
||||
LogSink func(line string)
|
||||
// SecurityOpts is passed through to Docker's HostConfig.SecurityOpt
|
||||
// (one entry per --security-opt CLI flag). The SteamCMD sidecar sets
|
||||
// "seccomp=unconfined" so that the 32-bit Steam runtime's socket
|
||||
// syscalls aren't ENOSYS'd by Docker's stricter default seccomp
|
||||
// profile in newer Docker releases (observed on Docker 29.4 + Ubuntu
|
||||
// 24.04 kernel 6.8: "CreateBoundSocket: failed to create socket,
|
||||
// error [no name available] (38)"). Same image works fine on
|
||||
// Docker 29.1 with the default profile.
|
||||
SecurityOpts []string
|
||||
// IsSidecar marks this as a transient helper container (fs browse,
|
||||
// steamcmd, backup, restore, wipe, warmseed, basemods) rather than a
|
||||
// main game-server instance. The runtime stamps a "panel.role" label
|
||||
// accordingly so orphan-detection (Rehydrate) can tell main instances
|
||||
// apart from helpers without brittle name-suffix matching.
|
||||
IsSidecar bool
|
||||
}
|
||||
|
||||
// VolumeSpec is a mount declaration, either a bind mount (host path →
|
||||
// container path) or a Docker named volume. The resolver is responsible
|
||||
// for substituting $DATA_PATH / $INSTANCE_ID before we see it here.
|
||||
type VolumeSpec struct {
|
||||
Type string // "bind" (default) or "volume"
|
||||
HostPath string // populated when Type == "bind"
|
||||
VolumeName string // populated when Type == "volume"
|
||||
ContainerPath string
|
||||
ReadOnly bool
|
||||
}
|
||||
|
||||
// PortSpec is a port mapping.
|
||||
type PortSpec struct {
|
||||
HostPort uint16
|
||||
ContainerPort uint16
|
||||
Proto string // "tcp" or "udp"
|
||||
// HostIP — bind address on the Target. Empty = 0.0.0.0.
|
||||
HostIP string
|
||||
}
|
||||
|
||||
// ResourceLimits caps CPU / memory for the instance.
|
||||
type ResourceLimits struct {
|
||||
CPUShares uint32
|
||||
MemoryBytes uint64
|
||||
PidsLimit int64 // 0 = unlimited
|
||||
NanoCPUs int64 // Docker "--cpus" as nano-CPUs (1.5 cpus = 1.5e9)
|
||||
StopGrace time.Duration
|
||||
}
|
||||
|
||||
// RuntimeInstanceState is the runtime's view of an instance at a point in
|
||||
// time, used by the Agent to rehydrate after restart or detect drift.
|
||||
type RuntimeInstanceState struct {
|
||||
ContainerID string
|
||||
Status string // "running" | "exited" | "created" | "paused" | "restarting" | "removing" | "dead"
|
||||
ExitCode int32
|
||||
}
|
||||
|
||||
// LogStream identifies which stream a log line came from.
|
||||
type LogStream string
|
||||
|
||||
const (
|
||||
LogStreamStdout LogStream = "stdout"
|
||||
LogStreamStderr LogStream = "stderr"
|
||||
)
|
||||
|
||||
// LogHandler receives one log line at a time. Lines are already stripped of
|
||||
// trailing newline. Handler must not block; if it does, log pressure will
|
||||
// build up in the Runtime.
|
||||
type LogHandler func(stream LogStream, line string, at time.Time)
|
||||
|
||||
// Runtime is the interface that both Docker and Host runtimes implement.
|
||||
type Runtime interface {
|
||||
// Create pulls the image (if needed) and creates the container.
|
||||
// Returns a runtime-specific identifier used by Start/Stop/Remove.
|
||||
Create(ctx context.Context, spec InstanceSpec) (id string, err error)
|
||||
|
||||
// Start starts a previously-created instance.
|
||||
Start(ctx context.Context, id string) error
|
||||
|
||||
// Stop stops a running instance, waiting up to grace before SIGKILL.
|
||||
Stop(ctx context.Context, id string, grace time.Duration) error
|
||||
|
||||
// Restart restarts a running instance, waiting up to grace before
|
||||
// SIGKILL. Container ends up running again — equivalent to
|
||||
// `docker container restart`. Used by guardrails (e.g. arkHangGuard)
|
||||
// that want a bounce without surfacing as an operator stop.
|
||||
Restart(ctx context.Context, id string, grace time.Duration) error
|
||||
|
||||
// Remove deletes the instance (after Stop). Safe to call on stopped.
|
||||
Remove(ctx context.Context, id string) error
|
||||
|
||||
// StreamLogs follows stdout/stderr and calls handler for each line.
|
||||
// Returns when ctx is cancelled or the instance exits.
|
||||
StreamLogs(ctx context.Context, id string, handler LogHandler) error
|
||||
|
||||
// Wait blocks until the instance exits and returns its exit code.
|
||||
Wait(ctx context.Context, id string) (exitCode int, err error)
|
||||
|
||||
// StatsStream returns a reader of Docker's stats stream (JSON frames,
|
||||
// one per second). Closer cancels the stream. Used by the per-instance
|
||||
// stats poller to compute CPU%/mem/net for UI + alerting.
|
||||
StatsStream(ctx context.Context, id string) (io.ReadCloser, error)
|
||||
|
||||
// InspectByName looks up an instance by its runtime-level name
|
||||
// (for Docker: "panel-<instance_id>") and returns its current state.
|
||||
// Returns an error whose predicate is not-found-able if the instance
|
||||
// has been removed from the runtime — used during agent rehydrate
|
||||
// to detect stale sidecar metadata.
|
||||
InspectByName(ctx context.Context, name string) (RuntimeInstanceState, error)
|
||||
|
||||
// ContainerExists reports whether a container with the given
|
||||
// runtime-level name currently exists. Unlike InspectByName, it
|
||||
// distinguishes "definitely gone" from "couldn't tell":
|
||||
// exists=false, err=nil → the runtime confirmed no such container
|
||||
// exists=false, err!=nil → the query itself failed (daemon down,
|
||||
// connection dropped, ctx cancelled) — the
|
||||
// container's fate is UNKNOWN, not gone.
|
||||
// Callers that gate destructive bookkeeping (e.g. deleting sidecar
|
||||
// metadata after a container remove) must treat a non-nil err as
|
||||
// "do not assume removal succeeded."
|
||||
ContainerExists(ctx context.Context, name string) (exists bool, err error)
|
||||
|
||||
// ---- Container-path file operations ----
|
||||
// These operate inside the running container (or container filesystem
|
||||
// for stopped containers where the runtime allows it). Used by the
|
||||
// file manager for volume-backed instances where the host-side bind
|
||||
// mount is absent or empty.
|
||||
|
||||
// ExecCapture runs a command inside the container and returns its
|
||||
// captured stdout + stderr + exit code. Requires container to be running.
|
||||
ExecCapture(ctx context.Context, id string, cmd []string) (stdout, stderr []byte, exitCode int, err error)
|
||||
|
||||
// CopyFileFromContainer returns the raw contents of a single file at
|
||||
// containerPath. Works even on stopped containers.
|
||||
CopyFileFromContainer(ctx context.Context, id, containerPath string) ([]byte, error)
|
||||
|
||||
// CopyFileToContainer writes content to containerPath, creating
|
||||
// parent dirs inside the container if the container is running.
|
||||
CopyFileToContainer(ctx context.Context, id, containerPath string, content []byte) error
|
||||
|
||||
// CopyTarToContainer writes a raw tar stream onto the container at
|
||||
// dstDir. The tar entries are extracted in place by the Docker API.
|
||||
// Used by the file manager's extract path so we can build one big
|
||||
// tar from the archive's entries and ship it in a single API call.
|
||||
CopyTarToContainer(ctx context.Context, id, dstDir string, tarStream io.Reader) error
|
||||
|
||||
// CopyDirFromContainer returns a tar stream of the contents at
|
||||
// containerPath. For files: a single-entry tar. For directories:
|
||||
// one entry per file/dir under it. Used by the compress path so the
|
||||
// agent can iterate file contents without per-file round-trips.
|
||||
CopyDirFromContainer(ctx context.Context, id, containerPath string) (io.ReadCloser, error)
|
||||
|
||||
// AttachStdio attaches to the container's stdin/stdout/stderr stream
|
||||
// and returns a read/write/close handle. Used by the stdio RCON adapter
|
||||
// for games whose admin surface is stdin-only. Requires the container
|
||||
// to have been created with OpenStdin=true.
|
||||
AttachStdio(ctx context.Context, id string) (io.ReadWriteCloser, error)
|
||||
|
||||
// ContainerMounts returns the mount points of a container by name.
|
||||
// Works on stopped containers (the daemon keeps the spec). Used by the
|
||||
// cluster player-save snapshotter to find the host path backing a 7DTD
|
||||
// instance's /cluster bind without depending on a path convention.
|
||||
ContainerMounts(ctx context.Context, name string) ([]MountInfo, error)
|
||||
|
||||
// Name returns the runtime kind ("docker", "host") for logging.
|
||||
Name() string
|
||||
}
|
||||
|
||||
// MountInfo is one container mount point, host-side. Type is "bind" or
|
||||
// "volume"; Source is the host path backing it (for a named volume that's
|
||||
// the docker volume's _data dir); Destination is the in-container path.
|
||||
type MountInfo struct {
|
||||
Type string
|
||||
Source string
|
||||
Destination string
|
||||
Name string
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
// Package state runs the per-instance state-tracking pipeline:
|
||||
// - RCON poll loops that execute manifest-declared commands and parse the
|
||||
// output into AppStateUpdate messages (players online, etc.)
|
||||
// - log-line event matching that runs manifest-declared regexes against
|
||||
// each log line and emits PlayerEvent messages (join/leave/chat/death)
|
||||
//
|
||||
// A Tracker is owned by the dispatcher for the lifetime of a running
|
||||
// instance. It is stopped by cancelling the context passed to Run.
|
||||
package state
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/rcon"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Emitter is the upward path for tracker output. Implementations forward
|
||||
// messages onto the gRPC stream back to the Controller.
|
||||
type Emitter interface {
|
||||
EmitAppState(*panelv1.AppStateUpdate)
|
||||
EmitPlayerEvent(*panelv1.PlayerEvent)
|
||||
}
|
||||
|
||||
// Config wires the tracker to an instance + its RCON endpoint.
|
||||
type Config struct {
|
||||
InstanceID string
|
||||
Manifest *modulepkg.Manifest
|
||||
RCONAddr string // "host:port" — required for tcp adapters (telnet, source_rcon)
|
||||
RCONPassword string
|
||||
// PasswordFunc, if set, overrides RCONPassword on each dial attempt.
|
||||
// Used when the module's entrypoint generates an RCON secret at first
|
||||
// boot (e.g. 7DTD's TelnetPassword in /game-saves/.panel-telnet-password).
|
||||
// Called with the dial context; the returned string is used as the
|
||||
// password for that single attempt.
|
||||
PasswordFunc func(ctx context.Context) (string, error)
|
||||
Emitter Emitter
|
||||
// ContainerID is the Docker container name/ID for the stdio adapter.
|
||||
// Ignored by tcp adapters. The stdio adapter calls Stdio.AttachStdio
|
||||
// with this id to get a stdin/stdout handle to the running game.
|
||||
ContainerID string
|
||||
// Stdio is the backend that knows how to attach to a container's
|
||||
// stdio. Required for stdio adapter, nil otherwise.
|
||||
Stdio rcon.StdioBackend
|
||||
// OnRconResult is an optional callback invoked after every RCON
|
||||
// exec attempt (poll loop or operator-driven). success=true means
|
||||
// the engine answered; success=false means the exec failed (dial,
|
||||
// timeout, dead conn). Used by the dispatcher's arkHangGuard to
|
||||
// count consecutive failures and trigger an auto-restart when the
|
||||
// Wine 9 listener wedges. Nil-safe: callers should check before invoke.
|
||||
OnRconResult func(success bool)
|
||||
}
|
||||
|
||||
// Tracker is created at instance start time and run in a goroutine.
|
||||
type Tracker struct {
|
||||
log *slog.Logger
|
||||
cfg Config
|
||||
events []compiledEvent
|
||||
// stateRE holds the state-source parse regexes, compiled once at New so
|
||||
// a bad pattern surfaces as a construction error instead of silently
|
||||
// recompiling (and silently failing) on every poll tick. Keyed by the
|
||||
// pattern string — pollOnce receives the StateSource by value, so the
|
||||
// pattern is the stable identity.
|
||||
stateRE map[string]*regexp.Regexp
|
||||
|
||||
mu sync.Mutex
|
||||
client rcon.Client
|
||||
}
|
||||
|
||||
type compiledEvent struct {
|
||||
name string
|
||||
re *regexp.Regexp
|
||||
kind panelv1.PlayerEvent_Kind
|
||||
}
|
||||
|
||||
// New constructs a Tracker. Event patterns are compiled now so config errors
|
||||
// surface before Run is called.
|
||||
func New(log *slog.Logger, cfg Config) (*Tracker, error) {
|
||||
if cfg.Manifest == nil {
|
||||
return nil, errors.New("manifest is required")
|
||||
}
|
||||
if cfg.Emitter == nil {
|
||||
return nil, errors.New("emitter is required")
|
||||
}
|
||||
events, err := compileEvents(cfg.Manifest.Events)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Compile state-source parse patterns up front — a bad pattern is a
|
||||
// manifest bug and should fail loudly at init, not degrade into a
|
||||
// per-poll recompile that silently yields no state updates.
|
||||
stateRE := map[string]*regexp.Regexp{}
|
||||
for i := range cfg.Manifest.StateSources {
|
||||
p := cfg.Manifest.StateSources[i].Parse
|
||||
if p == nil || p.Kind != "regex" || p.Pattern == "" {
|
||||
continue
|
||||
}
|
||||
re, err := regexp.Compile(p.Pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("state source %d: compile pattern: %w", i, err)
|
||||
}
|
||||
stateRE[p.Pattern] = re
|
||||
}
|
||||
return &Tracker{
|
||||
log: log.With("instance_id", cfg.InstanceID, "component", "state"),
|
||||
cfg: cfg,
|
||||
events: events,
|
||||
stateRE: stateRE,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Client returns the currently-connected RCON client, or nil if the tracker
|
||||
// hasn't finished dialing (or the manifest has no RCON at all). Used by the
|
||||
// dispatcher to handle ad-hoc RCON commands from operators.
|
||||
func (t *Tracker) Client() rcon.Client {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.client
|
||||
}
|
||||
|
||||
// Exec runs one operator command via the tracker's cached client. On a dead
|
||||
// connection (EOF, broken pipe — Palworld closes RCON after 60s idle) we
|
||||
// redial once and retry, so operators don't see the 60s-idle EOF in the UI.
|
||||
// Returns ("", "not_connected", ...) if the tracker hasn't finished dialing.
|
||||
func (t *Tracker) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
t.mu.Lock()
|
||||
client := t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return "", errors.New("RCON not yet connected; try again in a moment")
|
||||
}
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := client.Exec(execCtx, cmd)
|
||||
if err == nil || !isDeadConnErr(err) {
|
||||
return out, err
|
||||
}
|
||||
t.redial(ctx)
|
||||
t.mu.Lock()
|
||||
client = t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return out, err
|
||||
}
|
||||
retryCtx, retryCancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer retryCancel()
|
||||
return client.Exec(retryCtx, cmd)
|
||||
}
|
||||
|
||||
// OnLogLine runs each log line through compiled event patterns and emits a
|
||||
// PlayerEvent on the first match. Called from the dispatcher's log pump; must
|
||||
// be fast and non-blocking.
|
||||
func (t *Tracker) OnLogLine(line string) {
|
||||
for _, ev := range t.events {
|
||||
m := ev.re.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
name := firstGroup(ev.re, m, "name", "player_name")
|
||||
id := firstGroup(ev.re, m, "platform_id", "owner", "cross_id", "id", "player_id")
|
||||
detail := firstGroup(ev.re, m, "msg", "detail", "reason")
|
||||
t.cfg.Emitter.EmitPlayerEvent(&panelv1.PlayerEvent{
|
||||
InstanceId: t.cfg.InstanceID,
|
||||
Kind: ev.kind,
|
||||
PlayerName: name,
|
||||
PlayerId: id,
|
||||
Detail: detail,
|
||||
At: timestamppb.Now(),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Run connects to RCON (with retries) and starts one goroutine per declared
|
||||
// RCON state source. Returns when ctx is cancelled.
|
||||
//
|
||||
// If the manifest has no RCON config at all, Run is a no-op that blocks on
|
||||
// ctx.Done. If it has RCON but no state sources, we still dial once so
|
||||
// operator ad-hoc commands from the UI work — state-source poll loops just
|
||||
// don't spawn.
|
||||
func (t *Tracker) Run(ctx context.Context) {
|
||||
if t.cfg.Manifest.RCON == nil {
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
|
||||
client, err := t.dialWithRetries(ctx)
|
||||
if err != nil {
|
||||
// Context cancelled before we could connect; nothing to do.
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.client = client
|
||||
t.mu.Unlock()
|
||||
defer func() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.client != nil {
|
||||
_ = t.client.Close()
|
||||
t.client = nil
|
||||
}
|
||||
}()
|
||||
|
||||
t.log.Info("rcon connected", "addr", t.cfg.RCONAddr, "adapter", t.cfg.Manifest.RCON.Adapter)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range t.cfg.Manifest.StateSources {
|
||||
ss := t.cfg.Manifest.StateSources[i]
|
||||
if ss.Type != "rcon" || ss.Every.Std() <= 0 {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
t.runRCONPoller(ctx, ss)
|
||||
}()
|
||||
}
|
||||
// Block until the tracker is cancelled so the defer (which closes the
|
||||
// rcon client) fires on teardown, not immediately. With no rcon state
|
||||
// sources wg.Wait() returns instantly — without this extra wait the
|
||||
// client would vanish right after Run dialed, making ad-hoc commands
|
||||
// from the UI fail with "RCON not yet connected" forever.
|
||||
<-ctx.Done()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (t *Tracker) hasRCONStateSource() bool {
|
||||
for _, ss := range t.cfg.Manifest.StateSources {
|
||||
if ss.Type == "rcon" && ss.Every.Std() > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *Tracker) dialWithRetries(ctx context.Context) (rcon.Client, error) {
|
||||
rc := t.cfg.Manifest.RCON
|
||||
dialer, err := rcon.DialerFor(rc.Adapter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// stdio has no host/port — it attaches to the running container's stdin.
|
||||
// We still retry-loop because the container may not be up yet when the
|
||||
// tracker starts (race between Start + activate).
|
||||
if rc.Adapter == "stdio" {
|
||||
backoff := 2 * time.Second
|
||||
for {
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
client, err := dialer.Dial(dialCtx, rcon.DialOptions{
|
||||
ContainerID: t.cfg.ContainerID,
|
||||
Stdio: t.cfg.Stdio,
|
||||
})
|
||||
cancel()
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
t.log.Warn("stdio attach failed, retrying", "err", err, "backoff", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 30*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
host, portStr, err := net.SplitHostPort(t.cfg.RCONAddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rcon addr %q: %w", t.cfg.RCONAddr, err)
|
||||
}
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rcon port %q: %w", portStr, err)
|
||||
}
|
||||
|
||||
// Dial-retry backoff capped at 8s (not 30s) — during the post-Start
|
||||
// warm-up window the server's RCON flips between responsive and slow
|
||||
// every few seconds, and a 32s ceiling meant one unlucky timeout
|
||||
// stranded the panel for half a minute before the next retry. Cap
|
||||
// at 8s so ragnarok/etc. get back onto RCON within a few seconds of
|
||||
// ASA actually being ready, without hammering the port on a truly
|
||||
// dead server (8s × a few retries is ~30s before the tracker gives
|
||||
// up per-cycle, which the agent's outer loop still reschedules).
|
||||
backoff := 2 * time.Second
|
||||
for {
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
// Resolve password per-attempt when PasswordFunc is set — the
|
||||
// module's entrypoint may be mid-generation on first boot, so re-
|
||||
// reading on each retry lets us pick up the final value as soon
|
||||
// as it's written.
|
||||
pw := t.cfg.RCONPassword
|
||||
if t.cfg.PasswordFunc != nil {
|
||||
if p, err := t.cfg.PasswordFunc(dialCtx); err == nil && p != "" {
|
||||
pw = p
|
||||
}
|
||||
}
|
||||
client, err := dialer.Dial(dialCtx, rcon.DialOptions{
|
||||
Host: host,
|
||||
Port: port,
|
||||
Password: pw,
|
||||
ConnectTimeout: 5 * time.Second,
|
||||
ContainerID: t.cfg.ContainerID,
|
||||
})
|
||||
cancel()
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
t.log.Warn("rcon dial failed, retrying", "err", err, "backoff", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 8*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) runRCONPoller(ctx context.Context, ss modulepkg.StateSource) {
|
||||
every := ss.Every.Std()
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
t.pollOnce(ctx, ss) // immediate first tick
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
t.pollOnce(ctx, ss)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) pollOnce(ctx context.Context, ss modulepkg.StateSource) {
|
||||
t.mu.Lock()
|
||||
client := t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return
|
||||
}
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := client.Exec(execCtx, ss.Command)
|
||||
if err != nil {
|
||||
t.log.Warn("rcon exec failed", "cmd", ss.Command, "err", err)
|
||||
if t.cfg.OnRconResult != nil {
|
||||
t.cfg.OnRconResult(false)
|
||||
}
|
||||
if isDeadConnErr(err) {
|
||||
t.redial(ctx)
|
||||
}
|
||||
return
|
||||
}
|
||||
if t.cfg.OnRconResult != nil {
|
||||
t.cfg.OnRconResult(true)
|
||||
}
|
||||
app := t.parseState(out, ss)
|
||||
if app == nil {
|
||||
// Parser yielded no structured fields (regex didn't match this game's
|
||||
// output, fields all empty, etc.) but the RCON exec succeeded — that
|
||||
// alone is proof the server is responsive. Emit a bare update keyed
|
||||
// by InstanceID so the dispatcher can use it to promote a stale
|
||||
// CRASHED status back to RUNNING.
|
||||
app = &panelv1.AppStateUpdate{InstanceId: t.cfg.InstanceID}
|
||||
}
|
||||
t.cfg.Emitter.EmitAppState(app)
|
||||
}
|
||||
|
||||
// isDeadConnErr returns true for errors that indicate the RCON socket is no
|
||||
// longer usable (EOF, connection reset, broken pipe). Palworld's RCON, for
|
||||
// instance, closes the socket after ~60s idle — any subsequent Exec fails
|
||||
// with EOF and stays broken until we redial.
|
||||
func isDeadConnErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
return strings.Contains(s, "EOF") ||
|
||||
strings.Contains(s, "broken pipe") ||
|
||||
strings.Contains(s, "connection reset") ||
|
||||
strings.Contains(s, "forcibly closed") ||
|
||||
strings.Contains(s, "use of closed network connection")
|
||||
}
|
||||
|
||||
// redial closes the current client and dials a fresh one. Runs inline with
|
||||
// the poll loop so the next tick gets a working client. On failure we leave
|
||||
// t.client = nil and the next poll will call us again.
|
||||
func (t *Tracker) redial(ctx context.Context) {
|
||||
t.mu.Lock()
|
||||
if t.client != nil {
|
||||
_ = t.client.Close()
|
||||
t.client = nil
|
||||
}
|
||||
t.mu.Unlock()
|
||||
t.log.Info("rcon redialing after dead connection")
|
||||
fresh, err := t.dialWithRetries(ctx)
|
||||
if err != nil {
|
||||
t.log.Warn("rcon redial failed", "err", err)
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.client = fresh
|
||||
t.mu.Unlock()
|
||||
t.log.Info("rcon reconnected", "addr", t.cfg.RCONAddr, "adapter", t.cfg.Manifest.RCON.Adapter)
|
||||
}
|
||||
|
||||
// parseState runs the configured parser against raw RCON output using the
|
||||
// regex precompiled at New, returning an AppStateUpdate or nil if nothing
|
||||
// matched.
|
||||
func (t *Tracker) parseState(out string, ss modulepkg.StateSource) *panelv1.AppStateUpdate {
|
||||
if ss.Parse == nil || ss.Parse.Kind != "regex" {
|
||||
return nil
|
||||
}
|
||||
re := t.stateRE[ss.Parse.Pattern]
|
||||
if re == nil {
|
||||
return nil
|
||||
}
|
||||
return parseRegexCompiled(t.cfg.InstanceID, out, re, ss.Parse)
|
||||
}
|
||||
|
||||
// parseStateOutput is the uncached variant (compiles the pattern per call).
|
||||
// Kept for tests and out-of-tracker callers; the tracker's poll loop uses
|
||||
// the precompiled parseState path.
|
||||
func parseStateOutput(instanceID, out string, ss modulepkg.StateSource) *panelv1.AppStateUpdate {
|
||||
if ss.Parse == nil {
|
||||
return nil
|
||||
}
|
||||
switch ss.Parse.Kind {
|
||||
case "regex":
|
||||
return parseRegex(instanceID, out, ss.Parse)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func parseRegex(instanceID, out string, p *modulepkg.ParseConfig) *panelv1.AppStateUpdate {
|
||||
re, err := regexp.Compile(p.Pattern)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return parseRegexCompiled(instanceID, out, re, p)
|
||||
}
|
||||
|
||||
func parseRegexCompiled(instanceID, out string, re *regexp.Regexp, p *modulepkg.ParseConfig) *panelv1.AppStateUpdate {
|
||||
m := re.FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
app := &panelv1.AppStateUpdate{
|
||||
InstanceId: instanceID,
|
||||
At: timestamppb.Now(),
|
||||
}
|
||||
for outName, groupName := range p.Fields {
|
||||
val := firstGroup(re, m, groupName)
|
||||
switch outName {
|
||||
case "players_online":
|
||||
if n, err := strconv.Atoi(val); err == nil {
|
||||
app.PlayersOnline = int32(n)
|
||||
}
|
||||
case "players_max":
|
||||
if n, err := strconv.Atoi(val); err == nil {
|
||||
app.PlayersMax = int32(n)
|
||||
}
|
||||
case "uptime_seconds":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
app.UptimeSeconds = n
|
||||
}
|
||||
default:
|
||||
if app.ModuleFields == nil {
|
||||
app.ModuleFields = map[string]string{}
|
||||
}
|
||||
app.ModuleFields[outName] = val
|
||||
}
|
||||
}
|
||||
return app
|
||||
}
|
||||
|
||||
func compileEvents(events map[string]modulepkg.Event) ([]compiledEvent, error) {
|
||||
out := make([]compiledEvent, 0, len(events))
|
||||
for name, ev := range events {
|
||||
re, err := regexp.Compile(ev.Pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("event %q: compile pattern: %w", name, err)
|
||||
}
|
||||
out = append(out, compiledEvent{
|
||||
name: name,
|
||||
re: re,
|
||||
kind: parseKind(ev.Kind),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseKind(s string) panelv1.PlayerEvent_Kind {
|
||||
switch s {
|
||||
case "join":
|
||||
return panelv1.PlayerEvent_KIND_JOIN
|
||||
case "leave":
|
||||
return panelv1.PlayerEvent_KIND_LEAVE
|
||||
case "chat":
|
||||
return panelv1.PlayerEvent_KIND_CHAT
|
||||
case "death":
|
||||
return panelv1.PlayerEvent_KIND_DEATH
|
||||
default:
|
||||
return panelv1.PlayerEvent_KIND_CUSTOM
|
||||
}
|
||||
}
|
||||
|
||||
// firstGroup returns the first non-empty named submatch from names. Missing
|
||||
// or empty groups are treated as absent so we try the next name.
|
||||
func firstGroup(re *regexp.Regexp, m []string, names ...string) string {
|
||||
for _, name := range names {
|
||||
i := re.SubexpIndex(name)
|
||||
if i >= 0 && i < len(m) && m[i] != "" {
|
||||
return m[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
type captureEmitter struct {
|
||||
appStates []*panelv1.AppStateUpdate
|
||||
playerEvents []*panelv1.PlayerEvent
|
||||
}
|
||||
|
||||
func (c *captureEmitter) EmitAppState(a *panelv1.AppStateUpdate) { c.appStates = append(c.appStates, a) }
|
||||
func (c *captureEmitter) EmitPlayerEvent(p *panelv1.PlayerEvent) { c.playerEvents = append(c.playerEvents, p) }
|
||||
|
||||
func newTestTracker(t *testing.T, m *modulepkg.Manifest) (*Tracker, *captureEmitter) {
|
||||
t.Helper()
|
||||
cap := &captureEmitter{}
|
||||
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
tr, err := New(log, Config{
|
||||
InstanceID: "inst-1",
|
||||
Manifest: m,
|
||||
Emitter: cap,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
return tr, cap
|
||||
}
|
||||
|
||||
func Test7DTDJoinPattern(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, cap := newTestTracker(t, manifest)
|
||||
|
||||
// Real-world 7DTD log line shape
|
||||
line := `2026-04-19T10:00:00 42.123 INF PlayerSpawnedInWorld (by system): PltfmId='Steam_76561198000000001', CrossId='EOS_00000000', OwnerID='Steam_76561198000000001', PlayerName='Alice'`
|
||||
tr.OnLogLine(line)
|
||||
|
||||
if len(cap.playerEvents) != 1 {
|
||||
t.Fatalf("expected 1 player event, got %d", len(cap.playerEvents))
|
||||
}
|
||||
ev := cap.playerEvents[0]
|
||||
if ev.Kind != panelv1.PlayerEvent_KIND_JOIN {
|
||||
t.Errorf("kind = %v, want JOIN", ev.Kind)
|
||||
}
|
||||
if ev.PlayerName != "Alice" {
|
||||
t.Errorf("name = %q, want Alice", ev.PlayerName)
|
||||
}
|
||||
if ev.PlayerId == "" {
|
||||
t.Errorf("player_id empty; expected Steam_... from named groups")
|
||||
}
|
||||
}
|
||||
|
||||
func Test7DTDChatPattern(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, cap := newTestTracker(t, manifest)
|
||||
|
||||
line := `2026-04-19T10:00:00 42.123 INF Chat (from 'Steam_76561198000000001', entity id '123', to 'Global'): 'Alice': hello world`
|
||||
tr.OnLogLine(line)
|
||||
|
||||
if len(cap.playerEvents) != 1 {
|
||||
t.Fatalf("expected 1 player event, got %d", len(cap.playerEvents))
|
||||
}
|
||||
ev := cap.playerEvents[0]
|
||||
if ev.Kind != panelv1.PlayerEvent_KIND_CHAT {
|
||||
t.Errorf("kind = %v, want CHAT", ev.Kind)
|
||||
}
|
||||
if ev.PlayerName != "Alice" {
|
||||
t.Errorf("name = %q, want Alice", ev.PlayerName)
|
||||
}
|
||||
if ev.Detail != "hello world" {
|
||||
t.Errorf("detail = %q, want 'hello world'", ev.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func Test7DTDLPParse(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
var lp *modulepkg.StateSource
|
||||
for i := range manifest.StateSources {
|
||||
if manifest.StateSources[i].Type == "rcon" {
|
||||
lp = &manifest.StateSources[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if lp == nil {
|
||||
t.Fatal("no rcon state source in 7dtd manifest")
|
||||
}
|
||||
|
||||
sample := "Total of 3 in the game\n\n1. id=1, Alice, pos=(0,0,0)\n2. id=2, Bob, pos=(1,0,0)\n3. id=3, Carol, pos=(2,0,0)"
|
||||
app := parseStateOutput("inst-1", sample, *lp)
|
||||
if app == nil {
|
||||
t.Fatal("parse returned nil")
|
||||
}
|
||||
if app.PlayersOnline != 3 {
|
||||
t.Errorf("players_online = %d, want 3", app.PlayersOnline)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineWithNoMatchYieldsNoEvent(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, cap := newTestTracker(t, manifest)
|
||||
tr.OnLogLine("this is some random log line that matches nothing")
|
||||
if len(cap.playerEvents) != 0 {
|
||||
t.Errorf("expected 0 events, got %d", len(cap.playerEvents))
|
||||
}
|
||||
}
|
||||
|
||||
// ---- WI-05: hoisted state-source regex compilation ----
|
||||
|
||||
// TestNewRejectsBadStateSourcePattern: a malformed parse regex must fail at
|
||||
// construction (surfacing the manifest bug at init) instead of silently
|
||||
// recompiling-and-failing on every poll tick.
|
||||
func TestNewRejectsBadStateSourcePattern(t *testing.T) {
|
||||
m := &modulepkg.Manifest{
|
||||
StateSources: []modulepkg.StateSource{{
|
||||
Type: "rcon",
|
||||
Command: "lp",
|
||||
Parse: &modulepkg.ParseConfig{Kind: "regex", Pattern: "("},
|
||||
}},
|
||||
}
|
||||
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
if _, err := New(log, Config{InstanceID: "i", Manifest: m, Emitter: &captureEmitter{}}); err == nil {
|
||||
t.Fatal("New accepted an invalid state-source regex; want construction error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseStateUsesPrecompiledRegex drives the tracker's precompiled parse
|
||||
// path with the real 7DTD manifest and confirms it matches the uncached
|
||||
// package-level parser.
|
||||
func TestParseStateUsesPrecompiledRegex(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, _ := newTestTracker(t, manifest)
|
||||
var lp *modulepkg.StateSource
|
||||
for i := range manifest.StateSources {
|
||||
if manifest.StateSources[i].Type == "rcon" {
|
||||
lp = &manifest.StateSources[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if lp == nil {
|
||||
t.Fatal("no rcon state source in 7dtd manifest")
|
||||
}
|
||||
sample := "Total of 3 in the game\n\n1. id=1, Alice, pos=(0,0,0)\n2. id=2, Bob, pos=(1,0,0)\n3. id=3, Carol, pos=(2,0,0)"
|
||||
app := tr.parseState(sample, *lp)
|
||||
if app == nil {
|
||||
t.Fatal("precompiled parseState returned nil")
|
||||
}
|
||||
if app.PlayersOnline != 3 {
|
||||
t.Errorf("players_online = %d, want 3", app.PlayersOnline)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
"github.com/dbledeez/panel/pkg/steamvdf"
|
||||
)
|
||||
|
||||
// FetchAppInfoBranches runs a one-shot SteamCMD sidecar with
|
||||
//
|
||||
// +login anonymous +app_info_update 1 +app_info_print <appID> +quit
|
||||
//
|
||||
// captures its stdout, and parses the depots.branches map (branch name →
|
||||
// latest buildid). CHECK-ONLY: no game volumes are mounted, so this can
|
||||
// never touch an install. Only the shared panel-steamcmd-auth volume is
|
||||
// attached (same as update runs) so Steam's client config cache persists
|
||||
// across calls.
|
||||
//
|
||||
// tag disambiguates the sidecar container name per caller (usually the
|
||||
// instance id); appID appears too so parallel checks for different apps
|
||||
// can't collide.
|
||||
func FetchAppInfoBranches(ctx context.Context, rt runtime.Runtime, tag, appID string, log func(string)) (map[string]string, error) {
|
||||
if appID == "" {
|
||||
return nil, fmt.Errorf("app_info: app_id is required")
|
||||
}
|
||||
args := []string{
|
||||
"+login", "anonymous",
|
||||
"+app_info_update", "1",
|
||||
"+app_info_print", appID,
|
||||
"+quit",
|
||||
}
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: fmt.Sprintf("%s-appinfo-%s", tag, appID),
|
||||
IsSidecar: true,
|
||||
Image: sidecarImage,
|
||||
Command: args,
|
||||
Volumes: []runtime.VolumeSpec{{
|
||||
VolumeName: "panel-steamcmd-auth",
|
||||
ContainerPath: "/root/.local/share/Steam",
|
||||
Type: "volume",
|
||||
}},
|
||||
RestartPolicy: "no",
|
||||
// Same seccomp note as the update sidecar (steamcmd.go): newer
|
||||
// Docker default profiles ENOSYS a syscall the 32-bit Steam
|
||||
// runtime needs.
|
||||
SecurityOpts: []string{"seccomp=unconfined"},
|
||||
}
|
||||
contID, err := rt.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create app_info sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = rt.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := rt.Start(ctx, contID); err != nil {
|
||||
return nil, fmt.Errorf("start app_info sidecar: %w", err)
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = rt.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
out.WriteString(line)
|
||||
out.WriteByte('\n')
|
||||
})
|
||||
}()
|
||||
exitCode, err := rt.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wait app_info sidecar: %w", err)
|
||||
}
|
||||
if log != nil {
|
||||
log(fmt.Sprintf("app_info: steamcmd exit %d, %d bytes of output", exitCode, out.Len()))
|
||||
}
|
||||
// SteamCMD sometimes exits non-zero even after printing valid app
|
||||
// info — parse first, only surface the exit code when parsing fails.
|
||||
branches, perr := steamvdf.ParseAppInfoBranches(out.String(), appID)
|
||||
if perr != nil {
|
||||
if exitCode != 0 {
|
||||
return nil, fmt.Errorf("app_info: steamcmd exit %d (%v)", exitCode, perr)
|
||||
}
|
||||
return nil, perr
|
||||
}
|
||||
return branches, nil
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
archivepkg "github.com/dbledeez/panel/agent/internal/archive"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
"github.com/dbledeez/panel/pkg/version"
|
||||
)
|
||||
|
||||
// DirectProvider fetches a URL and writes the response body to a target
|
||||
// path under the instance. Used for vanilla Minecraft jars, custom mod
|
||||
// packs hosted on any HTTP server, etc.
|
||||
type DirectProvider struct{}
|
||||
|
||||
// Name identifies this provider kind in logs / manifest.
|
||||
func (DirectProvider) Name() string { return "direct" }
|
||||
|
||||
const maxDirectDownloadBytes = 1 << 30 // 1 GiB sanity cap
|
||||
|
||||
// Update downloads spec.URL and writes it to spec.TargetPath relative
|
||||
// to the instance. Optional SHA256 verification is reserved for a future
|
||||
// `sha256:` field on the spec.
|
||||
func (DirectProvider) Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error {
|
||||
if spec.URL == "" {
|
||||
return fmt.Errorf("direct provider: url is required")
|
||||
}
|
||||
if spec.TargetPath == "" {
|
||||
return fmt.Errorf("direct provider: target_path is required")
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", spec.URL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", version.UserAgent())
|
||||
|
||||
client := &http.Client{Timeout: 5 * time.Minute}
|
||||
uc.Log(fmt.Sprintf("GET %s", spec.URL))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("http status %d", resp.StatusCode)
|
||||
}
|
||||
if resp.ContentLength > 0 {
|
||||
uc.Log(fmt.Sprintf("Content-Length: %d bytes", resp.ContentLength))
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxDirectDownloadBytes+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
if int64(len(data)) > maxDirectDownloadBytes {
|
||||
return fmt.Errorf("response exceeds %d-byte cap", maxDirectDownloadBytes)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
uc.Log(fmt.Sprintf("downloaded %d bytes (sha256:%s)", len(data), hex.EncodeToString(sum[:])))
|
||||
|
||||
// Archive handling. Some direct URLs serve a whole game tree as an
|
||||
// archive (factorio's headless .tar.xz) — dumping those bytes as a
|
||||
// single file at TargetPath is never what the module wants. Extract
|
||||
// when the module opted in (spec.Extract) or when the content is a
|
||||
// recognized archive AND TargetPath looks like a directory (no file
|
||||
// extension). A TargetPath with an extension is always written
|
||||
// verbatim: minecraft server.jar IS a zip by magic bytes, and
|
||||
// terraria's entrypoint expects its .zip left on disk.
|
||||
format := archivepkg.DetectFormat(data, spec.URL)
|
||||
if spec.Extract && format == "" {
|
||||
return fmt.Errorf("extract requested but content is not a recognized archive (url %s)", spec.URL)
|
||||
}
|
||||
if format != "" && (spec.Extract || path.Ext(path.Base(spec.TargetPath)) == "") {
|
||||
uc.Log(fmt.Sprintf("content is a %s archive — extracting into %s", format, spec.TargetPath))
|
||||
entries, bytesOut, err := extractToInstance(ctx, uc, spec.TargetPath, data, spec.URL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extract into %s: %w", spec.TargetPath, err)
|
||||
}
|
||||
uc.Log(fmt.Sprintf("extracted %d entries (%d bytes) into %s", entries, bytesOut, spec.TargetPath))
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := writeToInstance(ctx, uc, spec.TargetPath, data); err != nil {
|
||||
return fmt.Errorf("write target: %w", err)
|
||||
}
|
||||
uc.Log(fmt.Sprintf("wrote %s", spec.TargetPath))
|
||||
return nil
|
||||
}
|
||||
|
||||
// containerTargetAbs resolves a provider target_path to a container-
|
||||
// absolute path. An absolute target that is equal to or under one of
|
||||
// the module's declared browseable roots or volume mount points is
|
||||
// used as-is (multi-root modules: factorio's /game volume sits BESIDE
|
||||
// its /game-saves default root). Anything else keeps the historical
|
||||
// behavior of joining under BrowseableRoot — terraria's entrypoint,
|
||||
// for one, expects /terraria-server.zip to land at
|
||||
// /game-saves/terraria-server.zip via exactly that join.
|
||||
func containerTargetAbs(uc *Context, targetPath string) string {
|
||||
if strings.HasPrefix(targetPath, "/") && uc.Manifest != nil {
|
||||
cleaned := path.Clean(targetPath)
|
||||
var roots []string
|
||||
if d := uc.Manifest.Runtime.Docker; d != nil {
|
||||
for _, r := range d.BrowseableRoots {
|
||||
if r.Path != "" {
|
||||
roots = append(roots, r.Path)
|
||||
}
|
||||
}
|
||||
for _, v := range d.Volumes {
|
||||
if v.Container != "" {
|
||||
roots = append(roots, v.Container)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, r := range roots {
|
||||
if cleaned == r || strings.HasPrefix(cleaned+"/", r+"/") {
|
||||
return cleaned
|
||||
}
|
||||
}
|
||||
}
|
||||
return path.Join(uc.BrowseableRoot, targetPath)
|
||||
}
|
||||
|
||||
// extractToInstance expands archive `data` into the instance-relative
|
||||
// directory targetPath. Mirrors writeToInstance's storage preference:
|
||||
// container-path ops when the instance has a BrowseableRoot and a
|
||||
// container (re-encode the archive as one tar stream and let Docker
|
||||
// extract it in place — same pattern as the file manager's FsExtract),
|
||||
// falling back to plain filesystem extraction under DataPath.
|
||||
func extractToInstance(ctx context.Context, uc *Context, targetPath string, data []byte, originalName string) (int64, int64, error) {
|
||||
if targetPath == "" {
|
||||
return 0, 0, fmt.Errorf("target_path is required")
|
||||
}
|
||||
if uc.BrowseableRoot != "" && uc.ContainerID != "" {
|
||||
destAbs := containerTargetAbs(uc, targetPath)
|
||||
// Best-effort mkdir; only possible on a running container, and
|
||||
// the usual target is a volume mount point that already exists.
|
||||
_, _, _, _ = uc.Runtime.ExecCapture(ctx, uc.ContainerID, []string{"sh", "-c", "mkdir -p -- '" + destAbs + "'"})
|
||||
pr, pw := io.Pipe()
|
||||
emitErr := make(chan error, 1)
|
||||
var entries, bytesOut int64
|
||||
go func() {
|
||||
n, b, err := archivepkg.WriteAsTar(data, originalName, pw)
|
||||
entries, bytesOut = n, b
|
||||
_ = pw.CloseWithError(err)
|
||||
emitErr <- err
|
||||
}()
|
||||
if err := uc.Runtime.CopyTarToContainer(ctx, uc.ContainerID, destAbs, pr); err != nil {
|
||||
// Prefer the encoder's error when it carries archive-format
|
||||
// detail — but NOT when it's just the pipe slamming shut
|
||||
// because Docker aborted the copy (that would mask the real
|
||||
// error, e.g. "destination directory does not exist").
|
||||
if encErr := <-emitErr; encErr != nil && !errors.Is(encErr, io.ErrClosedPipe) {
|
||||
return entries, bytesOut, encErr
|
||||
}
|
||||
return entries, bytesOut, fmt.Errorf("copy to container %s: %w", destAbs, err)
|
||||
}
|
||||
if encErr := <-emitErr; encErr != nil {
|
||||
return entries, bytesOut, encErr
|
||||
}
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if uc.DataPath == "" {
|
||||
return 0, 0, fmt.Errorf("no storage available: container has no BrowseableRoot and instance has no DataPath")
|
||||
}
|
||||
destAbs := filepath.Join(uc.DataPath, filepath.FromSlash(targetPath))
|
||||
if err := os.MkdirAll(destAbs, 0o755); err != nil {
|
||||
return 0, 0, fmt.Errorf("mkdir %s: %w", destAbs, err)
|
||||
}
|
||||
return archivepkg.Walk(data, originalName, func(name string, mode int64, isDir bool, body io.Reader) error {
|
||||
safe := archivepkg.SafeEntryPath(name)
|
||||
if safe == "" {
|
||||
return nil
|
||||
}
|
||||
out := filepath.Join(destAbs, filepath.FromSlash(safe))
|
||||
if isDir {
|
||||
return os.MkdirAll(out, 0o755)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
perm := os.FileMode(mode & 0o777)
|
||||
if perm == 0 {
|
||||
perm = 0o644
|
||||
}
|
||||
f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(f, body); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// makeFactorioTarXz builds a tiny tar.xz shaped like factorio's
|
||||
// headless tarball (factorio/bin/x64/factorio).
|
||||
func makeFactorioTarXz(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
xw, err := xz.NewWriter(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tw := tar.NewWriter(xw)
|
||||
body := []byte("ELF-not-really")
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "factorio/bin/x64/factorio", Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := xw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func testCtx(t *testing.T, dataPath string) *Context {
|
||||
t.Helper()
|
||||
return &Context{
|
||||
InstanceID: "test",
|
||||
DataPath: dataPath,
|
||||
Log: func(string) {},
|
||||
}
|
||||
}
|
||||
|
||||
// The factorio case: archive content + extensionless URL + extract:true
|
||||
// must land the extracted tree under TargetPath, not a raw blob.
|
||||
func TestDirectExtractsArchiveIntoTargetDir(t *testing.T) {
|
||||
blob := makeFactorioTarXz(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(blob)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/get-download/stable/headless/linux64", TargetPath: "/game", Extract: true}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin := filepath.Join(dir, "game", "factorio", "bin", "x64", "factorio")
|
||||
data, err := os.ReadFile(bin)
|
||||
if err != nil {
|
||||
t.Fatalf("expected extracted binary at %s: %v", bin, err)
|
||||
}
|
||||
if string(data) != "ELF-not-really" {
|
||||
t.Fatalf("binary content mismatch: %q", data)
|
||||
}
|
||||
// The old bug: raw blob written AS the target path.
|
||||
if fi, err := os.Stat(filepath.Join(dir, "game")); err != nil || !fi.IsDir() {
|
||||
t.Fatalf("target path should be a directory, got err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Same archive, no explicit extract flag: extensionless TargetPath
|
||||
// still triggers extraction (directory heuristic).
|
||||
func TestDirectHeuristicExtractsWithoutFlag(t *testing.T) {
|
||||
blob := makeFactorioTarXz(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(blob)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/linux64", TargetPath: "/game"}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "game", "factorio", "bin", "x64", "factorio")); err != nil {
|
||||
t.Fatalf("expected extraction: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The terraria/minecraft guard: a TargetPath WITH a file extension is
|
||||
// written verbatim even when the content has archive magic (jars are
|
||||
// zips; terraria's entrypoint unzips its own .zip).
|
||||
func TestDirectWritesArchiveVerbatimWhenTargetHasExtension(t *testing.T) {
|
||||
// A zip blob (like a .jar or terraria-server zip).
|
||||
var zipBuf bytes.Buffer
|
||||
zipBuf.Write([]byte{0x50, 0x4b, 0x03, 0x04}) // zip local-header magic
|
||||
zipBuf.WriteString("rest-of-zip-not-parsed")
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(zipBuf.Bytes())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/terraria-server-1449.zip", TargetPath: "/terraria-server.zip"}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, "terraria-server.zip"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(data, zipBuf.Bytes()) {
|
||||
t.Fatal("zip should have been written verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
// extract:true on non-archive content is a hard error, not a silent
|
||||
// verbatim write.
|
||||
func TestDirectExtractFlagOnNonArchiveErrors(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("just text"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/blob", TargetPath: "/game", Extract: true}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err == nil {
|
||||
t.Fatal("expected error for extract on non-archive content")
|
||||
}
|
||||
}
|
||||
|
||||
// containerTargetAbs: absolute targets matching a declared root or
|
||||
// volume mount stay container-absolute (factorio's /game volume);
|
||||
// anything else keeps the historical BrowseableRoot join (terraria's
|
||||
// /terraria-server.zip → /game-saves/terraria-server.zip).
|
||||
func TestContainerTargetAbs(t *testing.T) {
|
||||
uc := &Context{
|
||||
BrowseableRoot: "/game-saves",
|
||||
Manifest: &modulepkg.Manifest{
|
||||
Runtime: modulepkg.Runtime{Docker: &modulepkg.DockerSpec{
|
||||
BrowseableRoots: []modulepkg.BrowseableRoot{{Name: "Saves", Path: "/game-saves"}},
|
||||
Volumes: []modulepkg.Volume{
|
||||
{Type: "volume", Name: "panel-$INSTANCE_ID-saves", Container: "/game-saves"},
|
||||
{Type: "volume", Name: "panel-$INSTANCE_ID-game", Container: "/game"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
cases := map[string]string{
|
||||
"/game": "/game",
|
||||
"/game/sub": "/game/sub",
|
||||
"/game-saves/mods": "/game-saves/mods",
|
||||
"/terraria-server.zip": "/game-saves/terraria-server.zip",
|
||||
"relative/file": "/game-saves/relative/file",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := containerTargetAbs(uc, in); got != want {
|
||||
t.Errorf("containerTargetAbs(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Non-archive content with an extensionless target keeps the old
|
||||
// single-file write behavior.
|
||||
func TestDirectNonArchiveStillWritesFile(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("binary-ish payload"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/blob", TargetPath: "/some/file"}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, "some", "file"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "binary-ish payload" {
|
||||
t.Fatalf("content mismatch: %q", data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
"github.com/dbledeez/panel/pkg/version"
|
||||
)
|
||||
|
||||
// GitHubProvider queries a repo's latest release, finds an asset whose
|
||||
// name matches the provider's AssetRegex, and writes it to TargetPath.
|
||||
type GitHubProvider struct{}
|
||||
|
||||
// Name identifies this provider kind in logs / manifest.
|
||||
func (GitHubProvider) Name() string { return "github" }
|
||||
|
||||
type githubAsset struct {
|
||||
Name string `json:"name"`
|
||||
DownloadURL string `json:"browser_download_url"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type githubRelease struct {
|
||||
Name string `json:"name"`
|
||||
TagName string `json:"tag_name"`
|
||||
Assets []githubAsset `json:"assets"`
|
||||
}
|
||||
|
||||
// Update fetches the latest release of spec.Repo, picks the first asset
|
||||
// matching spec.AssetRegex, and writes its bytes to spec.TargetPath.
|
||||
func (GitHubProvider) Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error {
|
||||
if spec.Repo == "" {
|
||||
return fmt.Errorf("github provider: repo is required (owner/name)")
|
||||
}
|
||||
if spec.AssetRegex == "" {
|
||||
return fmt.Errorf("github provider: asset_regex is required")
|
||||
}
|
||||
if spec.TargetPath == "" {
|
||||
return fmt.Errorf("github provider: target_path is required")
|
||||
}
|
||||
re, err := regexp.Compile(spec.AssetRegex)
|
||||
if err != nil {
|
||||
return fmt.Errorf("asset_regex: %w", err)
|
||||
}
|
||||
|
||||
apiURL := fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", spec.Repo)
|
||||
uc.Log(fmt.Sprintf("querying GitHub: %s", apiURL))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("User-Agent", version.UserAgent())
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
return fmt.Errorf("github api %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
var release githubRelease
|
||||
if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
|
||||
return fmt.Errorf("decode release: %w", err)
|
||||
}
|
||||
uc.Log(fmt.Sprintf("latest release: %s (%s), %d assets", release.Name, release.TagName, len(release.Assets)))
|
||||
|
||||
var chosen *githubAsset
|
||||
for i := range release.Assets {
|
||||
if re.MatchString(release.Assets[i].Name) {
|
||||
chosen = &release.Assets[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if chosen == nil {
|
||||
return fmt.Errorf("no asset matches %q (saw: %s)", spec.AssetRegex, assetNames(release.Assets))
|
||||
}
|
||||
uc.Log(fmt.Sprintf("downloading asset %s (%d bytes)", chosen.Name, chosen.Size))
|
||||
|
||||
direct := &modulepkg.UpdateProvider{
|
||||
Kind: "direct",
|
||||
URL: chosen.DownloadURL,
|
||||
TargetPath: spec.TargetPath,
|
||||
}
|
||||
return DirectProvider{}.Update(ctx, uc, direct)
|
||||
}
|
||||
|
||||
func assetNames(as []githubAsset) string {
|
||||
out := ""
|
||||
for _, a := range as {
|
||||
if out != "" {
|
||||
out += ", "
|
||||
}
|
||||
out += a.Name
|
||||
}
|
||||
if out == "" {
|
||||
out = "(none)"
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Package updater implements the Generic Update system: pluggable
|
||||
// providers that fetch new versions of game files from SteamCMD,
|
||||
// GitHub releases, or a direct URL, then lay them down under the
|
||||
// instance's data path (bind mount) or container volume.
|
||||
//
|
||||
// The panel's manifest declares one or more update_providers per module;
|
||||
// the operator picks one to trigger. Each provider runs on the *Agent*
|
||||
// (close to the data) and streams progress back as LogLine envelopes
|
||||
// so the dashboard renders updates in the same events pane as normal
|
||||
// server output.
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// Context is the environment an update runs in.
|
||||
type Context struct {
|
||||
InstanceID string
|
||||
Manifest *modulepkg.Manifest
|
||||
ContainerID string // empty if container doesn't exist; present for running + stopped
|
||||
BrowseableRoot string // container-absolute install root, when applicable
|
||||
DataPath string // host bind-mount root, fallback
|
||||
Runtime runtime.Runtime // for container file ops + sidecar launches (future)
|
||||
Log func(line string) // streams a progress line upstream via LogLine
|
||||
|
||||
// Steam login credentials for SteamCMD apps that refuse +login
|
||||
// anonymous (DayZ, Arma, etc.). Populated by the controller from its
|
||||
// encrypted steam_credentials store, forwarded via the mTLS-protected
|
||||
// gRPC stream. The agent redacts SteamPassword from log output before
|
||||
// it ever hits the bus; see steamcmd.go.
|
||||
SteamUsername string
|
||||
SteamPassword string
|
||||
}
|
||||
|
||||
// Provider performs an update according to its kind.
|
||||
type Provider interface {
|
||||
Name() string
|
||||
Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error
|
||||
}
|
||||
|
||||
// Get returns the Provider implementation for the given kind, or an
|
||||
// error if the kind is unknown.
|
||||
func Get(kind string) (Provider, error) {
|
||||
switch kind {
|
||||
case "direct":
|
||||
return &DirectProvider{}, nil
|
||||
case "github":
|
||||
return &GitHubProvider{}, nil
|
||||
case "steamcmd":
|
||||
return &SteamCMDProvider{}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown update provider kind %q", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// writeToInstance puts data at the instance-relative path. Prefers
|
||||
// container-path ops when the instance has a BrowseableRoot and a live
|
||||
// container; otherwise falls back to writing under DataPath.
|
||||
func writeToInstance(ctx context.Context, uc *Context, targetPath string, data []byte) error {
|
||||
if targetPath == "" {
|
||||
return fmt.Errorf("target_path is required")
|
||||
}
|
||||
if uc.BrowseableRoot != "" && uc.ContainerID != "" {
|
||||
abs := path.Join(uc.BrowseableRoot, targetPath)
|
||||
// Ensure parent dir exists inside the container via runtime exec.
|
||||
// We don't have a helper in the interface; we just try to write
|
||||
// and rely on CopyFileToContainer returning a clear error if the
|
||||
// parent is missing. (itzg and most images pre-create /data.)
|
||||
return uc.Runtime.CopyFileToContainer(ctx, uc.ContainerID, abs, data)
|
||||
}
|
||||
if uc.DataPath == "" {
|
||||
return fmt.Errorf("no storage available: container has no BrowseableRoot and instance has no DataPath")
|
||||
}
|
||||
abs := filepath.Join(uc.DataPath, filepath.FromSlash(targetPath))
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", filepath.Dir(abs), err)
|
||||
}
|
||||
if err := os.WriteFile(abs, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write %s: %w", abs, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// SteamCMDProvider runs `steamcmd +force_install_dir <path> +login
|
||||
// anonymous +app_update <appid> [-beta <branch>] validate +quit` in a
|
||||
// one-shot sidecar container that mounts the target instance's volumes.
|
||||
// The sidecar writes game files into the same Docker volumes the main
|
||||
// container reads from, so "update then start" just works.
|
||||
//
|
||||
// Preconditions:
|
||||
// - Target instance's container must NOT be running (Steam holds file
|
||||
// locks that conflict with a live server). Error clearly otherwise.
|
||||
// - Manifest must declare at least one volume; install_path resolves
|
||||
// to spec.InstallPath → BrowseableRoot → first volume's container path.
|
||||
type SteamCMDProvider struct{}
|
||||
|
||||
// Name identifies this provider kind in logs / manifest.
|
||||
func (SteamCMDProvider) Name() string { return "steamcmd" }
|
||||
|
||||
// sidecarImage is the official Valve-maintained SteamCMD image. Its
|
||||
// ENTRYPOINT is `steamcmd`, so we pass steamcmd's `+...` flags as Cmd.
|
||||
const sidecarImage = "steamcmd/steamcmd:latest"
|
||||
|
||||
// redactSteamPassword replaces a password in a cmdline arg slice with
|
||||
// "[REDACTED]" so the Console tab never echoes it. Used before logging
|
||||
// steamcmd argv.
|
||||
func redactSteamPassword(args []string, password string) []string {
|
||||
if password == "" {
|
||||
return args
|
||||
}
|
||||
out := make([]string, len(args))
|
||||
for i, a := range args {
|
||||
if a == password {
|
||||
out[i] = "[REDACTED]"
|
||||
continue
|
||||
}
|
||||
out[i] = a
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// redactPasswordLine strips the password substring from a log line if it
|
||||
// ever appears (e.g. SteamCMD echoing `+login user PASS` back). Returns
|
||||
// the original string when password is empty so anonymous/public flows
|
||||
// are zero-cost.
|
||||
func redactPasswordLine(line, password string) string {
|
||||
if password == "" || len(password) < 4 {
|
||||
return line
|
||||
}
|
||||
if !strings.Contains(line, password) {
|
||||
return line
|
||||
}
|
||||
return strings.ReplaceAll(line, password, "[REDACTED]")
|
||||
}
|
||||
|
||||
func (SteamCMDProvider) Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error {
|
||||
if spec.AppID == "" {
|
||||
return fmt.Errorf("steamcmd provider: app_id is required")
|
||||
}
|
||||
|
||||
// Refuse if the main container is still running — Steam locks files.
|
||||
mainName := "panel-" + uc.InstanceID
|
||||
if state, err := uc.Runtime.InspectByName(ctx, mainName); err == nil && state.Status == "running" {
|
||||
return fmt.Errorf("stop the instance first — container %q is running; SteamCMD conflicts with a live server", mainName)
|
||||
}
|
||||
|
||||
installPath := spec.InstallPath
|
||||
if installPath == "" {
|
||||
installPath = uc.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return fmt.Errorf("steamcmd provider: install_path missing and module has no browseable_root/volumes to default from")
|
||||
}
|
||||
|
||||
// Mount the same volumes as the target instance so SteamCMD writes
|
||||
// where the main container will later read.
|
||||
volumes := agentmodule.ResolveVolumes(uc.Manifest, uc.InstanceID, uc.DataPath)
|
||||
if len(volumes) == 0 {
|
||||
return fmt.Errorf("steamcmd provider: module declares no volumes to mount")
|
||||
}
|
||||
|
||||
// Platform override MUST come before +login — SteamCMD evaluates its
|
||||
// forced-platform state at login time, not at app_update time. Used for
|
||||
// apps that ship only a Windows depot (run via Wine on Linux hosts).
|
||||
args := []string{}
|
||||
if spec.Platform != "" {
|
||||
args = append(args, "+@sSteamCmdForcePlatformType", spec.Platform)
|
||||
}
|
||||
args = append(args, "+force_install_dir", installPath)
|
||||
|
||||
// Non-anonymous Steam login for paid apps (DayZ, Arma). If the module
|
||||
// declared requires_steam_login and the controller forwarded creds,
|
||||
// use them. Otherwise fall back to anonymous (most free-to-download
|
||||
// dedicated servers work anonymously).
|
||||
//
|
||||
// CRITICAL for UX: pass ONLY the username when possible. SteamCMD's
|
||||
// `+login <user>` uses the cached refresh token from config.vdf
|
||||
// (persisted in panel-steamcmd-auth) — no Steam Guard push, no
|
||||
// password round-trip. Passing `+login <user> <password>` forces a
|
||||
// fresh password auth every time, which triggers a 2FA push on every
|
||||
// call even when a perfectly valid cached token is already on disk.
|
||||
//
|
||||
// First-ever login ceremony happens in the controller (steamauth.go)
|
||||
// with the full password+guard flow to prime the cache. After that,
|
||||
// this path rides the cache. If the cache is stale/expired, SteamCMD
|
||||
// exits non-zero — the retry loop below re-runs which will hit the
|
||||
// same error; the operator then clicks Update, which the controller
|
||||
// can short-circuit to re-run the login modal (the `steam_login_required`
|
||||
// response path).
|
||||
needsLogin := spec.RequiresSteamLogin && uc.SteamUsername != ""
|
||||
if needsLogin {
|
||||
args = append(args, "+login", uc.SteamUsername)
|
||||
} else {
|
||||
args = append(args, "+login", "anonymous")
|
||||
}
|
||||
updateArgs := []string{"+app_update", spec.AppID}
|
||||
if spec.Beta != "" {
|
||||
updateArgs = append(updateArgs, "-beta", spec.Beta)
|
||||
}
|
||||
if !spec.SkipValidate {
|
||||
updateArgs = append(updateArgs, "validate")
|
||||
}
|
||||
args = append(args, updateArgs...)
|
||||
args = append(args, "+quit")
|
||||
|
||||
uc.Log(fmt.Sprintf("steamcmd: image=%s install_path=%s app_id=%s beta=%s", sidecarImage, installPath, spec.AppID, spec.Beta))
|
||||
uc.Log(fmt.Sprintf("steamcmd argv: %v", redactSteamPassword(args, uc.SteamPassword)))
|
||||
|
||||
// SteamCMD is notoriously flaky on first invocation — exit 8 ("missing
|
||||
// configuration"), exit 2 (generic "retry later"), and Steam auth races
|
||||
// are all recoverable by simply running the same command again. AMP,
|
||||
// LinuxGSM, and Valve's own Steam client all retry on these codes. We
|
||||
// mirror that behaviour so operators don't have to click Update twice.
|
||||
const maxAttempts = 4
|
||||
var lastExit int
|
||||
var ranSuccessfully bool
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
exitCode, err := runSteamCMDOnce(ctx, uc, sidecarImage, args, volumes, attempt)
|
||||
lastExit = exitCode
|
||||
if err == nil && exitCode == 0 {
|
||||
uc.Log(fmt.Sprintf("steamcmd: completed successfully (app_id=%s, attempts=%d)", spec.AppID, attempt))
|
||||
ranSuccessfully = true
|
||||
break
|
||||
}
|
||||
// Transient exit codes SteamCMD is known to recover from. Anything
|
||||
// else we surface immediately (missing disk space, bad app id, etc.).
|
||||
transient := err == nil && (exitCode == 2 || exitCode == 5 || exitCode == 8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transient || attempt == maxAttempts {
|
||||
break
|
||||
}
|
||||
backoff := time.Duration(attempt*attempt*3) * time.Second // 3s, 12s, 27s
|
||||
uc.Log(fmt.Sprintf("steamcmd: exit %d — transient, retrying in %s (attempt %d/%d)", exitCode, backoff, attempt+1, maxAttempts))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
|
||||
// Finalize: SteamCMD with `+force_install_dir <p>` stages downloads
|
||||
// at <p>/steamapps/downloading/<appid>/ and atomically renames into
|
||||
// <p>/ on success. When SteamCMD exits with state 0x602 (often after
|
||||
// concurrent runs interrupt finalization, or a `validate` pass races
|
||||
// the move) the bytes are on disk but never get renamed — operator
|
||||
// sees an empty install dir and an exit-8 error from the panel.
|
||||
//
|
||||
// This step runs unconditionally after the retry loop. If the staging
|
||||
// dir is empty or absent (the happy path), it's a cheap no-op. If it
|
||||
// has content, we cp -a it into place and the install completes.
|
||||
finalized, ferr := finalizeStagingDir(ctx, uc, installPath, spec.AppID, volumes)
|
||||
if ferr != nil {
|
||||
uc.Log(fmt.Sprintf("steamcmd: finalize check failed: %s (continuing anyway)", ferr.Error()))
|
||||
} else if finalized {
|
||||
uc.Log("steamcmd: recovered staged install (state 0x602 workaround)")
|
||||
return nil
|
||||
}
|
||||
|
||||
if ranSuccessfully {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("steamcmd exited %d after %d attempt(s)", lastExit, maxAttempts)
|
||||
}
|
||||
|
||||
// finalizeStagingDir checks for a stranded SteamCMD staging directory
|
||||
// at <installPath>/steamapps/downloading/<appid>/ and, if present and
|
||||
// non-empty, moves its contents into <installPath>/. Runs as a one-shot
|
||||
// alpine sidecar with the same volume mounts as the SteamCMD run so
|
||||
// it has access to the install path. Returns true if a recovery move
|
||||
// actually happened, false if there was nothing to do.
|
||||
//
|
||||
// The script is intentionally defensive:
|
||||
// - mv is preferred over cp+rm because it's atomic on the same FS.
|
||||
// - If both source and destination have entries with the same name,
|
||||
// mv -f overwrites (rare; happens when steamcmd partially renamed
|
||||
// before dying).
|
||||
// - We exit 0 on "no staging dir" so this is a no-op for the happy
|
||||
// path.
|
||||
func finalizeStagingDir(ctx context.Context, uc *Context, installPath, appID string, volumes []runtime.VolumeSpec) (bool, error) {
|
||||
const stagingMarker = "/__staging__"
|
||||
script := fmt.Sprintf(`set -e
|
||||
DL=%q/steamapps/downloading/%s
|
||||
DEST=%q
|
||||
if [ ! -d "$DL" ] || [ -z "$(ls -A "$DL" 2>/dev/null)" ]; then
|
||||
echo "no-staging"
|
||||
exit 0
|
||||
fi
|
||||
echo "finalizing $DL -> $DEST"
|
||||
mkdir -p "$DEST"
|
||||
# mv every top-level entry; -f overwrites partial renames.
|
||||
for f in "$DL"/* "$DL"/.[!.]* "$DL"/..?*; do
|
||||
[ -e "$f" ] || continue
|
||||
mv -f "$f" "$DEST/" 2>&1 || cp -a "$f" "$DEST/" && rm -rf "$f"
|
||||
done
|
||||
rm -rf "$DL"
|
||||
touch %q
|
||||
echo "finalized"
|
||||
`, installPath, appID, installPath, installPath+stagingMarker)
|
||||
|
||||
sidecarSpec := runtime.InstanceSpec{
|
||||
InstanceID: fmt.Sprintf("%s-steamcmd-finalize", uc.InstanceID),
|
||||
IsSidecar: true,
|
||||
Image: "alpine:latest",
|
||||
Command: []string{"sh", "-c", script},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
contID, err := uc.Runtime.Create(ctx, sidecarSpec)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("create finalize sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = uc.Runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := uc.Runtime.Start(ctx, contID); err != nil {
|
||||
return false, fmt.Errorf("start finalize sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
var lastLine string
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = uc.Runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
lastLine = strings.TrimSpace(line)
|
||||
uc.Log("finalize: " + lastLine)
|
||||
})
|
||||
}()
|
||||
exitCode, err := uc.Runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("wait finalize sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return false, fmt.Errorf("finalize sidecar exit %d", exitCode)
|
||||
}
|
||||
return lastLine == "finalized", nil
|
||||
}
|
||||
|
||||
// runSteamCMDOnce is a single run of the steamcmd sidecar; called in a loop
|
||||
// by the retry wrapper. Returns the container's exit code on clean
|
||||
// lifecycle (even non-zero) or an error on infrastructure failure.
|
||||
func runSteamCMDOnce(ctx context.Context, uc *Context, image string, args []string, volumes []runtime.VolumeSpec, attempt int) (int, error) {
|
||||
sidecarID := fmt.Sprintf("%s-steamcmd-%d", uc.InstanceID, attempt)
|
||||
// Mount the panel-wide SteamCMD auth volume at /root/.local/share/Steam
|
||||
// (the real data dir of the steamcmd:latest image — NOT /root/Steam).
|
||||
// This persists config/config.vdf (login refresh token) + ssfn sentry,
|
||||
// which lets subsequent `+login <user>` runs skip the 2FA push. Anon
|
||||
// logins ignore this dir — no harm.
|
||||
volumesWithAuth := append([]runtime.VolumeSpec(nil), volumes...)
|
||||
volumesWithAuth = append(volumesWithAuth, runtime.VolumeSpec{
|
||||
VolumeName: "panel-steamcmd-auth",
|
||||
ContainerPath: "/root/.local/share/Steam",
|
||||
Type: "volume",
|
||||
})
|
||||
sidecarSpec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: image,
|
||||
Command: args,
|
||||
Volumes: volumesWithAuth,
|
||||
RestartPolicy: "no",
|
||||
// Docker 29.4+ default seccomp ENOSYS's a syscall the 32-bit
|
||||
// Steam runtime needs ("CreateBoundSocket error 38"). Older
|
||||
// Docker (29.1.x on princess/ivy) doesn't filter it. Unconfine
|
||||
// for the sidecar so install/validate works across hosts.
|
||||
SecurityOpts: []string{"seccomp=unconfined"},
|
||||
}
|
||||
contID, err := uc.Runtime.Create(ctx, sidecarSpec)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("create sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = uc.Runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := uc.Runtime.Start(ctx, contID); err != nil {
|
||||
return 0, fmt.Errorf("start sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = uc.Runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
uc.Log(redactPasswordLine(line, uc.SteamPassword))
|
||||
})
|
||||
}()
|
||||
exitCode, err := uc.Runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("wait sidecar: %w", err)
|
||||
}
|
||||
return exitCode, nil
|
||||
}
|
||||
Reference in New Issue
Block a user