panel v0.9.1 — open-source game server manager
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/dispatch"
|
||||
agentruntime "github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
"github.com/dbledeez/panel/pkg/version"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// loadClientTLS returns gRPC credentials configured for mTLS against
|
||||
// the controller's CA, or nil if cert files aren't present (caller then
|
||||
// decides whether to refuse or fall back to plaintext). The returned CN
|
||||
// is the agent identity the controller issued during pairing — callers
|
||||
// should use it as the agent_id so identity stays cryptographically bound
|
||||
// to the cert instead of drifting to whatever hostname the process sees.
|
||||
func loadClientTLS(certDir, controllerAddr string) (credentials.TransportCredentials, string, error) {
|
||||
caPath := filepath.Join(certDir, "ca.crt")
|
||||
certPath := filepath.Join(certDir, "agent.crt")
|
||||
keyPath := filepath.Join(certDir, "agent.key")
|
||||
|
||||
for _, p := range []string{caPath, certPath, keyPath} {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
return nil, "", fmt.Errorf("missing %s: %w", p, err)
|
||||
}
|
||||
}
|
||||
caPEM, err := os.ReadFile(caPath)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, "", fmt.Errorf("bad CA PEM")
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("load cert+key: %w", err)
|
||||
}
|
||||
var cn string
|
||||
if len(cert.Certificate) > 0 {
|
||||
if leaf, perr := x509.ParseCertificate(cert.Certificate[0]); perr == nil {
|
||||
cn = leaf.Subject.CommonName
|
||||
}
|
||||
}
|
||||
// ServerName defaults to the host part of controllerAddr. If it's
|
||||
// an IP we still set ServerName so Go's TLS stack uses SNI; the
|
||||
// controller cert includes 127.0.0.1 as an IP SAN so this works.
|
||||
serverName := controllerAddr
|
||||
if i := strings.LastIndex(serverName, ":"); i > 0 {
|
||||
serverName = serverName[:i]
|
||||
}
|
||||
tlsCfg := &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
RootCAs: pool,
|
||||
ServerName: serverName,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
return credentials.NewTLS(tlsCfg), cn, nil
|
||||
}
|
||||
|
||||
// agentVersion mirrors the shared build version (ldflags-injected via
|
||||
// github.com/dbledeez/panel/pkg/version.Version; "dev" in ad-hoc builds).
|
||||
var agentVersion = version.Version
|
||||
|
||||
func hostname() string {
|
||||
h, err := os.Hostname()
|
||||
if err != nil || h == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func main() {
|
||||
controllerAddr := flag.String("controller", "localhost:8443", "controller gRPC address host:port")
|
||||
agentID := flag.String("agent-id", "", "agent identifier (default: mTLS cert CN, or hostname when --insecure)")
|
||||
modulesDir := flag.String("modules-dir", "./modules", "path to modules directory")
|
||||
dataRoot := flag.String("data-root", "./data/instances", "root directory for instance data")
|
||||
metaDir := flag.String("meta-dir", "./data/instance-meta", "directory for per-instance sidecar metadata (survives agent restart)")
|
||||
backupDir := flag.String("backup-dir", "./data/backups", "directory where instance backup archives are written")
|
||||
certDir := flag.String("cert-dir", "./data/certs", "directory holding agent.crt + agent.key + ca.crt after pairing")
|
||||
heartbeatInterval := flag.Duration("heartbeat", 10*time.Second, "heartbeat interval")
|
||||
insecureDial := flag.Bool("insecure", false, "force plaintext (no TLS). Use --insecure=true only for dev/local setups.")
|
||||
pairToken := flag.String("pair-token", "", "if set, agent runs in pair mode: fetches CA + signed cert from --pair-url and exits")
|
||||
pairURL := flag.String("pair-url", "http://localhost:8080", "controller HTTP URL for pair/CA/issue endpoints (used with --pair-token)")
|
||||
logLevel := flag.String("log-level", "info", "log level: debug, info, warn, error")
|
||||
flag.Parse()
|
||||
|
||||
var lvl slog.Level
|
||||
if err := lvl.UnmarshalText([]byte(*logLevel)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "invalid log level %q: %v\n", *logLevel, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
// Pair mode: fetch CA + sign a client cert, then exit. Pair runs before
|
||||
// we touch the local cert so the agent-id has to come from either --agent-id
|
||||
// or hostname — there's no cert CN to read yet.
|
||||
if *pairToken != "" {
|
||||
if *agentID == "" {
|
||||
*agentID = hostname()
|
||||
}
|
||||
pairLogger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})).
|
||||
With("agent_id", *agentID)
|
||||
if err := runPair(pairConfig{
|
||||
AgentID: *agentID,
|
||||
PairURL: *pairURL,
|
||||
Token: *pairToken,
|
||||
CertDir: *certDir,
|
||||
Logger: pairLogger,
|
||||
}); err != nil {
|
||||
pairLogger.Error("pair failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// TLS creds: try loading cert files; fall back if --insecure.
|
||||
// The cert's CN is the canonical agent identity. If --agent-id wasn't
|
||||
// set explicitly, adopt the CN; if it was set to something different,
|
||||
// refuse — a mismatched identity just silently orphans instances in the
|
||||
// DB (the exact bug that motivated this change).
|
||||
var tlsCreds credentials.TransportCredentials
|
||||
var certCN string
|
||||
creds, cn, tlsErr := loadClientTLS(*certDir, *controllerAddr)
|
||||
switch {
|
||||
case tlsErr == nil:
|
||||
tlsCreds = creds
|
||||
certCN = cn
|
||||
case *insecureDial:
|
||||
// Plaintext path — cert CN unavailable, fall through to hostname.
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "no TLS certs found and --insecure not set. Run pairing first:\n"+
|
||||
" agent --pair-token <token> --pair-url http://controller:8080 --cert-dir %s\n"+
|
||||
"detail: %v\n", *certDir, tlsErr)
|
||||
os.Exit(2)
|
||||
}
|
||||
switch {
|
||||
case *agentID == "" && certCN != "":
|
||||
*agentID = certCN
|
||||
case *agentID == "":
|
||||
*agentID = hostname()
|
||||
case certCN != "" && *agentID != certCN:
|
||||
fmt.Fprintf(os.Stderr, "--agent-id=%q does not match mTLS cert CN=%q; refusing to start.\n"+
|
||||
"Re-pair the agent with the intended id or drop --agent-id to adopt the cert CN.\n",
|
||||
*agentID, certCN)
|
||||
os.Exit(2)
|
||||
}
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})).
|
||||
With("agent_id", *agentID)
|
||||
if tlsCreds != nil {
|
||||
logger.Info("mTLS enabled", "cert_dir", *certDir, "cert_cn", certCN)
|
||||
} else {
|
||||
logger.Warn("running WITHOUT TLS (--insecure) — agent traffic is plaintext")
|
||||
}
|
||||
|
||||
absDataRoot, err := filepath.Abs(*dataRoot)
|
||||
if err != nil {
|
||||
logger.Error("resolve data-root", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.MkdirAll(absDataRoot, 0o755); err != nil {
|
||||
logger.Error("mkdir data-root", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
absMetaDir, err := filepath.Abs(*metaDir)
|
||||
if err != nil {
|
||||
logger.Error("resolve meta-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.MkdirAll(absMetaDir, 0o755); err != nil {
|
||||
logger.Error("mkdir meta-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
absBackupDir, err := filepath.Abs(*backupDir)
|
||||
if err != nil {
|
||||
logger.Error("resolve backup-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.MkdirAll(absBackupDir, 0o755); err != nil {
|
||||
logger.Error("mkdir backup-dir", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
registry := modulepkg.NewRegistry()
|
||||
if err := registry.LoadDir(*modulesDir); err != nil {
|
||||
logger.Warn("module registry load failed", "dir", *modulesDir, "err", err)
|
||||
}
|
||||
logger.Info("modules loaded", "count", registry.Len(), "dir", *modulesDir)
|
||||
for _, m := range registry.List() {
|
||||
logger.Info("module registered", "id", m.ID, "name", m.Name, "version", m.Version)
|
||||
}
|
||||
|
||||
dockerRT, err := agentruntime.NewDocker()
|
||||
if err != nil {
|
||||
logger.Error("docker runtime init", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer dockerRT.Close()
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
if err := dockerRT.Ping(ctx); err != nil {
|
||||
logger.Warn("docker daemon not reachable; instance commands will fail", "err", err)
|
||||
} else {
|
||||
logger.Info("docker runtime ready")
|
||||
}
|
||||
|
||||
disp := dispatch.New(logger, registry, dockerRT, absDataRoot, absMetaDir, absBackupDir)
|
||||
|
||||
if err := disp.Rehydrate(ctx); err != nil {
|
||||
logger.Warn("rehydrate failed (agent will start fresh)", "err", err)
|
||||
}
|
||||
|
||||
// Reconnect loop: keep dialing the controller until ctx is cancelled.
|
||||
// A session ends either because ctx was cancelled (operator signal)
|
||||
// or because the stream died (controller blip, network, etc.).
|
||||
backoff := 2 * time.Second
|
||||
const maxBackoff = 30 * time.Second
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
err := runSession(ctx, logger, *controllerAddr, *agentID, *heartbeatInterval, disp, tlsCreds)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
logger.Warn("session ended, will reconnect", "err", err, "in", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(backoff + jitter(backoff)):
|
||||
}
|
||||
if backoff < maxBackoff {
|
||||
backoff *= 2
|
||||
if backoff > maxBackoff {
|
||||
backoff = maxBackoff
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("operator signal — gracefully stopping running instances")
|
||||
disp.Shutdown(context.Background())
|
||||
}
|
||||
|
||||
// runSession opens one gRPC connection, handshakes, (re)announces current
|
||||
// instance state, then runs the recv/send loops until the stream ends.
|
||||
// Returns nil if the operator signalled shutdown, or an error if the
|
||||
// connection broke.
|
||||
func runSession(ctx context.Context, logger *slog.Logger, addr, agentID string, hb time.Duration, disp *dispatch.Dispatcher, tlsCreds credentials.TransportCredentials) error {
|
||||
var dialOpt grpc.DialOption
|
||||
if tlsCreds != nil {
|
||||
dialOpt = grpc.WithTransportCredentials(tlsCreds)
|
||||
} else {
|
||||
dialOpt = grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
}
|
||||
// Lift the default 4 MiB per-message cap so big file uploads
|
||||
// (FsWrite carrying mod zips, scenario archives, save backups)
|
||||
// flow through end-to-end. Matches the controller-side cap.
|
||||
const grpcMaxMsgSize = 5 * 1024 * 1024 * 1024
|
||||
conn, err := grpc.NewClient(addr, dialOpt,
|
||||
grpc.WithDefaultCallOptions(
|
||||
grpc.MaxCallRecvMsgSize(grpcMaxMsgSize),
|
||||
grpc.MaxCallSendMsgSize(grpcMaxMsgSize),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("grpc client: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := panelv1.NewAgentClient(conn)
|
||||
|
||||
logger.Info("connecting to controller", "addr", addr)
|
||||
stream, err := client.Connect(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open stream: %w", err)
|
||||
}
|
||||
|
||||
hello := &panelv1.AgentEnvelope{
|
||||
CorrelationId: "hello",
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Hello{
|
||||
Hello: &panelv1.AgentHello{
|
||||
AgentId: agentID,
|
||||
AgentVersion: agentVersion,
|
||||
HostOs: runtime.GOOS,
|
||||
HostArch: runtime.GOARCH,
|
||||
Hostname: hostname(),
|
||||
SupportedRuntimes: []string{"docker"},
|
||||
Modules: summarizeModules(disp),
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := stream.Send(hello); err != nil {
|
||||
return fmt.Errorf("send hello: %w", err)
|
||||
}
|
||||
|
||||
resp, err := stream.Recv()
|
||||
if err != nil {
|
||||
return fmt.Errorf("recv hello: %w", err)
|
||||
}
|
||||
ch := resp.GetHello()
|
||||
if ch == nil {
|
||||
return fmt.Errorf("unexpected first message: %T", resp.Payload)
|
||||
}
|
||||
logger.Info("handshake complete", "controller_version", ch.ControllerVersion)
|
||||
|
||||
// Install the current stream as the upward sender and re-announce.
|
||||
disp.SetSender(stream)
|
||||
disp.Announce(ctx)
|
||||
|
||||
// Drop the sender before we return so no one sends into a dead stream.
|
||||
defer disp.SetSender(nil)
|
||||
|
||||
recvDone := make(chan struct{})
|
||||
recvErr := make(chan error, 1)
|
||||
go func() {
|
||||
defer close(recvDone)
|
||||
for {
|
||||
env, err := stream.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
logger.Info("controller closed stream")
|
||||
recvErr <- nil
|
||||
} else if ctx.Err() == nil {
|
||||
recvErr <- err
|
||||
} else {
|
||||
recvErr <- nil
|
||||
}
|
||||
return
|
||||
}
|
||||
// Dispatch each envelope in its OWN goroutine so a slow/blocked
|
||||
// handler can't freeze the recv loop (and thus the whole agent).
|
||||
// This was the Valheim+stdio hang: an RCON write to a wedged stdin
|
||||
// pipe blocked here forever, so the subsequent stop never got read.
|
||||
// Safe to parallelize: handlers guard shared state with d.mu, and the
|
||||
// controller serializes dependent ops (create→start) via its own
|
||||
// correlation-id awaits, so it never sends a dependent op before the
|
||||
// prior reply arrives.
|
||||
go disp.Handle(ctx, env)
|
||||
}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(hb)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = stream.CloseSend()
|
||||
<-recvDone
|
||||
return nil
|
||||
case <-recvDone:
|
||||
return <-recvErr
|
||||
case t := <-ticker.C:
|
||||
// Heartbeats go through the dispatcher's send queue (droppable)
|
||||
// so sendLoop stays the sole owner of stream.Send — sending here
|
||||
// directly raced the drainer goroutine on the same gRPC stream.
|
||||
// A dead stream is detected by the recv goroutine (stream.Recv
|
||||
// errors → recvDone), not by a failed heartbeat send.
|
||||
disp.SendHeartbeat(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// jitter returns up to ±25% of d as a random offset, so N agents don't
|
||||
// synchronize their reconnects against a restarting controller.
|
||||
func jitter(d time.Duration) time.Duration {
|
||||
if d <= 0 {
|
||||
return 0
|
||||
}
|
||||
half := int64(d / 4)
|
||||
return time.Duration(rand.Int64N(2*half+1) - half)
|
||||
}
|
||||
|
||||
// summarizeModules reduces the agent's loaded manifests to the small
|
||||
// ModuleSummary record the controller caches for UI picking.
|
||||
func summarizeModules(disp *dispatch.Dispatcher) []*panelv1.ModuleSummary {
|
||||
mods := disp.Modules().List()
|
||||
out := make([]*panelv1.ModuleSummary, 0, len(mods))
|
||||
for _, m := range mods {
|
||||
providers := make([]*panelv1.ModuleUpdateProviderSummary, 0, len(m.UpdateProviders))
|
||||
for _, p := range m.UpdateProviders {
|
||||
providers = append(providers, &panelv1.ModuleUpdateProviderSummary{
|
||||
Id: p.ID,
|
||||
Kind: p.Kind,
|
||||
RequiresSteamLogin: p.RequiresSteamLogin,
|
||||
})
|
||||
}
|
||||
out = append(out, &panelv1.ModuleSummary{
|
||||
Id: m.ID,
|
||||
Name: m.Name,
|
||||
Version: m.Version,
|
||||
SupportedModes: m.SupportedModes,
|
||||
UpdateProviders: providers,
|
||||
HasRcon: m.RCON != nil,
|
||||
Authors: m.Authors,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Pairing flow (agent side):
|
||||
//
|
||||
// 1. Fetch the controller's CA cert over HTTPS (with InsecureSkipVerify —
|
||||
// this is the only moment we're exposed; the one-time token gates
|
||||
// the actual cert issuance below).
|
||||
// 2. Generate an RSA-2048 keypair locally. The private key never
|
||||
// leaves this host.
|
||||
// 3. Build a CSR with CN = agent-id.
|
||||
// 4. POST {token, agent_id, csr_pem} to /api/pair/issue, now verifying
|
||||
// the server cert against the CA we just fetched.
|
||||
// 5. Persist ca.crt + agent.crt + agent.key to --cert-dir, mode 0600 on the key.
|
||||
//
|
||||
// After pairing, run the agent normally; it auto-detects the cert files
|
||||
// and connects to the controller with mTLS.
|
||||
|
||||
type pairConfig struct {
|
||||
AgentID string
|
||||
PairURL string // e.g. https://controller:8080 — the HTTP dashboard port
|
||||
Token string
|
||||
CertDir string
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
func runPair(cfg pairConfig) error {
|
||||
cfg.Logger.Info("pair: starting", "pair_url", cfg.PairURL, "agent_id", cfg.AgentID, "cert_dir", cfg.CertDir)
|
||||
|
||||
if err := os.MkdirAll(cfg.CertDir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir cert-dir: %w", err)
|
||||
}
|
||||
|
||||
// Step 1 — fetch CA with InsecureSkipVerify, then verify with it.
|
||||
caPEM, err := fetchCA(cfg.PairURL, cfg.Logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch CA: %w", err)
|
||||
}
|
||||
caPath := filepath.Join(cfg.CertDir, "ca.crt")
|
||||
if err := os.WriteFile(caPath, caPEM, 0o644); err != nil {
|
||||
return fmt.Errorf("write ca.crt: %w", err)
|
||||
}
|
||||
cfg.Logger.Info("pair: CA saved", "path", caPath, "bytes", len(caPEM))
|
||||
|
||||
// Step 2 — keypair + CSR
|
||||
priv, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gen key: %w", err)
|
||||
}
|
||||
tpl := &x509.CertificateRequest{
|
||||
Subject: pkix.Name{CommonName: cfg.AgentID, Organization: []string{"panel-agent"}},
|
||||
DNSNames: []string{cfg.AgentID},
|
||||
}
|
||||
csrDER, err := x509.CreateCertificateRequest(rand.Reader, tpl, priv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create CSR: %w", err)
|
||||
}
|
||||
csrPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrDER})
|
||||
|
||||
// Step 3 — POST to /api/pair/issue, verifying CA.
|
||||
certPEM, err := issueCert(cfg.PairURL, caPEM, cfg.Token, cfg.AgentID, csrPEM, cfg.Logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("issue cert: %w", err)
|
||||
}
|
||||
|
||||
// Step 4 — persist.
|
||||
if err := os.WriteFile(filepath.Join(cfg.CertDir, "agent.crt"), certPEM, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
|
||||
if err := os.WriteFile(filepath.Join(cfg.CertDir, "agent.key"), keyPEM, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Logger.Info("pair: cert issued", "cert_dir", cfg.CertDir, "agent_id", cfg.AgentID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchCA(baseURL string, log *slog.Logger) ([]byte, error) {
|
||||
trimmed := strings.TrimSuffix(baseURL, "/")
|
||||
url := trimmed + "/api/pair/ca"
|
||||
log.Info("pair: fetching CA", "url", url)
|
||||
|
||||
// Until we have the CA we can't verify the server. InsecureSkipVerify
|
||||
// is narrowly scoped to this one request — the cert-issuance POST
|
||||
// below verifies against the CA we just got.
|
||||
cli := &http.Client{
|
||||
Timeout: 15 * time.Second,
|
||||
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
|
||||
}
|
||||
resp, err := cli.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return nil, fmt.Errorf("http %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
block, _ := pem.Decode(data)
|
||||
if block == nil || block.Type != "CERTIFICATE" {
|
||||
return nil, fmt.Errorf("response isn't a CERTIFICATE PEM")
|
||||
}
|
||||
if _, err := x509.ParseCertificate(block.Bytes); err != nil {
|
||||
return nil, fmt.Errorf("parse CA cert: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func issueCert(baseURL string, caPEM []byte, token, agentID string, csrPEM []byte, log *slog.Logger) ([]byte, error) {
|
||||
trimmed := strings.TrimSuffix(baseURL, "/")
|
||||
url := trimmed + "/api/pair/issue"
|
||||
log.Info("pair: requesting cert", "url", url, "agent_id", agentID)
|
||||
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("append CA to pool failed")
|
||||
}
|
||||
cli := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}},
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"token": token,
|
||||
"agent_id": agentID,
|
||||
"csr_pem": string(csrPEM),
|
||||
})
|
||||
req, _ := http.NewRequest("POST", url, bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
errBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
return nil, fmt.Errorf("http %d: %s", resp.StatusCode, string(errBody))
|
||||
}
|
||||
var out struct {
|
||||
CertPEM string `json:"cert_pem"`
|
||||
CAPEM string `json:"ca_pem"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out.CertPEM == "" {
|
||||
return nil, fmt.Errorf("response missing cert_pem")
|
||||
}
|
||||
return []byte(out.CertPEM), nil
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
)
|
||||
|
||||
// region-medic autoheal — the auto-on-reboot flow, designed to run (in a sidecar)
|
||||
// while the 7DTD server is STOPPED, just before the agent starts it:
|
||||
//
|
||||
// 1. Validate EVERY .7rg in the active world's Region dir (catches whole-file
|
||||
// corruption like a bad magic header, which has no error_backups and so is
|
||||
// invisible to the error_backup-cluster scan).
|
||||
// 2. Heal each corrupt region from the newest source that has a validated-clean
|
||||
// copy: the rolling folder snapshots first (fast, local), then the panel's
|
||||
// tar backups as a fallback. Corrupt originals are quarantined first.
|
||||
// 3. Snapshot the now-clean Region dir to a fresh rolling slot (real folder
|
||||
// copy — instant restore, no unzip) and rotate to keep the newest -keep.
|
||||
//
|
||||
// Emits a JSON report the agent relays to the controller for the Discord ping.
|
||||
//
|
||||
// region-medic autoheal -region-dir D -rolling-dir R [-backups-dir B] [-keep 2] [-apply]
|
||||
//
|
||||
// Without -apply it only validates + reports (dry run: no heal, no snapshot).
|
||||
|
||||
type ahHeal struct {
|
||||
Region string `json:"region"` // r.X.Z
|
||||
Source string `json:"source"` // "rolling:<slot>" | "backup:<id>"
|
||||
SourceTime string `json:"source_time"` // RFC3339 of the snapshot/backup
|
||||
}
|
||||
|
||||
type ahReport struct {
|
||||
RegionDir string `json:"region_dir"`
|
||||
TotalRegions int `json:"total_regions"`
|
||||
CorruptFound int `json:"corrupt_found"`
|
||||
Healed []ahHeal `json:"healed"`
|
||||
Unhealable []string `json:"unhealable"`
|
||||
SnapshotSlot string `json:"snapshot_slot"`
|
||||
RollingKept int `json:"rolling_kept"`
|
||||
Applied bool `json:"applied"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
}
|
||||
|
||||
func cmdAutoheal(args []string) error {
|
||||
var regionDir, rollingDir, backupsDir, savesRoot, discordChannel, discordTokenFile, label string
|
||||
keep := 2
|
||||
apply := false
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "-region-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
regionDir = args[i]
|
||||
}
|
||||
case "-saves-root":
|
||||
i++
|
||||
if i < len(args) {
|
||||
savesRoot = args[i]
|
||||
}
|
||||
case "-rolling-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
rollingDir = args[i]
|
||||
}
|
||||
case "-backups-dir":
|
||||
i++
|
||||
if i < len(args) {
|
||||
backupsDir = args[i]
|
||||
}
|
||||
case "-keep":
|
||||
i++
|
||||
if i < len(args) {
|
||||
keep, _ = strconv.Atoi(args[i])
|
||||
}
|
||||
case "-discord-channel":
|
||||
i++
|
||||
if i < len(args) {
|
||||
discordChannel = args[i]
|
||||
}
|
||||
case "-discord-token-file":
|
||||
i++
|
||||
if i < len(args) {
|
||||
discordTokenFile = args[i]
|
||||
}
|
||||
case "-label":
|
||||
i++
|
||||
if i < len(args) {
|
||||
label = args[i]
|
||||
}
|
||||
case "-apply", "--apply":
|
||||
apply = true
|
||||
default:
|
||||
return fmt.Errorf("unknown flag %q", args[i])
|
||||
}
|
||||
}
|
||||
// -saves-root auto-resolves the active world's Region dir (so the agent
|
||||
// doesn't have to know GameWorld/GameName, and RWG seed-named worlds work).
|
||||
// Per-server config lives in the saves volume (panel writes it via FsWrite,
|
||||
// no recreate): world pin + keep + Discord channel; enabled:false skips.
|
||||
var cfgWorld string
|
||||
if savesRoot != "" {
|
||||
if c, ok := readSavesMedicConfig(savesRoot); ok {
|
||||
if c.Enabled != nil && !*c.Enabled {
|
||||
fmt.Fprintln(os.Stderr, "autoheal: disabled via region-medic.json, skipping")
|
||||
return nil
|
||||
}
|
||||
if c.Keep > 0 {
|
||||
keep = c.Keep
|
||||
}
|
||||
if c.DiscordChannel != "" {
|
||||
discordChannel = c.DiscordChannel
|
||||
}
|
||||
cfgWorld = strings.TrimSpace(c.World)
|
||||
}
|
||||
}
|
||||
// Resolve the Region dir. An explicit world pin wins over auto-detect — some
|
||||
// servers carry multiple saves, so the operator picks which one to heal. If
|
||||
// the pinned world has no Region dir we SKIP rather than silently auto-pick a
|
||||
// different world the operator didn't choose.
|
||||
if regionDir == "" && savesRoot != "" {
|
||||
if cfgWorld != "" {
|
||||
wd := filepath.Join(savesRoot, ".local/share/7DaysToDie/Saves", cfgWorld, "Region")
|
||||
if st, err := os.Stat(wd); err != nil || !st.IsDir() {
|
||||
fmt.Fprintf(os.Stderr, "autoheal: pinned world %q has no Region dir at %s — skipping\n", cfgWorld, wd)
|
||||
return nil
|
||||
}
|
||||
regionDir = wd
|
||||
} else {
|
||||
rd, rerr := resolveRegionDir(savesRoot)
|
||||
if rerr != nil {
|
||||
return rerr
|
||||
}
|
||||
regionDir = rd
|
||||
}
|
||||
}
|
||||
if regionDir == "" || rollingDir == "" {
|
||||
return fmt.Errorf("required: -region-dir (or -saves-root) and -rolling-dir (and -backups-dir for tar fallback)")
|
||||
}
|
||||
if keep < 1 {
|
||||
keep = 1
|
||||
}
|
||||
start := time.Now()
|
||||
rep := ahReport{RegionDir: regionDir, RollingKept: keep, Applied: apply, Healed: []ahHeal{}, Unhealable: []string{}}
|
||||
|
||||
// 1. Validate every .7rg.
|
||||
ents, err := os.ReadDir(regionDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read region dir: %w", err)
|
||||
}
|
||||
var corrupt []regionmedic.RegionCoord
|
||||
for _, e := range ents {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
rc, ok := regionmedic.ParseRegionFileName(e.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rep.TotalRegions++
|
||||
if !regionmedic.ValidateRegionFile(filepath.Join(regionDir, e.Name()), false).Healthy() {
|
||||
corrupt = append(corrupt, rc)
|
||||
}
|
||||
}
|
||||
rep.CorruptFound = len(corrupt)
|
||||
|
||||
// 2. Heal (only with -apply).
|
||||
quarantineDir := filepath.Join(rollingDir, "_quarantine")
|
||||
if apply && len(corrupt) > 0 {
|
||||
slots := listRollingSlots(rollingDir) // newest first
|
||||
var panelBackups []regionmedic.BackupRef
|
||||
if backupsDir != "" {
|
||||
panelBackups, _ = regionmedic.ListPanelBackups(backupsDir)
|
||||
}
|
||||
stamp := start.UTC().Format("20060102-150405")
|
||||
for _, rc := range corrupt {
|
||||
fn := rc.FileName()
|
||||
done := false
|
||||
// 2a. newest rolling snapshot with a clean copy.
|
||||
for _, s := range slots {
|
||||
cand := filepath.Join(s.path, fn)
|
||||
if fi, e := os.Stat(cand); e == nil && !fi.IsDir() &&
|
||||
regionmedic.ValidateRegionFile(cand, false).Healthy() {
|
||||
if e := healFromFile(regionDir, fn, cand, quarantineDir, stamp); e == nil {
|
||||
rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "rolling:" + filepath.Base(s.path), SourceTime: s.t.UTC().Format(time.RFC3339)})
|
||||
done = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2b. panel tar backups fallback.
|
||||
if !done && len(panelBackups) > 0 {
|
||||
entry := regionmedic.SaveRelEntry(regionDir, fn)
|
||||
best, _, _ := regionmedic.FindLastGoodRegion(panelBackups, entry, false, time.Time{})
|
||||
if best.Found && best.Clean() {
|
||||
if e := healFromBytes(regionDir, fn, best.Bytes, quarantineDir, stamp); e == nil {
|
||||
rep.Healed = append(rep.Healed, ahHeal{Region: rc.String(), Source: "backup:" + best.Backup.ID, SourceTime: best.Backup.CreatedAt.UTC().Format(time.RFC3339)})
|
||||
done = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !done {
|
||||
rep.Unhealable = append(rep.Unhealable, rc.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Snapshot the now-clean Region dir to a fresh rolling slot + rotate.
|
||||
if apply {
|
||||
slotName := fmt.Sprintf("slot-%d", start.Unix())
|
||||
if err := snapshotRegions(regionDir, filepath.Join(rollingDir, slotName)); err != nil {
|
||||
rep.SnapshotSlot = "ERROR: " + err.Error()
|
||||
} else {
|
||||
rep.SnapshotSlot = slotName
|
||||
rotateRollingSlots(rollingDir, keep)
|
||||
}
|
||||
}
|
||||
|
||||
rep.DurationMs = time.Since(start).Milliseconds()
|
||||
b, _ := json.MarshalIndent(rep, "", " ")
|
||||
fmt.Println(string(b))
|
||||
fmt.Fprintf(os.Stderr, "autoheal: %d regions, %d corrupt, %d healed, %d unhealable, snapshot=%s (%dms)\n",
|
||||
rep.TotalRegions, rep.CorruptFound, len(rep.Healed), len(rep.Unhealable), rep.SnapshotSlot, rep.DurationMs)
|
||||
// Discord alert — only when we actually healed something on this reboot.
|
||||
if apply {
|
||||
if derr := postDiscordHealReport(discordTokenFile, discordChannel, label, rep); derr != nil {
|
||||
fmt.Fprintf(os.Stderr, "discord post failed: %v\n", derr)
|
||||
} else if discordChannel != "" && len(rep.Healed) > 0 {
|
||||
fmt.Fprintf(os.Stderr, "discord: posted heal embed to channel %s\n", discordChannel)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type rollingSlot struct {
|
||||
path string
|
||||
t time.Time
|
||||
}
|
||||
|
||||
// listRollingSlots returns slot dirs named "slot-<unix>" newest first.
|
||||
func listRollingSlots(dir string) []rollingSlot {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []rollingSlot
|
||||
for _, e := range ents {
|
||||
if !e.IsDir() || !strings.HasPrefix(e.Name(), "slot-") {
|
||||
continue
|
||||
}
|
||||
secs, err := strconv.ParseInt(strings.TrimPrefix(e.Name(), "slot-"), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, rollingSlot{path: filepath.Join(dir, e.Name()), t: time.Unix(secs, 0)})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].t.After(out[j].t) })
|
||||
return out
|
||||
}
|
||||
|
||||
// rotateRollingSlots keeps the newest `keep` slot dirs, removing older ones.
|
||||
func rotateRollingSlots(dir string, keep int) {
|
||||
for i, s := range listRollingSlots(dir) {
|
||||
if i >= keep {
|
||||
_ = os.RemoveAll(s.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// snapshotRegions copies every *.7rg from regionDir into slotPath.
|
||||
func snapshotRegions(regionDir, slotPath string) error {
|
||||
if err := os.MkdirAll(slotPath, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
ents, err := os.ReadDir(regionDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range ents {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".7rg") {
|
||||
continue
|
||||
}
|
||||
if err := copyFile(filepath.Join(regionDir, e.Name()), filepath.Join(slotPath, e.Name())); err != nil {
|
||||
return fmt.Errorf("copy %s: %w", e.Name(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// healFromFile quarantines the corrupt original then overwrites the live region
|
||||
// file in place with a validated-clean source file.
|
||||
func healFromFile(regionDir, fn, srcPath, quarantineDir, stamp string) error {
|
||||
_ = os.MkdirAll(quarantineDir, 0o755)
|
||||
live := filepath.Join(regionDir, fn)
|
||||
_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
|
||||
return copyFileInPlace(srcPath, live)
|
||||
}
|
||||
|
||||
// healFromBytes is the same but the clean source is in-memory (a tar backup).
|
||||
func healFromBytes(regionDir, fn string, data []byte, quarantineDir, stamp string) error {
|
||||
_ = os.MkdirAll(quarantineDir, 0o755)
|
||||
live := filepath.Join(regionDir, fn)
|
||||
_ = copyFile(live, filepath.Join(quarantineDir, fn+".corrupt-"+stamp))
|
||||
tmp := live + ".medic-new"
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, live)
|
||||
}
|
||||
|
||||
// copyFile copies src→dst atomically (temp+rename), same filesystem assumed for
|
||||
// the rename target dir. Used for snapshots + quarantine (rolling dir).
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
tmp := dst + ".tmp-copy"
|
||||
out, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
// copyFileInPlace overwrites dst with src via a temp file in dst's own dir, so
|
||||
// the rename is same-filesystem even when src lives elsewhere (a rolling slot).
|
||||
func copyFileInPlace(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
tmp := dst + ".medic-new"
|
||||
out, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
_ "time/tzdata" // embed the IANA tz database so America/Los_Angeles resolves
|
||||
// even in debian:12-slim, which ships no /usr/share/zoneinfo.
|
||||
)
|
||||
|
||||
// resolveRegionDir finds the active world's Region dir under a 7DTD saves root
|
||||
// (the dir mounted as the saves volume — holds serverconfig.xml + .local/...).
|
||||
// Prefers serverconfig GameWorld/GameName when that world has real data
|
||||
// (main.ttw); else falls back to the most-recently-written Saves/*/<GameName>
|
||||
// with data — handles RWG seed-named worlds.
|
||||
func resolveRegionDir(savesRoot string) (string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(savesRoot, "serverconfig.xml"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read serverconfig.xml: %w", err)
|
||||
}
|
||||
gw := xmlProp(string(raw), "GameWorld")
|
||||
gn := xmlProp(string(raw), "GameName")
|
||||
base := filepath.Join(savesRoot, ".local", "share", "7DaysToDie", "Saves")
|
||||
hasWorld := func(d string) bool {
|
||||
if _, e := os.Stat(filepath.Join(d, "main.ttw")); e == nil {
|
||||
return true
|
||||
}
|
||||
fi, e := os.Stat(filepath.Join(d, "Region"))
|
||||
return e == nil && fi.IsDir()
|
||||
}
|
||||
if gw != "" && gn != "" {
|
||||
lit := filepath.Join(base, gw, gn)
|
||||
if hasWorld(lit) {
|
||||
return filepath.Join(lit, "Region"), nil
|
||||
}
|
||||
}
|
||||
if gn != "" { // RWG fallback: newest world dir with this GameName.
|
||||
worlds, _ := os.ReadDir(base)
|
||||
var newest string
|
||||
var newestT time.Time
|
||||
for _, w := range worlds {
|
||||
if !w.IsDir() {
|
||||
continue
|
||||
}
|
||||
cand := filepath.Join(base, w.Name(), gn)
|
||||
if !hasWorld(cand) {
|
||||
continue
|
||||
}
|
||||
if fi, e := os.Stat(cand); e == nil && fi.ModTime().After(newestT) {
|
||||
newestT, newest = fi.ModTime(), cand
|
||||
}
|
||||
}
|
||||
if newest != "" {
|
||||
return filepath.Join(newest, "Region"), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("could not resolve active Region dir (GameWorld=%q GameName=%q)", gw, gn)
|
||||
}
|
||||
|
||||
// xmlProp pulls the value from <property name="X" value="Y" /> (tolerant of
|
||||
// attribute order + quote style; skips comments).
|
||||
func xmlProp(content, name string) string {
|
||||
for _, raw := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "<!--") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(line, `name="`+name+`"`) {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(line, "value=")
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
rest := line[i+len("value="):]
|
||||
if len(rest) < 1 {
|
||||
continue
|
||||
}
|
||||
q := rest[0]
|
||||
rest = rest[1:]
|
||||
if end := strings.IndexByte(rest, q); end >= 0 {
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// postDiscordHealReport posts a rich embed heal summary to a Discord channel via
|
||||
// a bot token read from tokenFile. No-op when token/channel missing or nothing
|
||||
// healed. All times are rendered in Pacific (America/Los_Angeles → PST/PDT).
|
||||
func postDiscordHealReport(tokenFile, channel, label string, rep ahReport) error {
|
||||
if tokenFile == "" || channel == "" || len(rep.Healed) == 0 {
|
||||
return nil
|
||||
}
|
||||
tb, err := os.ReadFile(tokenFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read discord token: %w", err)
|
||||
}
|
||||
token := strings.TrimSpace(string(tb))
|
||||
if token == "" {
|
||||
return nil
|
||||
}
|
||||
if label == "" {
|
||||
label = "7DTD server"
|
||||
}
|
||||
loc, lerr := time.LoadLocation("America/Los_Angeles")
|
||||
if lerr != nil || loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
pst := func(rfc string) string {
|
||||
if t, e := time.Parse(time.RFC3339, rfc); e == nil {
|
||||
return t.In(loc).Format("Jan 2, 3:04 PM MST")
|
||||
}
|
||||
return rfc
|
||||
}
|
||||
srcLabel := func(s string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(s, "rolling:"):
|
||||
return "rolling snapshot"
|
||||
case strings.HasPrefix(s, "backup:"):
|
||||
return "panel backup"
|
||||
default:
|
||||
return s
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
for _, h := range rep.Healed {
|
||||
lines = append(lines, fmt.Sprintf("• `%s` ← %s _(saved %s)_", h.Region, srcLabel(h.Source), pst(h.SourceTime)))
|
||||
}
|
||||
color := 0x57F287 // green — everything healed
|
||||
fields := []map[string]any{{
|
||||
"name": fmt.Sprintf("✅ Healed %d region(s)", len(rep.Healed)),
|
||||
"value": strings.Join(lines, "\n"),
|
||||
"inline": false,
|
||||
}}
|
||||
if len(rep.Unhealable) > 0 {
|
||||
color = 0xED4245 // red — some couldn't be healed
|
||||
fields = append(fields, map[string]any{
|
||||
"name": fmt.Sprintf("⚠️ Could NOT heal %d (no clean copy anywhere)", len(rep.Unhealable)),
|
||||
"value": "`" + strings.Join(rep.Unhealable, "`, `") + "`",
|
||||
"inline": false,
|
||||
})
|
||||
}
|
||||
nowPST := time.Now().In(loc).Format("Mon Jan 2, 3:04 PM MST")
|
||||
embed := map[string]any{
|
||||
"title": "🩺 Region Medic — " + label,
|
||||
"description": fmt.Sprintf("Detected and repaired **%d** corrupt region file(s) on reboot, before the world loaded.", rep.CorruptFound),
|
||||
"color": color,
|
||||
"fields": fields,
|
||||
"footer": map[string]any{"text": "Region Medic · " + nowPST},
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"embeds": []any{embed}})
|
||||
req, err := http.NewRequest(http.MethodPost, "https://discord.com/api/v10/channels/"+channel+"/messages", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bot "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "RefugeBot-RegionMedic (https://refugegaming.org, 1.0)")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("discord HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// savesMedicConfig is the per-server medic config the panel writes (via FsWrite,
|
||||
// no container recreate) into the saves volume at region-medic.json. Enabled is a
|
||||
// pointer so an absent field defaults to ON.
|
||||
type savesMedicConfig struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
Keep int `json:"keep"`
|
||||
World string `json:"world"` // "<World>/<GameName>"; empty = auto-detect active world
|
||||
DiscordChannel string `json:"discord_channel"`
|
||||
}
|
||||
|
||||
// readSavesMedicConfig reads <savesRoot>/region-medic.json; ok=false if absent or
|
||||
// unparseable (the caller then keeps its flag defaults).
|
||||
func readSavesMedicConfig(savesRoot string) (savesMedicConfig, bool) {
|
||||
b, err := os.ReadFile(filepath.Join(savesRoot, "region-medic.json"))
|
||||
if err != nil {
|
||||
return savesMedicConfig{}, false
|
||||
}
|
||||
var c savesMedicConfig
|
||||
if json.Unmarshal(b, &c) != nil {
|
||||
return savesMedicConfig{}, false
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
// Command region-medic is a standalone operator tool that detects and heals
|
||||
// corrupted 7DTD ".7rg" region files from the panel's own backups, using the
|
||||
// shared github.com/dbledeez/panel/pkg/regionmedic engine.
|
||||
//
|
||||
// It is the same logic that ships inside the agent's RegionScan/RegionHeal RPCs,
|
||||
// exposed as a CLI so it can be run by hand against a stopped instance without
|
||||
// redeploying the controller. Subcommands:
|
||||
//
|
||||
// region-medic validate <file.7rg> [-deep]
|
||||
// region-medic scan <region-dir>
|
||||
// region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
|
||||
// region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply
|
||||
// [-deep] [-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
|
||||
// [-controller URL -instance ID] # stop before / start after (panel-safe)
|
||||
// [-container NAME] # post-start: watch for ready + new corruption
|
||||
// region-medic verify <region-dir> # count error_backups now
|
||||
//
|
||||
// stop/start go through the controller HTTP API (never raw docker), so panel
|
||||
// state stays correct; readiness/state polling reads docker locally.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
var err error
|
||||
switch os.Args[1] {
|
||||
case "validate":
|
||||
err = cmdValidate(os.Args[2:])
|
||||
case "scan":
|
||||
err = cmdScan(os.Args[2:])
|
||||
case "plan":
|
||||
err = cmdHeal(os.Args[2:], false)
|
||||
case "heal":
|
||||
err = cmdHeal(os.Args[2:], true)
|
||||
case "verify":
|
||||
err = cmdVerify(os.Args[2:])
|
||||
case "autoheal":
|
||||
err = cmdAutoheal(os.Args[2:])
|
||||
case "-h", "--help", "help":
|
||||
usage()
|
||||
return
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1])
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `region-medic — detect & heal corrupt 7DTD .7rg regions from panel backups
|
||||
|
||||
region-medic validate <file.7rg> [-deep]
|
||||
region-medic scan <region-dir>
|
||||
region-medic plan -region-dir D -region r.X.Z -backups-dir B [-deep] [-not-after RFC3339]
|
||||
region-medic heal -region-dir D -region r.X.Z -backups-dir B -apply [-deep]
|
||||
[-not-after RFC3339] -snapshot-dir S -quarantine-dir Q
|
||||
[-controller URL -instance ID] [-container NAME]
|
||||
region-medic verify <region-dir>
|
||||
`)
|
||||
}
|
||||
|
||||
func printJSON(v any) {
|
||||
b, _ := json.MarshalIndent(v, "", " ")
|
||||
fmt.Println(string(b))
|
||||
}
|
||||
|
||||
// --- validate ---------------------------------------------------------------
|
||||
|
||||
func cmdValidate(args []string) error {
|
||||
deep := false
|
||||
var file string
|
||||
for _, a := range args {
|
||||
if a == "-deep" || a == "--deep" {
|
||||
deep = true
|
||||
} else if !strings.HasPrefix(a, "-") {
|
||||
file = a
|
||||
}
|
||||
}
|
||||
if file == "" {
|
||||
return fmt.Errorf("usage: region-medic validate <file.7rg> [-deep]")
|
||||
}
|
||||
rep := regionmedic.ValidateRegionFile(file, deep)
|
||||
printJSON(rep)
|
||||
if rep.Healthy() {
|
||||
fmt.Fprintf(os.Stderr, "HEALTHY: %d present chunks, %d sectors\n", rep.PresentChunks, rep.TotalSectors)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "UNHEALTHY: err=%q badChunks=%d\n", rep.Err, len(rep.BadChunks))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- scan -------------------------------------------------------------------
|
||||
|
||||
func cmdScan(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("usage: region-medic scan <region-dir>")
|
||||
}
|
||||
scan, err := regionmedic.ScanRegionDir(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printJSON(scan)
|
||||
fmt.Fprintf(os.Stderr, "%d error_backups across %d affected region(s)\n", scan.ErrorBackups, len(scan.Affected))
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- verify -----------------------------------------------------------------
|
||||
|
||||
func cmdVerify(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return fmt.Errorf("usage: region-medic verify <region-dir>")
|
||||
}
|
||||
scan, err := regionmedic.ScanRegionDir(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "error_backups now: %d (affected regions: %d)\n", scan.ErrorBackups, len(scan.Affected))
|
||||
printJSON(scan.Affected)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- plan / heal ------------------------------------------------------------
|
||||
|
||||
type healFlags struct {
|
||||
regionDir, region, backupsDir string
|
||||
snapshotDir, quarantineDir string
|
||||
deep, apply bool
|
||||
notAfter string
|
||||
controller, instance, container string
|
||||
}
|
||||
|
||||
func parseHealFlags(args []string) (healFlags, error) {
|
||||
f := healFlags{}
|
||||
for i := 0; i < len(args); i++ {
|
||||
a := args[i]
|
||||
next := func() string {
|
||||
if i+1 < len(args) {
|
||||
i++
|
||||
return args[i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
switch a {
|
||||
case "-region-dir":
|
||||
f.regionDir = next()
|
||||
case "-region":
|
||||
f.region = next()
|
||||
case "-backups-dir":
|
||||
f.backupsDir = next()
|
||||
case "-snapshot-dir":
|
||||
f.snapshotDir = next()
|
||||
case "-quarantine-dir":
|
||||
f.quarantineDir = next()
|
||||
case "-not-after":
|
||||
f.notAfter = next()
|
||||
case "-controller":
|
||||
f.controller = next()
|
||||
case "-instance":
|
||||
f.instance = next()
|
||||
case "-container":
|
||||
f.container = next()
|
||||
case "-deep", "--deep":
|
||||
f.deep = true
|
||||
case "-apply", "--apply":
|
||||
f.apply = true
|
||||
default:
|
||||
return f, fmt.Errorf("unknown flag %q", a)
|
||||
}
|
||||
}
|
||||
if f.regionDir == "" || f.region == "" || f.backupsDir == "" {
|
||||
return f, fmt.Errorf("required: -region-dir, -region, -backups-dir")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func cmdHeal(args []string, allowApply bool) error {
|
||||
f, err := parseHealFlags(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
region, ok := regionmedic.ParseRegionFileName(regionArg(f.region))
|
||||
if !ok {
|
||||
return fmt.Errorf("bad -region %q (want r.X.Z or r.X.Z.7rg)", f.region)
|
||||
}
|
||||
var notAfter time.Time
|
||||
if f.notAfter != "" {
|
||||
notAfter, err = time.Parse(time.RFC3339, f.notAfter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bad -not-after: %w", err)
|
||||
}
|
||||
}
|
||||
backups, err := regionmedic.ListPanelBackups(f.backupsDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list backups: %w", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "found %d backups in %s\n", len(backups), f.backupsDir)
|
||||
|
||||
plan, err := regionmedic.PlanHeal(f.regionDir, region, backups, f.deep, notAfter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summarizePlan(plan)
|
||||
|
||||
if !allowApply || !f.apply {
|
||||
fmt.Fprintln(os.Stderr, "(dry run — re-run `heal ... -apply` to execute)")
|
||||
return nil
|
||||
}
|
||||
if !plan.Ready() {
|
||||
return fmt.Errorf("refusing to heal: no clean backup copy found for %s", region)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// Orchestrate stop (panel-safe) if controller configured.
|
||||
if f.controller != "" && f.instance != "" {
|
||||
fmt.Fprintf(os.Stderr, "stopping %s via panel...\n", f.instance)
|
||||
if err := panelAction(ctx, f.controller, f.instance, "stop"); err != nil {
|
||||
return fmt.Errorf("panel stop: %w", err)
|
||||
}
|
||||
if f.container != "" {
|
||||
if err := waitContainer(ctx, f.container, false, 90*time.Second); err != nil {
|
||||
return fmt.Errorf("wait stopped: %w", err)
|
||||
}
|
||||
} else {
|
||||
time.Sleep(8 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
snapDir := orDefault(f.snapshotDir, f.regionDir+"/_medic_snapshots")
|
||||
qDir := orDefault(f.quarantineDir, f.regionDir+"/_medic_quarantine")
|
||||
res, err := regionmedic.ApplyHeal(plan, snapDir, qDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("apply heal: %w", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "HEALED %s: wrote %d bytes from %s; snapshot=%s; quarantined=%d\n",
|
||||
res.Region, res.BytesWritten, plan.Source.Backup.ID, res.SnapshotPath, len(res.QuarantinedFiles))
|
||||
printJSON(res)
|
||||
|
||||
// Start back up (panel-safe) + verify.
|
||||
if f.controller != "" && f.instance != "" {
|
||||
fmt.Fprintf(os.Stderr, "starting %s via panel...\n", f.instance)
|
||||
if err := panelAction(ctx, f.controller, f.instance, "start"); err != nil {
|
||||
return fmt.Errorf("panel start: %w", err)
|
||||
}
|
||||
if f.container != "" {
|
||||
fmt.Fprintln(os.Stderr, "watching boot for readiness + new corruption...")
|
||||
if err := watchBoot(ctx, f.container, f.regionDir, region, 240*time.Second); err != nil {
|
||||
return fmt.Errorf("post-heal verify: %w", err)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "VERIFY OK: instance ready, no new Wrong chunk header for the healed region")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func summarizePlan(plan regionmedic.HealPlan) {
|
||||
fmt.Fprintf(os.Stderr, "region %s: live healthy=%v (badChunks=%d, err=%q)\n",
|
||||
plan.Region, plan.LiveReport.Healthy(), len(plan.LiveReport.BadChunks), plan.LiveReport.Err)
|
||||
fmt.Fprintf(os.Stderr, "error_backups for region: %d\n", len(plan.ErrorBackups))
|
||||
for _, c := range plan.Evaluated {
|
||||
status := "no region in backup"
|
||||
if c.Found {
|
||||
if c.Report.Healthy() {
|
||||
status = fmt.Sprintf("CLEAN (%d chunks)", c.Report.PresentChunks)
|
||||
} else {
|
||||
status = fmt.Sprintf("corrupt (%d bad chunks, err=%q)", len(c.Report.BadChunks), c.Report.Err)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " %s %s %s\n", c.Backup.CreatedAt.Format(time.RFC3339), c.Backup.ID, status)
|
||||
}
|
||||
if plan.Ready() {
|
||||
fmt.Fprintf(os.Stderr, "=> heal source: %s (%s)\n", plan.Source.Backup.ID, plan.Source.Backup.CreatedAt.Format(time.RFC3339))
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "=> NO clean source found")
|
||||
}
|
||||
}
|
||||
|
||||
// --- orchestration helpers --------------------------------------------------
|
||||
|
||||
func panelAction(ctx context.Context, controller, instance, action string) error {
|
||||
url := strings.TrimRight(controller, "/") + "/api/instances/" + instance + "/" + action
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, bytes.TrimSpace(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitContainer polls `docker inspect` until the container's Running state
|
||||
// matches want, or timeout.
|
||||
func waitContainer(ctx context.Context, name string, want bool, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
out, _ := exec.CommandContext(ctx, "docker", "inspect", "-f", "{{.State.Running}}", name).Output()
|
||||
running := strings.TrimSpace(string(out)) == "true"
|
||||
if running == want {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for %s running=%v", name, want)
|
||||
}
|
||||
|
||||
// watchBoot tails the container log until the 7DTD ready marker appears, while
|
||||
// asserting no "Wrong chunk header!" reappears AND no new error_backup files
|
||||
// land for the healed region.
|
||||
func watchBoot(ctx context.Context, container, regionDir string, region regionmedic.RegionCoord, timeout time.Duration) error {
|
||||
if err := waitContainer(ctx, container, true, 60*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
readyRe := []string{"StartGame done", "GameStartDone", "by Telnet from"}
|
||||
for time.Now().Before(deadline) {
|
||||
out, _ := exec.CommandContext(ctx, "docker", "logs", "--since", "5m", container).CombinedOutput()
|
||||
s := string(out)
|
||||
if strings.Contains(s, "Wrong chunk header") {
|
||||
// Only fail if it's for our region — re-scan the dir to be sure.
|
||||
scan, _ := regionmedic.ScanRegionDir(regionDir)
|
||||
for _, a := range scan.Affected {
|
||||
if a.Region == region {
|
||||
return fmt.Errorf("healed region %s threw new error_backups (%d) after restart", region, a.ErrorBackups)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range readyRe {
|
||||
if strings.Contains(s, m) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
time.Sleep(4 * time.Second)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for ready marker")
|
||||
}
|
||||
|
||||
func regionArg(s string) string {
|
||||
if strings.HasSuffix(s, ".7rg") {
|
||||
return s
|
||||
}
|
||||
return s + ".7rg"
|
||||
}
|
||||
|
||||
func orDefault(s, def string) string {
|
||||
if s == "" {
|
||||
return def
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user