panel — open-source game server manager (public release)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,388 @@
|
||||
// Package archive walks and re-encodes archive files (zip / tar /
|
||||
// tar.gz / tar.bz2 / tar.xz / 7z / rar) fully in-process — no native
|
||||
// deps. It is shared by the file browser's extract/compress endpoints
|
||||
// (internal/dispatch) and the direct update provider
|
||||
// (internal/updater), which extracts archive downloads into the
|
||||
// instance instead of dumping the raw archive bytes.
|
||||
//
|
||||
// 7z and rar use pure-Go libraries (bodgit/sevenzip,
|
||||
// nwaples/rardecode/v2). Encrypted rars and solid 7z volumes are
|
||||
// surfaced as errors, not silently skipped.
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bodgit/sevenzip"
|
||||
"github.com/nwaples/rardecode/v2"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// Entry-count / byte caps so a malformed or hostile archive can't pin
|
||||
// the agent forever walking phantom records.
|
||||
const (
|
||||
MaxEntries = 200000
|
||||
MaxTotalBytes = 8 * 1024 * 1024 * 1024
|
||||
EntrySoftBytes = 4 * 1024 * 1024 * 1024
|
||||
)
|
||||
|
||||
// EntryFunc is called once per archive entry. body is a streaming
|
||||
// reader (nil for directories); the callback may read fully or skip.
|
||||
type EntryFunc func(name string, mode int64, isDir bool, body io.Reader) error
|
||||
|
||||
// Walk opens `data` as an archive (auto-detected via signature plus
|
||||
// filename hint) and calls fn for each entry. Returns entry and byte
|
||||
// counts alongside any error.
|
||||
func Walk(data []byte, name string, fn EntryFunc) (int64, int64, error) {
|
||||
switch DetectFormat(data, name) {
|
||||
case "zip":
|
||||
return walkZip(data, fn)
|
||||
case "tar.gz":
|
||||
gz, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("gzip open: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
return walkTar(gz, fn)
|
||||
case "tar.bz2":
|
||||
return walkTar(bzip2.NewReader(bytes.NewReader(data)), fn)
|
||||
case "tar.xz":
|
||||
xr, err := xz.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("xz open: %w", err)
|
||||
}
|
||||
return walkTar(xr, fn)
|
||||
case "tar":
|
||||
return walkTar(bytes.NewReader(data), fn)
|
||||
case "7z":
|
||||
return walk7z(data, fn)
|
||||
case "rar":
|
||||
return walkRar(data, fn)
|
||||
default:
|
||||
return 0, 0, fmt.Errorf("unsupported archive format for %q (panel knows zip / tar / tar.gz / tar.bz2 / tar.xz / 7z / rar)", name)
|
||||
}
|
||||
}
|
||||
|
||||
// WriteAsTar parses an archive from `data` and re-emits its entries as
|
||||
// a tar stream into `out`. Used by container-side extract paths so the
|
||||
// agent can feed Docker's CopyToContainer in a single API call.
|
||||
func WriteAsTar(data []byte, originalName string, out io.Writer) (int64, int64, error) {
|
||||
tw := tar.NewWriter(out)
|
||||
defer tw.Close()
|
||||
var entries, bytesOut int64
|
||||
_, _, err := Walk(data, originalName, func(name string, mode int64, isDir bool, body io.Reader) error {
|
||||
safe := SafeEntryPath(name)
|
||||
if safe == "" {
|
||||
return nil
|
||||
}
|
||||
// Buffer the entry body so we can compute its length before
|
||||
// writing the tar header. zip / gzip readers return streaming
|
||||
// readers; the tar writer needs Size up front.
|
||||
var bodyBuf bytes.Buffer
|
||||
var sz int64
|
||||
if !isDir {
|
||||
n, err := io.Copy(&bodyBuf, body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("entry %q: %w", name, err)
|
||||
}
|
||||
sz = n
|
||||
if sz > EntrySoftBytes {
|
||||
return fmt.Errorf("entry %q exceeds %d-byte cap", name, int64(EntrySoftBytes))
|
||||
}
|
||||
}
|
||||
hdr := &tar.Header{Name: safe, ModTime: time.Now(), Mode: 0o644}
|
||||
if mode != 0 {
|
||||
hdr.Mode = mode & 0o777
|
||||
}
|
||||
if isDir {
|
||||
hdr.Typeflag = tar.TypeDir
|
||||
hdr.Mode = 0o755
|
||||
if !strings.HasSuffix(hdr.Name, "/") {
|
||||
hdr.Name += "/"
|
||||
}
|
||||
} else {
|
||||
hdr.Typeflag = tar.TypeReg
|
||||
hdr.Size = sz
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isDir {
|
||||
if _, err := tw.Write(bodyBuf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
entries++
|
||||
bytesOut += sz
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
return entries, bytesOut, tw.Close()
|
||||
}
|
||||
|
||||
// DetectFormat classifies `data` by magic bytes first (handles renamed
|
||||
// extensions), then falls back to the filename hint. Returns "" when
|
||||
// the content is not a recognized archive.
|
||||
func DetectFormat(data []byte, name string) string {
|
||||
lower := strings.ToLower(name)
|
||||
// Magic bytes win when present — handles renamed extensions.
|
||||
if len(data) >= 4 && data[0] == 0x50 && data[1] == 0x4b && (data[2] == 0x03 || data[2] == 0x05 || data[2] == 0x07) {
|
||||
return "zip"
|
||||
}
|
||||
if len(data) >= 3 && data[0] == 0x1f && data[1] == 0x8b && data[2] == 0x08 {
|
||||
return "tar.gz"
|
||||
}
|
||||
if len(data) >= 3 && data[0] == 'B' && data[1] == 'Z' && data[2] == 'h' {
|
||||
return "tar.bz2"
|
||||
}
|
||||
// 7z signature: 37 7A BC AF 27 1C
|
||||
if len(data) >= 6 && data[0] == 0x37 && data[1] == 0x7a && data[2] == 0xbc && data[3] == 0xaf && data[4] == 0x27 && data[5] == 0x1c {
|
||||
return "7z"
|
||||
}
|
||||
// RAR 5.0 signature: 52 61 72 21 1A 07 01 00 ("Rar!\x1a\x07\x01\x00")
|
||||
// RAR 1.5–4.x: 52 61 72 21 1A 07 00 ("Rar!\x1a\x07\x00")
|
||||
if len(data) >= 7 && data[0] == 'R' && data[1] == 'a' && data[2] == 'r' && data[3] == '!' && data[4] == 0x1a && data[5] == 0x07 {
|
||||
return "rar"
|
||||
}
|
||||
// xz signature: FD 37 7A 58 5A 00
|
||||
if len(data) >= 6 && data[0] == 0xfd && data[1] == '7' && data[2] == 'z' && data[3] == 'X' && data[4] == 'Z' && data[5] == 0x00 {
|
||||
return "tar.xz"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") {
|
||||
return "tar.gz"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.bz2") || strings.HasSuffix(lower, ".tbz2") || strings.HasSuffix(lower, ".tbz") {
|
||||
return "tar.bz2"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.xz") || strings.HasSuffix(lower, ".txz") {
|
||||
return "tar.xz"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar") {
|
||||
return "tar"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".zip") {
|
||||
return "zip"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".7z") {
|
||||
return "7z"
|
||||
}
|
||||
if strings.HasSuffix(lower, ".rar") {
|
||||
return "rar"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func walkZip(data []byte, fn EntryFunc) (int64, int64, error) {
|
||||
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("zip open: %w", err)
|
||||
}
|
||||
var entries, bytesOut int64
|
||||
var totalBytes int64
|
||||
for _, f := range zr.File {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
isDir := f.FileInfo().IsDir() || strings.HasSuffix(f.Name, "/")
|
||||
if isDir {
|
||||
if err := fn(f.Name, int64(f.Mode()), true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("zip entry %q: %w", f.Name, err)
|
||||
}
|
||||
// Cap each entry by `EntrySoftBytes` so a malformed/zip-bomb
|
||||
// entry can't run away with memory. We DO NOT trust
|
||||
// f.UncompressedSize64 — some zip writers leave it 0 for streamed
|
||||
// entries; using it for accounting (bytesOut += size) caused the
|
||||
// per-archive total to be charged the 4 GB soft cap per such
|
||||
// entry, tripping MaxTotalBytes after just a couple of
|
||||
// files and aborting the extract mid-stream — silently leaving
|
||||
// the first N files extracted and the rest dropped (this was the
|
||||
// root cause of the 7DTD mod-upload "Config/ folders missing"
|
||||
// bug). Use a counting reader so accounting reflects real bytes.
|
||||
cr := &countingReader{r: io.LimitReader(rc, EntrySoftBytes)}
|
||||
if err := fn(f.Name, int64(f.Mode()), false, cr); err != nil {
|
||||
_ = rc.Close()
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
_ = rc.Close()
|
||||
entries++
|
||||
bytesOut += cr.n
|
||||
totalBytes += cr.n
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
|
||||
// countingReader wraps an io.Reader and tracks the byte count actually
|
||||
// consumed. Used by walkZip / walkTar accounting so the per-archive
|
||||
// total reflects real-world bytes, not zip-header declared sizes (which
|
||||
// some encoders leave as 0).
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func walkTar(r io.Reader, fn EntryFunc) (int64, int64, error) {
|
||||
tr := tar.NewReader(r)
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("tar next: %w", err)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := fn(hdr.Name, hdr.Mode, true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
if err := fn(hdr.Name, hdr.Mode, false, tr); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
bytesOut += hdr.Size
|
||||
totalBytes += hdr.Size
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func walk7z(data []byte, fn EntryFunc) (int64, int64, error) {
|
||||
zr, err := sevenzip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("7z open: %w", err)
|
||||
}
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for _, f := range zr.File {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
isDir := f.FileInfo().IsDir() || strings.HasSuffix(f.Name, "/")
|
||||
if isDir {
|
||||
if err := fn(f.Name, int64(f.Mode()), true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("7z entry %q: %w", f.Name, err)
|
||||
}
|
||||
// See countingReader rationale in walkZip — header-declared
|
||||
// size can be 0 for streamed entries, charging the soft-cap per
|
||||
// entry would falsely abort the extract.
|
||||
cr := &countingReader{r: io.LimitReader(rc, EntrySoftBytes)}
|
||||
if err := fn(f.Name, int64(f.Mode()), false, cr); err != nil {
|
||||
_ = rc.Close()
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
_ = rc.Close()
|
||||
entries++
|
||||
bytesOut += cr.n
|
||||
totalBytes += cr.n
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
|
||||
func walkRar(data []byte, fn EntryFunc) (int64, int64, error) {
|
||||
rr, err := rardecode.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("rar open: %w", err)
|
||||
}
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for {
|
||||
if entries >= MaxEntries {
|
||||
return entries, bytesOut, fmt.Errorf("archive has more than %d entries", int64(MaxEntries))
|
||||
}
|
||||
hdr, err := rr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if err != nil {
|
||||
return entries, bytesOut, fmt.Errorf("rar next: %w", err)
|
||||
}
|
||||
mode := int64(hdr.Mode().Perm())
|
||||
if hdr.IsDir {
|
||||
if err := fn(hdr.Name, mode, true, nil); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
continue
|
||||
}
|
||||
// See countingReader rationale in walkZip.
|
||||
cr := &countingReader{r: io.LimitReader(rr, EntrySoftBytes)}
|
||||
if err := fn(hdr.Name, mode, false, cr); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
bytesOut += cr.n
|
||||
totalBytes += cr.n
|
||||
if totalBytes > MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("archive uncompressed size exceeds %d bytes", int64(MaxTotalBytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SafeEntryPath strips leading slashes / drive letters and
|
||||
// rejects any path that tries to ../ out of the destination. Returns
|
||||
// "" on rejected entries — caller skips them silently.
|
||||
func SafeEntryPath(name string) string {
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
// Normalize separators + strip leading drive / slash.
|
||||
clean := strings.ReplaceAll(name, "\\", "/")
|
||||
for strings.HasPrefix(clean, "/") {
|
||||
clean = clean[1:]
|
||||
}
|
||||
if len(clean) >= 2 && clean[1] == ':' {
|
||||
clean = clean[2:]
|
||||
}
|
||||
clean = path.Clean(clean)
|
||||
if clean == "." || clean == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(clean, "../") || clean == ".." || strings.Contains(clean, "/../") {
|
||||
return ""
|
||||
}
|
||||
return clean
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// buildTar writes a small dir+file tree as a tar stream.
|
||||
func buildTar(t *testing.T, w io.Writer) {
|
||||
t.Helper()
|
||||
tw := tar.NewWriter(w)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "factorio/", Typeflag: tar.TypeDir, Mode: 0o755}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("#!/bin/sh\necho hi\n")
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "factorio/bin/x64/factorio", Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func makeTarXz(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
xw, err := xz.NewWriter(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buildTar(t, xw)
|
||||
if err := xw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func makeTarGz(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
buildTar(t, gw)
|
||||
if err := gw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func makeZip(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
f, err := zw.Create("factorio/bin/x64/factorio")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.Write([]byte("#!/bin/sh\necho hi\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestDetectFormatMagicBytes(t *testing.T) {
|
||||
// Extensionless names — magic bytes must carry detection (factorio's
|
||||
// download URL has no file extension).
|
||||
cases := []struct {
|
||||
data []byte
|
||||
want string
|
||||
}{
|
||||
{makeTarXz(t), "tar.xz"},
|
||||
{makeTarGz(t), "tar.gz"},
|
||||
{makeZip(t), "zip"},
|
||||
{[]byte("plain text, not an archive"), ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := DetectFormat(c.data, "https://example.com/get-download/stable/headless/linux64"); got != c.want {
|
||||
t.Errorf("DetectFormat = %q, want %q", got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectFormatExtensionFallback(t *testing.T) {
|
||||
// Content too short for magic — extension hint decides.
|
||||
if got := DetectFormat([]byte{}, "thing.tar.xz"); got != "tar.xz" {
|
||||
t.Errorf("extension fallback = %q, want tar.xz", got)
|
||||
}
|
||||
if got := DetectFormat([]byte{}, "thing.tgz"); got != "tar.gz" {
|
||||
t.Errorf("extension fallback = %q, want tar.gz", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkAllFormats(t *testing.T) {
|
||||
for name, data := range map[string][]byte{
|
||||
"tar.xz": makeTarXz(t),
|
||||
"tar.gz": makeTarGz(t),
|
||||
"zip": makeZip(t),
|
||||
} {
|
||||
var files []string
|
||||
var content []byte
|
||||
entries, _, err := Walk(data, "blob", func(n string, mode int64, isDir bool, body io.Reader) error {
|
||||
if isDir {
|
||||
return nil
|
||||
}
|
||||
files = append(files, n)
|
||||
b, err := io.ReadAll(body)
|
||||
content = b
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: Walk: %v", name, err)
|
||||
}
|
||||
if entries == 0 || len(files) != 1 || files[0] != "factorio/bin/x64/factorio" {
|
||||
t.Fatalf("%s: unexpected entries %d files %v", name, entries, files)
|
||||
}
|
||||
if string(content) != "#!/bin/sh\necho hi\n" {
|
||||
t.Fatalf("%s: content mismatch: %q", name, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteAsTarRoundTrip(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
entries, bytesOut, err := WriteAsTar(makeTarXz(t), "blob", &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entries != 2 {
|
||||
t.Fatalf("entries = %d, want 2", entries)
|
||||
}
|
||||
if bytesOut == 0 {
|
||||
t.Fatal("bytesOut = 0")
|
||||
}
|
||||
// Re-walk the emitted tar and check the file mode survived.
|
||||
tr := tar.NewReader(&out)
|
||||
var sawFile bool
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeReg {
|
||||
sawFile = true
|
||||
if hdr.Name != "factorio/bin/x64/factorio" {
|
||||
t.Fatalf("name = %q", hdr.Name)
|
||||
}
|
||||
if hdr.Mode&0o111 == 0 {
|
||||
t.Fatalf("exec bit lost: mode %o", hdr.Mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !sawFile {
|
||||
t.Fatal("no regular file in re-emitted tar")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeEntryPathRejectsTraversal(t *testing.T) {
|
||||
for _, bad := range []string{"../evil", "a/../../evil", "..", ""} {
|
||||
if got := SafeEntryPath(bad); got != "" {
|
||||
t.Errorf("SafeEntryPath(%q) = %q, want rejection", bad, got)
|
||||
}
|
||||
}
|
||||
if got := SafeEntryPath("/abs/path"); got != "abs/path" {
|
||||
t.Errorf("leading slash strip = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package dispatch
|
||||
|
||||
// ArkSaveRestore: swap a SavedArks/<subdir>/ rolling backup into the
|
||||
// active <subdir>.ark slot. The previously-active save is moved aside
|
||||
// (NEVER deleted) — even abandoned attempts are recoverable from the
|
||||
// timestamped *_replaced.ark filename.
|
||||
//
|
||||
// Sequence (all server-side, atomic from the operator's POV):
|
||||
// 1. Validate inputs.
|
||||
// 2. Confirm the main container is NOT running. We refuse the swap
|
||||
// if it is, because mid-write file ops in ARK's save dir corrupt
|
||||
// the rolling backup chain.
|
||||
// 3. mv <subdir>/<subdir>.ark -> <subdir>/<subdir>_<dd.mm.yyyy>_<HH.MM.SS>_replaced.ark
|
||||
// 4. cp <subdir>/<target_filename> -> <subdir>/<subdir>.ark
|
||||
// For .arkrbf rolling backups the dest still ends in .ark — the
|
||||
// engine doesn't care about the source file's extension, only the
|
||||
// slot name.
|
||||
//
|
||||
// Caller (controller) is responsible for stopping the container before
|
||||
// this RPC. If the operator started the container in the meantime,
|
||||
// this handler short-circuits cleanly with a friendly error.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// arkSavedArksContainerRoot is the relative path to ARK's rolling-save
|
||||
// directory FROM the ark-sa module's BrowseableRoot. The module roots
|
||||
// browsing at ".../ShooterGame/Saved" (see modules/ark-sa/module.yaml),
|
||||
// so this is the segment beneath that — NOT the full container path.
|
||||
const arkSavedArksContainerRoot = "SavedArks"
|
||||
|
||||
func (d *Dispatcher) handleArkSaveRestore(corrID string, req *panelv1.ArkSaveRestoreRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
subdir := strings.TrimSpace(req.SavedArksSubdir)
|
||||
target := strings.TrimSpace(req.TargetFilename)
|
||||
if subdir == "" || target == "" {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "saved_arks_subdir and target_filename are required")
|
||||
return
|
||||
}
|
||||
// Path-safety: subdir is one segment under SavedArks/, target is
|
||||
// one segment under that. Reject any traversal attempt up front.
|
||||
if strings.ContainsAny(subdir, "/\\") || subdir == "." || subdir == ".." {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "saved_arks_subdir must be a single directory name")
|
||||
return
|
||||
}
|
||||
if strings.ContainsAny(target, "/\\") || target == "." || target == ".." {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "target_filename must be a single file name")
|
||||
return
|
||||
}
|
||||
// Sanity-check extension. Active slot is .ark; sources may be .ark
|
||||
// or .arkrbf (rolling backup). .bak is ARK's anti-corruption file
|
||||
// and we don't restore from it directly — operators rename to .ark
|
||||
// manually if they need to.
|
||||
lowerTarget := strings.ToLower(target)
|
||||
if !strings.HasSuffix(lowerTarget, ".ark") && !strings.HasSuffix(lowerTarget, ".arkrbf") {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "target_filename must end in .ark or .arkrbf")
|
||||
return
|
||||
}
|
||||
|
||||
activeName := subdir + ".ark"
|
||||
if target == activeName {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "target is already the active save")
|
||||
return
|
||||
}
|
||||
|
||||
// Stamp the aside name with current wall time. We use the user's
|
||||
// "dd.mm.yyyy_HH.MM.SS" format from ARK's own rolling backups so
|
||||
// the aside file sorts naturally next to genuine engine backups
|
||||
// in the file browser. Suffix "_replaced" disambiguates them.
|
||||
// Uses UTC because that's what the engine writes — keeps the
|
||||
// frontend's parser (which assumes UTC) honest for both sources.
|
||||
now := time.Now().UTC()
|
||||
asideName := fmt.Sprintf("%s_%s_replaced.ark",
|
||||
subdir, now.Format("02.01.2006_15.04.05"))
|
||||
|
||||
if d.useContainerOps(rec) {
|
||||
d.arkRestoreInContainer(corrID, rec, subdir, target, activeName, asideName)
|
||||
return
|
||||
}
|
||||
d.arkRestoreOnHost(corrID, rec, subdir, target, activeName, asideName)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) arkRestoreInContainer(corrID string, rec *instanceRecord, subdir, target, activeName, asideName string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// Refuse if the main game container is still running. We don't
|
||||
// want file ops happening inside SavedArks while the engine is
|
||||
// writing rolling backups.
|
||||
mainName := "panel-" + rec.InstanceID
|
||||
if st, err := d.runtime.InspectByName(ctx, mainName); err == nil && st.Status == "running" {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "",
|
||||
"refusing to restore while server is running — stop the server first")
|
||||
return
|
||||
}
|
||||
|
||||
subdirRel := path.Join(arkSavedArksContainerRoot, subdir)
|
||||
subdirAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), subdirRel)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
activeAbs := path.Join(subdirAbs, activeName)
|
||||
asideAbs := path.Join(subdirAbs, asideName)
|
||||
targetAbs := path.Join(subdirAbs, target)
|
||||
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Verify the target file exists before we touch the active slot.
|
||||
// `test -f` returns exit 1 (no file), 0 (exists), other (error).
|
||||
_, _, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"test -f " + shellQuote(targetAbs)})
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", fmt.Sprintf("target save %q not found", target))
|
||||
return
|
||||
}
|
||||
|
||||
// Move the previously-active save aside (only if it exists; some
|
||||
// ops may run on a freshly-installed instance with no active save).
|
||||
finalAside := ""
|
||||
_, _, code, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"test -f " + shellQuote(activeAbs)})
|
||||
if code == 0 {
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mv -- " + shellQuote(activeAbs) + " " + shellQuote(asideAbs)})
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("mv aside exited %d", code)
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", msg)
|
||||
return
|
||||
}
|
||||
finalAside = asideName
|
||||
}
|
||||
|
||||
// Copy the chosen backup into the active slot. cp preserves the
|
||||
// source so the operator can pick it again later if they change
|
||||
// their mind. -p preserves mtime so the engine sees the original
|
||||
// save's modification time.
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"cp -p -- " + shellQuote(targetAbs) + " " + shellQuote(activeAbs)})
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("cp exited %d", code)
|
||||
}
|
||||
// Roll the aside back if cp failed mid-flight.
|
||||
if finalAside != "" {
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mv -- " + shellQuote(asideAbs) + " " + shellQuote(activeAbs)})
|
||||
finalAside = ""
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", msg)
|
||||
return
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, activeName, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) arkRestoreOnHost(corrID string, rec *instanceRecord, subdir, target, activeName, asideName string) {
|
||||
if rec.DataPath == "" {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", "no file storage available")
|
||||
return
|
||||
}
|
||||
subdirRel := filepath.Join(arkSavedArksContainerRoot, subdir)
|
||||
subdirAbs, err := safeJoinHost(rec.DataPath, subdirRel)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
activeAbs := filepath.Join(subdirAbs, activeName)
|
||||
asideAbs := filepath.Join(subdirAbs, asideName)
|
||||
targetAbs := filepath.Join(subdirAbs, target)
|
||||
|
||||
if _, err := os.Stat(targetAbs); err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", fmt.Sprintf("target save %q not found", target))
|
||||
return
|
||||
}
|
||||
finalAside := ""
|
||||
if _, err := os.Stat(activeAbs); err == nil {
|
||||
if err := os.Rename(activeAbs, asideAbs); err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, "", "", err.Error())
|
||||
return
|
||||
}
|
||||
finalAside = asideName
|
||||
}
|
||||
in, err := os.Open(targetAbs)
|
||||
if err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.OpenFile(activeAbs, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
// Roll aside back so we don't leave the server with no save.
|
||||
if finalAside != "" {
|
||||
_ = os.Rename(asideAbs, activeAbs)
|
||||
finalAside = ""
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
_ = out.Close()
|
||||
if finalAside != "" {
|
||||
_ = os.Remove(activeAbs)
|
||||
_ = os.Rename(asideAbs, activeAbs)
|
||||
finalAside = ""
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, "", err.Error())
|
||||
return
|
||||
}
|
||||
d.sendArkSaveRestoreResult(corrID, finalAside, activeName, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendArkSaveRestoreResult(corrID, asideFilename, installedFilename, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_ArkSaveRestoreResult{
|
||||
ArkSaveRestoreResult: &panelv1.ArkSaveRestoreResult{
|
||||
AsideFilename: asideFilename,
|
||||
InstalledFilename: installedFilename,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,505 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// buildBackupTarCmd assembles the tar command run inside the sidecar.
|
||||
// Manifest BackupSpec wins when Include is set: the tar runs against
|
||||
// the listed paths (each relative to BrowseableRoot, missing ones
|
||||
// silently skipped). Otherwise it falls back to a whole-tree archive
|
||||
// of BrowseableRoot. Excludes are passed through tar's --exclude.
|
||||
//
|
||||
// Resulting tarball preserves the directory layout under BrowseableRoot
|
||||
// so restore can extract straight into the same root.
|
||||
func buildBackupTarCmd(manifest *modulepkgManifest, root, fileName string, keep int) string {
|
||||
if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
|
||||
root = manifest.Backup.Root
|
||||
}
|
||||
var base string
|
||||
if manifest == nil || manifest.Backup == nil || len(manifest.Backup.Include) == 0 {
|
||||
// Fallback: whole tree.
|
||||
base = fmt.Sprintf("cd %q && tar czf /backup/%s . && stat -c %%s /backup/%s",
|
||||
root, shellEscape(fileName), shellEscape(fileName))
|
||||
} else {
|
||||
// Per-game include list. Missing paths are non-fatal — players will
|
||||
// create new instances on a fresh server before the save dir exists,
|
||||
// and we don't want their first Back-up-now to fail the run.
|
||||
excludes := ""
|
||||
for _, e := range manifest.Backup.Exclude {
|
||||
excludes += " --exclude=" + shellEscape(e)
|
||||
}
|
||||
// Build "test -e <path> && echo <path>" for each include — only
|
||||
// existing paths get added to the tar argv, so the operator doesn't
|
||||
// see a confusing "Cannot stat: No such file" on first run.
|
||||
//
|
||||
// Newline-delimited (NOT NUL-delimited) because the sidecar image is
|
||||
// alpine, which ships busybox tar — and busybox tar doesn't grok
|
||||
// `--null`. Manifest paths never contain whitespace or newlines (we
|
||||
// hand-curate them per game), so a plain newline list is safe.
|
||||
var existsChecks []string
|
||||
for _, p := range manifest.Backup.Include {
|
||||
clean := strings.TrimPrefix(p, "./")
|
||||
clean = strings.TrimPrefix(clean, "/")
|
||||
existsChecks = append(existsChecks, fmt.Sprintf("[ -e %s ] && printf '%%s\\n' %s", shellEscape(clean), shellEscape(clean)))
|
||||
}
|
||||
listExpr := "{ " + strings.Join(existsChecks, " ; ") + " ; }"
|
||||
base = fmt.Sprintf("cd %q && %s | tar czf /backup/%s%s -T - && stat -c %%s /backup/%s",
|
||||
root, listExpr, shellEscape(fileName), excludes, shellEscape(fileName))
|
||||
}
|
||||
return base + backupPruneSuffix(keep)
|
||||
}
|
||||
|
||||
// backupPruneSuffix appends a best-effort retention prune that keeps only the
|
||||
// newest `keep` *.tar.gz in /backup (by mtime) and deletes the rest. It runs in
|
||||
// the SAME root sidecar that just wrote the new tarball — the backup files are
|
||||
// root-owned, so the unprivileged agent can't prune them itself. keep<=0 leaves
|
||||
// pruning off. busybox-safe (alpine): ls -1t + tail -n +N + while-read — no
|
||||
// `head -n -N` (GNU-only) and no `xargs -r`.
|
||||
func backupPruneSuffix(keep int) string {
|
||||
if keep <= 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" ; echo 'prune: keeping newest %d backups' ; ls -1t /backup/*.tar.gz 2>/dev/null | tail -n +%d | while IFS= read -r f; do echo \"prune: rm $f\" ; rm -f \"$f\" ; done",
|
||||
keep, keep+1)
|
||||
}
|
||||
|
||||
// readBackupKeep returns the backup retention for this instance: the integer in
|
||||
// <dataPath>/backup-keep when present, else the PANEL_BACKUP_KEEP env default,
|
||||
// else 10. A rolling cap stops the unbounded pileup that filled 90 GB on a busy
|
||||
// 7DTD season. Set per-instance by writing the file (the Backups UI exposes it).
|
||||
func readBackupKeep(dataPath string) int {
|
||||
def := 10
|
||||
if v := os.Getenv("PANEL_BACKUP_KEEP"); v != "" {
|
||||
if n, e := strconv.Atoi(strings.TrimSpace(v)); e == nil && n > 0 {
|
||||
def = n
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dataPath, "backup-keep"))
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
if n, e := strconv.Atoi(strings.TrimSpace(string(b))); e == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// modulepkgManifest is a local alias so the helper signature below
|
||||
// doesn't drag modulepkg into its return type at the file top.
|
||||
type modulepkgManifest = modulepkg.Manifest
|
||||
|
||||
// Backup / restore via sidecar containers. Two operations:
|
||||
//
|
||||
// - Backup: tar -czf /backup/<file>.tar.gz -C /src . (volume readonly at /src)
|
||||
// - Restore: tar -xzf /backup/<file>.tar.gz -C /dst (volume read-write at /dst)
|
||||
//
|
||||
// The sidecar uses alpine:3.21 + busybox tar. Backup files live on the
|
||||
// agent host under $BACKUP_DIR/<instance_id>/<ts>-<bkpID>.tar.gz; both
|
||||
// sidecars bind-mount $BACKUP_DIR/<instance_id> at /backup.
|
||||
//
|
||||
// For restore we refuse if the main container is running: extracting
|
||||
// files under a live server would corrupt world state. Operator stops
|
||||
// first. (Symmetrical to the SteamCMD updater's running-container check.)
|
||||
|
||||
const (
|
||||
backupImage = "alpine:3.21"
|
||||
backupSidecarTTL = 30 * time.Minute
|
||||
)
|
||||
|
||||
// ---- Backup ----
|
||||
|
||||
func (d *Dispatcher) handleBackup(ctx context.Context, corrID string, req *panelv1.BackupRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||||
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, "", "", 0, err.Error())
|
||||
return
|
||||
}
|
||||
if rec.BrowseableRoot == "" {
|
||||
d.sendBackupResult(corrID, "", "", 0, "instance has no BrowseableRoot to tar (module declares no volumes)")
|
||||
return
|
||||
}
|
||||
if d.backupDir == "" {
|
||||
d.sendBackupResult(corrID, "", "", 0, "agent has no --backup-dir configured")
|
||||
return
|
||||
}
|
||||
|
||||
// Prepare the agent-side target directory.
|
||||
bkpID := "bkp_" + randHex(8)
|
||||
hostDir := filepath.Join(d.backupDir, req.InstanceId)
|
||||
if err := os.MkdirAll(hostDir, 0o755); err != nil {
|
||||
d.sendBackupResult(corrID, "", "", 0, fmt.Sprintf("mkdir %s: %s", hostDir, err.Error()))
|
||||
return
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
fileName := fmt.Sprintf("%s-%s.tar.gz", ts, bkpID)
|
||||
hostFile := filepath.Join(hostDir, fileName)
|
||||
|
||||
log.Info("backup starting", "backup_id", bkpID, "host_file", hostFile)
|
||||
|
||||
// Build sidecar spec that mounts the same volumes as the instance,
|
||||
// plus the host backup directory at /backup.
|
||||
volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
|
||||
// Mark volumes read-only for backup safety.
|
||||
for i := range volumes {
|
||||
volumes[i].ReadOnly = true
|
||||
}
|
||||
// Add the host backup dir as a (read-write) bind.
|
||||
volumes = append(volumes, runtime.VolumeSpec{
|
||||
Type: "bind",
|
||||
HostPath: hostDir,
|
||||
ContainerPath: "/backup",
|
||||
})
|
||||
|
||||
// Per-game backup paths. If the manifest declares Backup.Include,
|
||||
// only those paths get archived (saves-only — keeps ARK SA tarballs
|
||||
// at ~250 MB instead of the 18 GB whole-install). Without a spec,
|
||||
// fall back to the original "tar everything under BrowseableRoot"
|
||||
// behavior so older modules don't regress.
|
||||
manifest := d.modulesMustGet(rec.ModuleID)
|
||||
keep := readBackupKeep(rec.DataPath)
|
||||
tarCmd := buildBackupTarCmd(manifest, rec.BrowseableRoot, fileName, keep)
|
||||
if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
|
||||
log.Info("backup using manifest-declared paths", "include_count", len(manifest.Backup.Include))
|
||||
} else {
|
||||
log.Warn("backup falling back to whole BrowseableRoot — module manifest has no `backup.include`",
|
||||
"module_id", rec.ModuleID, "root", rec.BrowseableRoot)
|
||||
}
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: req.InstanceId + "-backup",
|
||||
IsSidecar: true,
|
||||
Image: backupImage,
|
||||
Entrypoint: []string{"sh"},
|
||||
Command: []string{"-c", tarCmd},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, "create sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, "start sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
// Stream log lines for visibility.
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID, Stream: "backup",
|
||||
At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, "wait: "+err.Error())
|
||||
return
|
||||
}
|
||||
if exitCode != 0 {
|
||||
d.sendBackupResult(corrID, bkpID, "", 0, fmt.Sprintf("sidecar exited %d", exitCode))
|
||||
return
|
||||
}
|
||||
|
||||
info, err := os.Stat(hostFile)
|
||||
if err != nil {
|
||||
d.sendBackupResult(corrID, bkpID, hostFile, 0, "stat result: "+err.Error())
|
||||
return
|
||||
}
|
||||
log.Info("backup complete", "backup_id", bkpID, "size_bytes", info.Size())
|
||||
d.sendBackupResult(corrID, bkpID, hostFile, info.Size(), "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendBackupResult(corrID, bkpID, path string, size int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_BackupResult{
|
||||
BackupResult: &panelv1.BackupResult{
|
||||
BackupId: bkpID,
|
||||
Path: path,
|
||||
SizeBytes: size,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- Wipe paths (used by the env-config recreate flow) ----
|
||||
//
|
||||
// Spawns an alpine sidecar that mounts the same volumes the instance
|
||||
// uses, runs `rm -rf` on each path (relative to BrowseableRoot), then
|
||||
// exits. Designed for the ARK map-switch flow — operator picked a new
|
||||
// SERVER_MAP and checked "wipe existing world data" so the old map's
|
||||
// SavedArks/<MapName>/ goes away. Generic enough that any future
|
||||
// env-config field can declare its own wipe paths.
|
||||
func (d *Dispatcher) runWipeSidecar(ctx context.Context, manifest *modulepkg.Manifest, instanceID, dataPath string, paths []string) error {
|
||||
if manifest == nil {
|
||||
return errors.New("nil manifest")
|
||||
}
|
||||
root := resolveBrowseableRoot(manifest)
|
||||
if root == "" {
|
||||
return errors.New("module declares no browseable_root — wipe needs a base path")
|
||||
}
|
||||
volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
|
||||
// Build the rm -rf command. Each path is rooted under
|
||||
// BrowseableRoot. Missing paths are silently skipped (first run).
|
||||
var rmParts []string
|
||||
for _, p := range paths {
|
||||
clean := p
|
||||
// Defensive: reject absolute paths and parent traversal so a
|
||||
// rogue config_values value can't `rm -rf /` the container.
|
||||
if strings.HasPrefix(clean, "/") || strings.Contains(clean, "..") {
|
||||
return fmt.Errorf("refusing wipe of suspicious path %q", p)
|
||||
}
|
||||
full := root + "/" + clean
|
||||
rmParts = append(rmParts, fmt.Sprintf("rm -rf %s", shellEscape(full)))
|
||||
}
|
||||
cmd := strings.Join(rmParts, " && ") + " && echo wiped"
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: instanceID + "-wipe",
|
||||
IsSidecar: true,
|
||||
Image: backupImage,
|
||||
Entrypoint: []string{"sh"},
|
||||
Command: []string{"-c", cmd},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create wipe sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return fmt.Errorf("start wipe sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: instanceID, Stream: "wipe",
|
||||
At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return fmt.Errorf("wait wipe sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return fmt.Errorf("wipe sidecar exited %d", exitCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- Restore ----
|
||||
|
||||
func (d *Dispatcher) handleRestore(ctx context.Context, corrID string, req *panelv1.RestoreRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "backup_id", req.BackupId, "correlation_id", corrID)
|
||||
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendRestoreResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if rec.BrowseableRoot == "" {
|
||||
d.sendRestoreResult(corrID, 0, "instance has no BrowseableRoot to restore into")
|
||||
return
|
||||
}
|
||||
if req.BackupPath == "" {
|
||||
d.sendRestoreResult(corrID, 0, "backup_path required")
|
||||
return
|
||||
}
|
||||
// Refuse if container is running.
|
||||
if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil && state.Status == "running" {
|
||||
d.sendRestoreResult(corrID, 0, "stop the instance first — main container is running")
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(req.BackupPath)
|
||||
if err != nil || info.IsDir() {
|
||||
d.sendRestoreResult(corrID, 0, "backup file missing or is a directory")
|
||||
return
|
||||
}
|
||||
|
||||
volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
|
||||
// Add the backup file's parent as a read-only bind at /backup.
|
||||
hostDir := filepath.Dir(req.BackupPath)
|
||||
fileName := filepath.Base(req.BackupPath)
|
||||
volumes = append(volumes, runtime.VolumeSpec{
|
||||
Type: "bind",
|
||||
HostPath: hostDir,
|
||||
ContainerPath: "/backup",
|
||||
ReadOnly: true,
|
||||
})
|
||||
|
||||
// Extract strategy depends on whether the backup is per-game-paths
|
||||
// or a whole-tree archive. When the manifest declares Backup.Include,
|
||||
// only THOSE paths are wiped before extraction — preserves the
|
||||
// 18 GB game install while still giving a clean save dir. Without a
|
||||
// spec, fall back to wiping all of BrowseableRoot (legacy whole-tree
|
||||
// behavior).
|
||||
manifest := d.modulesMustGet(rec.ModuleID)
|
||||
// Resolve the SAME root the backup wrote against. buildBackupTarCmd
|
||||
// archives relative to manifest.Backup.Root when set (e.g. 7DTD's
|
||||
// /game-saves, where world data actually lives — distinct from the
|
||||
// /game install volume that is BrowseableRoot). Restore MUST extract
|
||||
// into that same root or the tarball lands in the wrong volume and the
|
||||
// live server never sees it (silent data loss: restore "succeeds" but
|
||||
// players' bases are gone). Mirror the backup's resolution exactly.
|
||||
restoreRoot := rec.BrowseableRoot
|
||||
if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
|
||||
restoreRoot = manifest.Backup.Root
|
||||
}
|
||||
var wipeCmd string
|
||||
if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
|
||||
// Wipe only the manifest-declared include paths so we don't
|
||||
// nuke unrelated install files. Each path is rm -rf'd; missing
|
||||
// ones are skipped silently (first-run case).
|
||||
var parts []string
|
||||
for _, p := range manifest.Backup.Include {
|
||||
clean := strings.TrimPrefix(strings.TrimPrefix(p, "./"), "/")
|
||||
parts = append(parts, fmt.Sprintf("rm -rf %s/%s", shellEscape(restoreRoot), shellEscape(clean)))
|
||||
}
|
||||
wipeCmd = strings.Join(parts, " ; ") + " 2>/dev/null"
|
||||
log.Info("restore: wiping manifest-declared paths only", "root", restoreRoot, "include_count", len(manifest.Backup.Include))
|
||||
} else {
|
||||
wipeCmd = fmt.Sprintf("rm -rf %q/* %q/.[!.]* 2>/dev/null", restoreRoot, restoreRoot)
|
||||
log.Warn("restore: wiping entire root (whole-tree backup)", "root", restoreRoot)
|
||||
}
|
||||
extractCmd := fmt.Sprintf(
|
||||
"%s ; tar xzf /backup/%s -C %q && du -sb %q | cut -f1",
|
||||
wipeCmd,
|
||||
shellEscape(fileName),
|
||||
restoreRoot,
|
||||
restoreRoot,
|
||||
)
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: req.InstanceId + "-restore",
|
||||
IsSidecar: true,
|
||||
Image: backupImage,
|
||||
Entrypoint: []string{"sh"},
|
||||
Command: []string{"-c", extractCmd},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
d.sendRestoreResult(corrID, 0, "create sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
d.sendRestoreResult(corrID, 0, "start sidecar: "+err.Error())
|
||||
return
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID, Stream: "restore",
|
||||
At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
d.sendRestoreResult(corrID, 0, "wait: "+err.Error())
|
||||
return
|
||||
}
|
||||
if exitCode != 0 {
|
||||
d.sendRestoreResult(corrID, 0, fmt.Sprintf("sidecar exited %d", exitCode))
|
||||
return
|
||||
}
|
||||
log.Info("restore complete")
|
||||
d.sendRestoreResult(corrID, info.Size(), "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendRestoreResult(corrID string, bytesRestored int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_RestoreResult{
|
||||
RestoreResult: &panelv1.RestoreResult{BytesRestored: bytesRestored, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
func randHex(n int) string {
|
||||
b := make([]byte, n)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// shellEscape wraps a filename in single quotes for safe inclusion in an
|
||||
// sh -c command. Assumes filename never contains a single-quote (ours
|
||||
// never do — timestamps + bkp_<hex>).
|
||||
func shellEscape(s string) string { return "'" + s + "'" }
|
||||
|
||||
// modulesMustGet returns the dispatcher's manifest for the given id, or
|
||||
// nil if it's gone missing (shouldn't happen for a live instance).
|
||||
func (d *Dispatcher) modulesMustGet(id string) *modulepkgManifest {
|
||||
m, _ := d.modules.Get(id)
|
||||
return m
|
||||
}
|
||||
|
||||
// Keep errors imported for future error-wrapping.
|
||||
var _ = errors.New
|
||||
@@ -0,0 +1,198 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Backup browse — list / read entries inside a backup tarball without
|
||||
// extracting it. The agent has direct host access to the .tar.gz under
|
||||
// $BACKUP_DIR, so we don't need a sidecar; standard library archive/tar
|
||||
// + compress/gzip handles both the list and the single-file extract.
|
||||
//
|
||||
// Sized for safety: BackupReadFile caps the in-memory buffer per-call
|
||||
// (default 8 MiB) so a malicious or corrupt tar entry can't OOM the
|
||||
// agent. Truncation is reported back to the caller so the UI can show
|
||||
// "(truncated, download to see full file)".
|
||||
|
||||
const (
|
||||
defaultBackupReadCap = 8 << 20 // 8 MiB
|
||||
maxBackupReadCap = 64 << 20 // 64 MiB hard ceiling regardless of caller request
|
||||
binarySniffLen = 8 << 10 // 8 KiB
|
||||
)
|
||||
|
||||
func (d *Dispatcher) handleBackupListEntries(corrID string, req *panelv1.BackupListEntriesRequest) {
|
||||
d.log.Info("backup list entries: start", "correlation_id", corrID, "backup_path", req.BackupPath, "instance_id", req.InstanceId)
|
||||
if req.BackupPath == "" {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "backup_path required")
|
||||
return
|
||||
}
|
||||
f, err := os.Open(req.BackupPath)
|
||||
if err != nil {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "open backup: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "gzip: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(gz)
|
||||
|
||||
var entries []*panelv1.BackupTarEntry
|
||||
var totalBytes int64
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
d.sendBackupListEntriesResult(corrID, nil, 0, "tar header: "+err.Error())
|
||||
return
|
||||
}
|
||||
isDir := hdr.Typeflag == tar.TypeDir || strings.HasSuffix(hdr.Name, "/")
|
||||
clean := strings.TrimPrefix(hdr.Name, "./")
|
||||
clean = strings.TrimPrefix(clean, "/")
|
||||
entry := &panelv1.BackupTarEntry{
|
||||
Path: clean,
|
||||
Size: hdr.Size,
|
||||
IsDir: isDir,
|
||||
Mode: hdr.FileInfo().Mode().String(),
|
||||
ModTime: timestamppb.New(hdr.ModTime),
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
if !isDir {
|
||||
totalBytes += hdr.Size
|
||||
}
|
||||
}
|
||||
d.log.Info("backup list entries: ok", "correlation_id", corrID, "count", len(entries), "total_bytes", totalBytes)
|
||||
d.sendBackupListEntriesResult(corrID, entries, totalBytes, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) handleBackupReadFile(corrID string, req *panelv1.BackupReadFileRequest) {
|
||||
if req.BackupPath == "" {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "backup_path required")
|
||||
return
|
||||
}
|
||||
if req.EntryPath == "" {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "entry_path required")
|
||||
return
|
||||
}
|
||||
maxRead := req.MaxBytes
|
||||
if maxRead <= 0 {
|
||||
maxRead = defaultBackupReadCap
|
||||
}
|
||||
if maxRead > maxBackupReadCap {
|
||||
maxRead = maxBackupReadCap
|
||||
}
|
||||
|
||||
f, err := os.Open(req.BackupPath)
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "open backup: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "gzip: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(gz)
|
||||
|
||||
// Match by exact path; tarballs sometimes prefix entries with "./" so
|
||||
// also accept that variant.
|
||||
want := strings.TrimPrefix(req.EntryPath, "/")
|
||||
wantAlt := "./" + want
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, fmt.Sprintf("entry %q not found in backup", req.EntryPath))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, 0, false, false, "tar header: "+err.Error())
|
||||
return
|
||||
}
|
||||
if hdr.Name != want && hdr.Name != wantAlt {
|
||||
continue
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeDir || strings.HasSuffix(hdr.Name, "/") {
|
||||
d.sendBackupReadFileResult(corrID, nil, hdr.Size, false, false, "entry is a directory")
|
||||
return
|
||||
}
|
||||
// Read up to maxRead+1 to detect truncation cleanly.
|
||||
buf := &bytes.Buffer{}
|
||||
_, err = io.Copy(buf, io.LimitReader(tr, maxRead+1))
|
||||
if err != nil {
|
||||
d.sendBackupReadFileResult(corrID, nil, hdr.Size, false, false, "read entry: "+err.Error())
|
||||
return
|
||||
}
|
||||
truncated := false
|
||||
content := buf.Bytes()
|
||||
if int64(len(content)) > maxRead {
|
||||
content = content[:maxRead]
|
||||
truncated = true
|
||||
}
|
||||
isBinary := looksBinary(content)
|
||||
d.sendBackupReadFileResult(corrID, content, hdr.Size, truncated, isBinary, "")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// looksBinary returns true if the first binarySniffLen bytes contain a
|
||||
// NUL byte. Standard heuristic — text files don't contain NULs, binaries
|
||||
// almost always do (executables, images, compressed save formats).
|
||||
func looksBinary(b []byte) bool {
|
||||
limit := binarySniffLen
|
||||
if len(b) < limit {
|
||||
limit = len(b)
|
||||
}
|
||||
for i := 0; i < limit; i++ {
|
||||
if b[i] == 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendBackupListEntriesResult(corrID string, entries []*panelv1.BackupTarEntry, totalBytes int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_BackupListEntriesResult{
|
||||
BackupListEntriesResult: &panelv1.BackupListEntriesResult{
|
||||
Entries: entries,
|
||||
TotalBytes: totalBytes,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendBackupReadFileResult(corrID string, content []byte, size int64, truncated, isBinary bool, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_BackupReadFileResult{
|
||||
BackupReadFileResult: &panelv1.BackupReadFileResult{
|
||||
Content: content,
|
||||
Size: size,
|
||||
Truncated: truncated,
|
||||
IsBinary: isBinary,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
"github.com/dbledeez/panel/agent/internal/updater"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// installBaseMods runs a SteamCMD validate (if the module has a steamcmd
|
||||
// update provider) followed by an alpine sidecar that downloads + installs
|
||||
// the third-party 7DTD base-mod set:
|
||||
//
|
||||
// - Allocs Server Fixes — tarball from illy.bz (canonical upstream).
|
||||
// Contains Allocs_CommandExtensions + Allocs_WebAndMapRendering +
|
||||
// Alloc_XmlPatch, all under a Mods/ prefix when extracted to /game.
|
||||
// - PrismaCore — latest GitHub release .zip from Prisma501/PrismaCore.
|
||||
//
|
||||
// SteamCMD validate step (new):
|
||||
// - If the module declares at least one update provider with kind="steamcmd",
|
||||
// the first such provider is used to run `app_update <app_id> validate`
|
||||
// with anonymous login BEFORE the alpine sidecar runs. This re-downloads
|
||||
// or restores stock TFP mods (TFP_*, 0_TFP_*, xMarkers) that ship with
|
||||
// the dedicated server install.
|
||||
// - If the SteamCMD validate fails, the function returns an error and does
|
||||
// NOT proceed to the alpine step.
|
||||
// - If there is no steamcmd provider (e.g., Conan Exiles), the validate is
|
||||
// skipped and the function goes straight to the alpine path.
|
||||
//
|
||||
// What the alpine step does NOT touch (and does not need to):
|
||||
// - TFP_* / 0_TFP_* — shipped by SteamCMD as part of the dedicated
|
||||
// server install. TFP_Harmony is the runtime patcher; wiping it
|
||||
// would brick the engine. Always present in /game/Mods/ after a
|
||||
// fresh `app_update 294420`.
|
||||
// - xMarkers — also ships with the SteamCMD 7DTD install per the
|
||||
// operator (2026-05-04). Already in /game/Mods/ after install.
|
||||
// - Custom mods (RefugeBot, AGF-VP-*, AsylumRoboticInbox, anything not
|
||||
// in the recognized base-mod name set). Those are operator territory.
|
||||
//
|
||||
// Behavior on existing installs: any directory with a name that matches
|
||||
// the incoming mod gets renamed to `<name>.bak-<UTCstamp>` first so a
|
||||
// re-run is reversible. Direct port of the safety pattern in
|
||||
// refugebotserver's DependencyInstaller.cs.
|
||||
//
|
||||
// Caller responsibility: the instance container MUST be stopped before
|
||||
// calling this — the game holds DLLs in /game/Mods open while running,
|
||||
// and rename-on-extract will fail or corrupt under that.
|
||||
func (d *Dispatcher) installBaseMods(ctx context.Context, instanceID string, manifest *modulepkg.Manifest, dataPath string) error {
|
||||
if manifest == nil || len(manifest.UpdateProviders) == 0 {
|
||||
return errors.New("install-base-mods: module has no update provider")
|
||||
}
|
||||
installPath := manifest.UpdateProviders[0].InstallPath
|
||||
if installPath == "" {
|
||||
installPath = manifest.Runtime.Docker.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return errors.New("install-base-mods: install_path missing on module")
|
||||
}
|
||||
volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
|
||||
gameVol := volumeNameForPath(volumes, installPath)
|
||||
if gameVol == "" {
|
||||
return fmt.Errorf("install-base-mods: install_path %q not in module volumes", installPath)
|
||||
}
|
||||
|
||||
// ── SteamCMD validate step ──────────────────────────────────────────
|
||||
// If the module has a steamcmd update provider, run app_update validate
|
||||
// with anonymous login FIRST to re-download/restore stock TFP mods.
|
||||
// This ensures TFP_*, 0_TFP_*, and xMarkers are present before the
|
||||
// alpine sidecar installs third-party base mods.
|
||||
var steamcmdSpec *modulepkg.UpdateProvider
|
||||
for i := range manifest.UpdateProviders {
|
||||
if manifest.UpdateProviders[i].Kind == "steamcmd" {
|
||||
steamcmdSpec = &manifest.UpdateProviders[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if steamcmdSpec != nil {
|
||||
d.log.Info("install-base-mods: steamcmd validate starting", "instance_id", instanceID, "app_id", steamcmdSpec.AppID)
|
||||
prov, err := updater.Get("steamcmd")
|
||||
if err != nil {
|
||||
return fmt.Errorf("install-base-mods: get steamcmd provider: %w", err)
|
||||
}
|
||||
// Build a log sink that streams through the same "install-base-mods log" logger.
|
||||
sink := func(line string) {
|
||||
d.log.Info("install-base-mods log", "instance_id", instanceID, "line", line)
|
||||
}
|
||||
uc := &updater.Context{
|
||||
InstanceID: instanceID,
|
||||
Manifest: manifest,
|
||||
BrowseableRoot: installPath,
|
||||
DataPath: dataPath,
|
||||
Runtime: d.runtime,
|
||||
Log: sink,
|
||||
SteamUsername: "", // anonymous login
|
||||
SteamPassword: "", // anonymous login
|
||||
}
|
||||
if err := prov.Update(ctx, uc, steamcmdSpec); err != nil {
|
||||
return fmt.Errorf("basemods: steamcmd validate failed: %w", err)
|
||||
}
|
||||
d.log.Info("install-base-mods: steamcmd validate succeeded", "instance_id", instanceID)
|
||||
}
|
||||
|
||||
// alpine:3.19 is a 7 MB image with apk available. apk add fetches
|
||||
// curl + unzip + tar + jq in ~5s on a warm cache. Cheaper than
|
||||
// shipping our own base image and good enough for occasional runs.
|
||||
script := `set -e
|
||||
apk add --no-cache --quiet curl tar unzip jq ca-certificates >/dev/null
|
||||
STAMP=$(date -u +%Y%m%d%H%M%S)
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
backup_if_exists() {
|
||||
for d in "$@"; do
|
||||
if [ -d "$d" ] && [ ! -L "$d" ]; then
|
||||
mv "$d" "$d.bak-$STAMP" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ── Allocs Server Fixes ───────────────────────────────────────────────
|
||||
echo "[basemods] downloading Allocs from illy.bz"
|
||||
curl -fsSL -o "$TMP/allocs.tgz" "https://illy.bz/fi/7dtd/server_fixes.tar.gz"
|
||||
# Magic-byte check: gzip starts with 0x1f 0x8b — protects against
|
||||
# CDN-served HTML 404 pages landing as a "valid" archive.
|
||||
head -c 2 "$TMP/allocs.tgz" | od -An -tx1 | tr -d ' \n' | grep -q '^1f8b' \
|
||||
|| { echo "[basemods] Allocs download wasn't gzip — aborting"; exit 1; }
|
||||
backup_if_exists /game/Mods/Allocs_CommandExtensions /game/Mods/Allocs_WebAndMapRendering /game/Mods/Alloc_XmlPatch
|
||||
tar -xzf "$TMP/allocs.tgz" -C /game
|
||||
echo "[basemods] Allocs installed"
|
||||
|
||||
# ── PrismaCore (latest GitHub release zip) ─────────────────────────────
|
||||
echo "[basemods] resolving latest PrismaCore release"
|
||||
PRISMA_JSON="$TMP/prisma.json"
|
||||
curl -fsSL -A "panel-basemods/1.0" -H "Accept: application/vnd.github+json" \
|
||||
-o "$PRISMA_JSON" "https://api.github.com/repos/Prisma501/PrismaCore/releases/latest"
|
||||
PRISMA_TAG=$(jq -r '.tag_name // empty' "$PRISMA_JSON")
|
||||
PRISMA_URL=$(jq -r '.assets[]? | select(.name | test("\\.zip$"; "i")) | .browser_download_url' "$PRISMA_JSON" | head -1)
|
||||
if [ -z "$PRISMA_URL" ]; then
|
||||
echo "[basemods] no PrismaCore .zip asset — skipping"
|
||||
else
|
||||
echo "[basemods] downloading PrismaCore $PRISMA_TAG from $PRISMA_URL"
|
||||
curl -fsSL -A "panel-basemods/1.0" -o "$TMP/prisma.zip" "$PRISMA_URL"
|
||||
head -c 2 "$TMP/prisma.zip" | od -An -tx1 | tr -d ' \n' | grep -q '^504b' \
|
||||
|| { echo "[basemods] PrismaCore download wasn't a zip — aborting"; exit 1; }
|
||||
for d in /game/Mods/Prisma*; do backup_if_exists "$d"; done
|
||||
unzip -o -q "$TMP/prisma.zip" -d /game/Mods/
|
||||
echo "[basemods] PrismaCore installed"
|
||||
fi
|
||||
|
||||
# Note: TFP_* and xMarkers are shipped as part of the SteamCMD dedicated
|
||||
# server install (app 294420). They're already in /game/Mods/ after a
|
||||
# fresh install and we don't touch them here — wiping TFP_Harmony would
|
||||
# brick the engine.
|
||||
echo BASEMODS_OK
|
||||
`
|
||||
|
||||
sidecarID := fmt.Sprintf("%s-basemods", instanceID)
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: "alpine:3.19",
|
||||
Command: []string{"sh", "-c", script},
|
||||
Volumes: []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: gameVol, ContainerPath: installPath},
|
||||
},
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
d.log.Info("install-base-mods: starting sidecar", "instance_id", instanceID, "vol", gameVol)
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("install-base-mods: create: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return fmt.Errorf("install-base-mods: start: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.log.Info("install-base-mods log", "instance_id", instanceID, "line", line)
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return fmt.Errorf("install-base-mods: wait: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return fmt.Errorf("install-base-mods: sidecar exited %d", exitCode)
|
||||
}
|
||||
d.log.Info("install-base-mods: success", "instance_id", instanceID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Cluster player-save snapshots.
|
||||
//
|
||||
// A 7DTD cluster shares ONE Player/ folder across every member server (each
|
||||
// member's active-world Player dir is a symlink to the shared /cluster/Player
|
||||
// bind). That shared dir holds every character's .ttp (inventory, skills,
|
||||
// position, quests) — the single most irreplaceable per-player state. But it
|
||||
// is NOT captured by the panel tar backup: the backup tars /game-saves, where
|
||||
// the active Player dir is a symlink pointing OUT to /cluster, so tar stores
|
||||
// the dangling symlink, not the contents. Region Medic only heals world
|
||||
// regions, not player files. Net result before this: one corrupt write to a
|
||||
// .ttp and that character was gone with no restore path.
|
||||
//
|
||||
// Fix (mirrors the region-medic rolling-snapshot pattern): on every 7DTD
|
||||
// start, if the instance is clustered, copy the shared Player dir to a fresh
|
||||
// rolling slot keyed by cluster id and prune to keep N. All members write to
|
||||
// the same per-cluster rolling history, so a member's 4×/day restart cadence
|
||||
// gives a dense set of restore points. Best-effort: never blocks the start.
|
||||
|
||||
func clusterPlayerRollingRoot() string {
|
||||
return medicEnvOr("PANEL_CLUSTER_PLAYER_ROLLING_DIR", "/home/refuge/cluster-player-rolling")
|
||||
}
|
||||
|
||||
func clusterPlayerKeep() int {
|
||||
if v := os.Getenv("PANEL_CLUSTER_PLAYER_KEEP"); v != "" {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 12
|
||||
}
|
||||
|
||||
// snapshotClusterPlayerOnStart resolves the instance's /cluster bind to its
|
||||
// host path, and if it's a real shared bind holding player data, snapshots
|
||||
// <bind>/Player into a rolling slot keyed by the cluster id (= the bind dir's
|
||||
// basename, e.g. cl_37e3a7f72cdc). No-op for standalone servers (whose
|
||||
// /cluster is an empty per-instance named volume, not a bind).
|
||||
func (d *Dispatcher) snapshotClusterPlayerOnStart(ctx context.Context, rec *instanceRecord) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
log := d.log.With("instance_id", rec.InstanceID, "cluster_player_snapshot", true)
|
||||
|
||||
mounts, err := d.runtime.ContainerMounts(ctx, "panel-"+rec.InstanceID)
|
||||
if err != nil {
|
||||
log.Warn("cluster-player snapshot: inspect mounts failed (skipping)", "err", err)
|
||||
return
|
||||
}
|
||||
var bind string
|
||||
for _, m := range mounts {
|
||||
if m.Destination == "/cluster" && m.Type == "bind" && m.Source != "" {
|
||||
bind = m.Source
|
||||
break
|
||||
}
|
||||
}
|
||||
if bind == "" {
|
||||
return // standalone server (named-volume /cluster) — nothing shared to snapshot
|
||||
}
|
||||
playerDir := filepath.Join(bind, "Player")
|
||||
if st, err := os.Stat(playerDir); err != nil || !st.IsDir() {
|
||||
return // cluster joined but no Player dir yet (first boot)
|
||||
}
|
||||
if empty, _ := dirIsEmpty(playerDir); empty {
|
||||
return
|
||||
}
|
||||
|
||||
clusterID := filepath.Base(bind)
|
||||
destRoot := filepath.Join(clusterPlayerRollingRoot(), clusterID)
|
||||
if err := os.MkdirAll(destRoot, 0o755); err != nil {
|
||||
log.Warn("cluster-player snapshot: mkdir rolling root failed", "err", err)
|
||||
return
|
||||
}
|
||||
slot := filepath.Join(destRoot, "slot-"+strconv.FormatInt(time.Now().Unix(), 10))
|
||||
dst := filepath.Join(slot, "Player")
|
||||
if err := os.MkdirAll(slot, 0o755); err != nil {
|
||||
log.Warn("cluster-player snapshot: mkdir slot failed", "err", err)
|
||||
return
|
||||
}
|
||||
// cp -a preserves perms/acls/timestamps and the dir tree (player subdirs
|
||||
// of .7rm map files). figaro is Linux; cp is always present. Best-effort.
|
||||
cmd := exec.CommandContext(ctx, "cp", "-a", playerDir+"/.", dst)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
log.Warn("cluster-player snapshot: copy failed", "err", err, "out", strings.TrimSpace(string(out)))
|
||||
_ = os.RemoveAll(slot) // don't leave a half-copied slot that prune would keep
|
||||
return
|
||||
}
|
||||
keep := clusterPlayerKeep()
|
||||
pruned := pruneRollingSlots(destRoot, keep)
|
||||
log.Info("cluster-player snapshot taken", "cluster", clusterID, "slot", filepath.Base(slot), "keep", keep, "pruned", pruned)
|
||||
}
|
||||
|
||||
// dirIsEmpty reports whether dir has no entries.
|
||||
func dirIsEmpty(dir string) (bool, error) {
|
||||
f, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer f.Close()
|
||||
names, err := f.Readdirnames(1)
|
||||
if err != nil {
|
||||
// io.EOF → empty.
|
||||
return len(names) == 0, nil
|
||||
}
|
||||
return len(names) == 0, nil
|
||||
}
|
||||
|
||||
// pruneRollingSlots keeps the newest `keep` slot-* dirs under root (by name,
|
||||
// which embeds a unix timestamp so lexical == chronological) and removes the
|
||||
// rest. Returns how many it deleted.
|
||||
func pruneRollingSlots(root string, keep int) int {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var slots []string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && strings.HasPrefix(e.Name(), "slot-") {
|
||||
slots = append(slots, e.Name())
|
||||
}
|
||||
}
|
||||
if len(slots) <= keep {
|
||||
return 0
|
||||
}
|
||||
sort.Strings(slots) // oldest first
|
||||
removed := 0
|
||||
for _, name := range slots[:len(slots)-keep] {
|
||||
if err := os.RemoveAll(filepath.Join(root, name)); err == nil {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package dispatch
|
||||
|
||||
// DayZ mod install/uninstall + FsSymlink handlers. The controller drives
|
||||
// the high-level workflow (SteamCMD workshop_download runs controller-side,
|
||||
// writing to the shared panel-dayz-workshop volume); the agent's job is to
|
||||
// wire the downloaded content into the instance's /game tree:
|
||||
//
|
||||
// 1. Create /game/<folder_name> as a symlink to
|
||||
// /game/steamapps/workshop/content/221100/<workshop_id>
|
||||
// 2. Copy every .bikey from that mod's keys/ (or Keys/) subfolder into
|
||||
// /game/keys/ so BattlEye signs mod content on boot.
|
||||
// 3. On uninstall, remove the symlink + every bikey whose content hash
|
||||
// matches a bikey in the workshop mod's keys dir (so we don't remove
|
||||
// keys that belong to OTHER installed mods of the same vendor).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// ---- FsSymlink ----
|
||||
//
|
||||
// The symlink is created inside the instance's container namespace so that
|
||||
// any `/game/@Mod` symlink resolves correctly from the DayZServer binary's
|
||||
// perspective.
|
||||
|
||||
func (d *Dispatcher) handleFsSymlink(corrID string, req *panelv1.FsSymlinkRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsSymlinkResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsSymlinkResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
// `ln -sfn` replaces an existing symlink atomically. We require
|
||||
// abs paths for both target and link_path and refuse anything outside
|
||||
// the configured browseable roots for link_path (the symlink itself;
|
||||
// target can point outside for workshop shares).
|
||||
if !strings.HasPrefix(req.LinkPath, "/") || !strings.HasPrefix(req.Target, "/") {
|
||||
d.sendFsSymlinkResult(corrID, "target and link_path must be absolute")
|
||||
return
|
||||
}
|
||||
if _, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.LinkPath); err != nil {
|
||||
d.sendFsSymlinkResult(corrID, "link_path outside browseable roots: "+err.Error())
|
||||
return
|
||||
}
|
||||
// `ln` errors with "No such file or directory" if the parent doesn't
|
||||
// exist yet — common on first mod install for modules whose entrypoint
|
||||
// hasn't lazily created the Mods/ folder yet (Conan Exiles). `mkdir -p`
|
||||
// the parent before linking so the symlink-then-game-restart flow works
|
||||
// even on a fresh recreate. Idempotent: mkdir on existing dir is a
|
||||
// no-op, and the safeJoinAny check above already proved link_path is
|
||||
// inside a declared root so we're not creating dirs in arbitrary spots.
|
||||
parentDir := path.Dir(req.LinkPath)
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("mkdir -p %q && ln -sfn %q %q", parentDir, req.Target, req.LinkPath)})
|
||||
if err != nil {
|
||||
d.sendFsSymlinkResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendFsSymlinkResult(corrID, fmt.Sprintf("ln exited %d: %s", code, strings.TrimSpace(string(stderr))))
|
||||
return
|
||||
}
|
||||
d.sendFsSymlinkResult(corrID, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsSymlinkResult(corrID, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsSymlinkResult{
|
||||
FsSymlinkResult: &panelv1.FsSymlinkResult{Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- DayzModInstall ----
|
||||
|
||||
func (d *Dispatcher) handleDayzModInstall(corrID string, req *panelv1.DayzModInstallRequest) {
|
||||
d.log.Info("dayz mod install", "instance_id", req.InstanceId, "workshop_id", req.WorkshopId, "folder", req.FolderName)
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, err.Error(), nil, "")
|
||||
return
|
||||
}
|
||||
if req.WorkshopId == "" || req.FolderName == "" {
|
||||
d.sendDayzModInstallResult(corrID, "workshop_id and folder_name are required", nil, "")
|
||||
return
|
||||
}
|
||||
// Normalize folder name: must start with '@', no slashes, no whitespace.
|
||||
folder := strings.TrimSpace(req.FolderName)
|
||||
if !strings.HasPrefix(folder, "@") {
|
||||
folder = "@" + folder
|
||||
}
|
||||
if strings.ContainsAny(folder, "/\\ \t\n") {
|
||||
d.sendDayzModInstallResult(corrID, "folder_name must not contain slashes or whitespace", nil, "")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, err.Error(), nil, "")
|
||||
return
|
||||
}
|
||||
|
||||
modPath := "/game/steamapps/workshop/content/221100/" + req.WorkshopId
|
||||
linkPath := "/game/" + folder
|
||||
|
||||
// 1. Verify workshop content exists.
|
||||
_, _, code, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("test -d %q", modPath)})
|
||||
if code != 0 {
|
||||
d.sendDayzModInstallResult(corrID, "workshop content not found at "+modPath+" — did SteamCMD download succeed?", nil, modPath)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Create symlink.
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("ln -sfn %q %q", modPath, linkPath)})
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, "symlink: "+err.Error(), nil, modPath)
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendDayzModInstallResult(corrID, "symlink failed: "+strings.TrimSpace(string(stderr)), nil, modPath)
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Copy bikeys. DayZ mods ship them under keys/ OR Keys/ inside the
|
||||
// workshop tree. Copy every .bikey we find there into /game/keys/.
|
||||
// Keep a manifest of copied filenames so uninstall can remove them.
|
||||
// Use `find` + `cp` for portability; busybox and GNU both understand.
|
||||
stdout, stderr2, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf(`mkdir -p /game/keys && \
|
||||
for src in %q/keys %q/Keys; do \
|
||||
[ -d "$src" ] || continue; \
|
||||
for f in "$src"/*.bikey "$src"/*.BIKEY; do \
|
||||
[ -f "$f" ] || continue; \
|
||||
base=$(basename "$f"); \
|
||||
cp -f "$f" /game/keys/"$base"; \
|
||||
printf '%%s\n' "$base"; \
|
||||
done; \
|
||||
done`, modPath, modPath)})
|
||||
if err != nil {
|
||||
d.sendDayzModInstallResult(corrID, "copy bikeys: "+err.Error(), nil, modPath)
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendDayzModInstallResult(corrID, "copy bikeys failed: "+strings.TrimSpace(string(stderr2)), nil, modPath)
|
||||
return
|
||||
}
|
||||
var bikeys []string
|
||||
for _, ln := range strings.Split(string(stdout), "\n") {
|
||||
ln = strings.TrimSpace(strings.TrimPrefix(ln, `\n`))
|
||||
if ln == "" {
|
||||
continue
|
||||
}
|
||||
bikeys = append(bikeys, ln)
|
||||
}
|
||||
d.sendDayzModInstallResult(corrID, "", bikeys, modPath)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendDayzModInstallResult(corrID, errMsg string, bikeys []string, resolvedPath string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_DayzModInstallResult{
|
||||
DayzModInstallResult: &panelv1.DayzModInstallResult{
|
||||
Error: errMsg,
|
||||
BikeysCopied: bikeys,
|
||||
ResolvedModPath: resolvedPath,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- DayzModUninstall ----
|
||||
|
||||
func (d *Dispatcher) handleDayzModUninstall(corrID string, req *panelv1.DayzModUninstallRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendDayzModUninstallResult(corrID, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
folder := strings.TrimSpace(req.FolderName)
|
||||
if !strings.HasPrefix(folder, "@") || strings.ContainsAny(folder, "/\\ \t\n") {
|
||||
d.sendDayzModUninstallResult(corrID, "invalid folder_name", nil)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendDayzModUninstallResult(corrID, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
linkPath := "/game/" + folder
|
||||
|
||||
// Resolve the symlink to find the workshop directory.
|
||||
stdout, _, code, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("readlink -f %q 2>/dev/null || true", linkPath)})
|
||||
modPath := strings.TrimSpace(string(stdout))
|
||||
if code != 0 || modPath == "" {
|
||||
// Symlink is already gone or broken — just drop stale /game/keys entries.
|
||||
modPath = ""
|
||||
}
|
||||
|
||||
// Remove bikeys whose *name* matches a bikey in the workshop mod's keys.
|
||||
// Matching by name (not hash) is what DayZ tooling does — the server
|
||||
// looks up bikeys by exact basename, and mods typically ship unique
|
||||
// key names.
|
||||
var removed []string
|
||||
if modPath != "" {
|
||||
out, _, _, _ := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf(`for d in %q/keys %q/Keys; do \
|
||||
[ -d "$d" ] || continue; \
|
||||
for f in "$d"/*.bikey "$d"/*.BIKEY; do \
|
||||
[ -f "$f" ] || continue; \
|
||||
base=$(basename "$f"); \
|
||||
if [ -f "/game/keys/$base" ]; then \
|
||||
rm -f "/game/keys/$base"; \
|
||||
echo "$base"; \
|
||||
fi; \
|
||||
done; \
|
||||
done`, modPath, modPath)})
|
||||
for _, ln := range strings.Split(string(out), "\n") {
|
||||
ln = strings.TrimSpace(ln)
|
||||
if ln == "" {
|
||||
continue
|
||||
}
|
||||
removed = append(removed, ln)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the symlink itself.
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
fmt.Sprintf("rm -f %q", linkPath)})
|
||||
if err != nil {
|
||||
d.sendDayzModUninstallResult(corrID, "rm symlink: "+err.Error(), removed)
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendDayzModUninstallResult(corrID, "rm symlink failed: "+strings.TrimSpace(string(stderr)), removed)
|
||||
return
|
||||
}
|
||||
d.sendDayzModUninstallResult(corrID, "", removed)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendDayzModUninstallResult(corrID, errMsg string, bikeys []string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_DayzModUninstallResult{
|
||||
DayzModUninstallResult: &panelv1.DayzModUninstallResult{
|
||||
Error: errMsg,
|
||||
BikeysRemoved: bikeys,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Auto-provision a freshly-generated clustered 7DTD world (zero-touch create),
|
||||
// AND keep a visible "setting up" status until it is actually done so an operator
|
||||
// who refreshes the dashboard sees it is not ready yet (don't restart it).
|
||||
//
|
||||
// A new cluster world bakes its decorations (decoration.7dt / multiblocks.7dt)
|
||||
// with the engine's NATIVE block ids at generation. The entrypoint stamps the
|
||||
// cluster CANONICAL .nim over the world's table every boot so shared inventory
|
||||
// (.ttp) decodes uniformly — but native-baked decorations then resolve to a null
|
||||
// Block under canonical -> DecoManager NRE -> the world loads but never reaches
|
||||
// GameStartDone ("online but unjoinable"). The entrypoint fixes this by remapping
|
||||
// the decorations native->canonical on the FIRST boot it sees them in the native
|
||||
// band — which can only be a boot AFTER the gen boot (the running server writes
|
||||
// decoration.7dt; the entrypoint can't see it until next start).
|
||||
//
|
||||
// This watcher (a) closes that 2-phase gap so create is zero-touch — it bounces
|
||||
// the container ONCE so the entrypoint's remap runs — and (b) holds a persistent
|
||||
// "Finishing cluster setup…" status detail (the dashboard renders it as a "setting
|
||||
// up" pill instead of a green "running" one) until the world carries the
|
||||
// .canon-provisioned marker, so a page refresh always shows it isn't done yet.
|
||||
//
|
||||
// Strict no-op for non-cluster servers (no /cluster bind -> the watcher exits) and
|
||||
// already-canon worlds (deco in canon band -> "CANON" on the first probe -> the
|
||||
// indicator clears immediately). Single-flight; the entrypoint is the backstop if
|
||||
// the watcher ever misses (the world still provisions on its next restart).
|
||||
// Mirrors the hang-guard single-flight container-restart pattern (hangguard.go).
|
||||
|
||||
const (
|
||||
provisionPollInterval = 20 * time.Second
|
||||
provisionPollTimeout = 15 * time.Minute
|
||||
provisionExecTimeout = 20 * time.Second
|
||||
provisionRestartGrace = 30 * time.Second
|
||||
)
|
||||
|
||||
// provisionDetail is the persistent status detail shown while a clustered world is
|
||||
// still aligning to the cluster canonical. The dashboard matches this text to paint
|
||||
// a "setting up" pill, so a refresh shows the server isn't ready yet.
|
||||
const provisionDetail = "Finishing cluster setup — aligning decorations to the cluster (it auto-restarts once); please wait, don't restart it."
|
||||
|
||||
// provisionProbeScript runs inside the container and classifies the active cluster
|
||||
// world's provisioning state. Emits exactly one token on stdout:
|
||||
//
|
||||
// NOWORLD no recorded save dir (RWG world not yet picked in the Cluster tab)
|
||||
// PROVISIONED .canon-provisioned marker present (already aligned)
|
||||
// GENERATING no main.ttw yet (world still generating)
|
||||
// CANON decorations already in the canon band (>=24000) -> nothing to do
|
||||
// ATTEMPTED already bounced once and still not aligned -> stop (avoid a loop)
|
||||
// NEEDS_ALIGN gen done; decorations native-banded OR not baked yet -> bounce once
|
||||
//
|
||||
// Gated on main.ttw (= gen finished) not on decoration.7dt existing, because some
|
||||
// worlds only write decoration.7dt on the first save — the bounce's graceful save
|
||||
// writes it (native), then the entrypoint remaps it on the align boot. First deco
|
||||
// block-id = u16 @ offset 17 (hdr 5 + record id-offset 12), masked to 15 bits.
|
||||
const provisionProbeScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
|
||||
`[ -z "$SB" ] && { echo NOWORLD; exit 0; }; ` +
|
||||
`[ -f "$SB/.canon-provisioned" ] && { echo PROVISIONED; exit 0; }; ` +
|
||||
`[ -f "$SB/main.ttw" ] || { echo GENERATING; exit 0; }; ` +
|
||||
`D="$SB/decoration.7dt"; ` +
|
||||
`if [ -f "$D" ]; then raw="$(od -An -tu2 -j17 -N2 "$D" 2>/dev/null | tr -d ' ')"; id=$(( ${raw:-0} & 0x7FFF )); [ "$id" -ge 24000 ] && { echo CANON; exit 0; }; fi; ` +
|
||||
`[ -f "$SB/.panel-provision-attempted" ] && { echo ATTEMPTED; exit 0; }; ` +
|
||||
`echo NEEDS_ALIGN`
|
||||
|
||||
// provisionAttemptScript drops a durable one-bounce guard so a world that doesn't
|
||||
// align after its single bounce is never bounced again (the entrypoint remains the
|
||||
// backstop on ordinary restarts).
|
||||
const provisionAttemptScript = `SB="$(cat /game-saves/.panel-save-base 2>/dev/null)"; ` +
|
||||
`[ -n "$SB" ] && touch "$SB/.panel-provision-attempted" 2>/dev/null; true`
|
||||
|
||||
// autoProvisionClusterWorldOnStart starts (single-flight) the provision watcher for
|
||||
// a just-started 7DTD instance. Returns immediately; the watch runs off a background
|
||||
// goroutine bounded by provisionPollTimeout.
|
||||
func (d *Dispatcher) autoProvisionClusterWorldOnStart(rec *instanceRecord) {
|
||||
if rec.ModuleID != "7dtd" {
|
||||
return
|
||||
}
|
||||
if !rec.provisionGuard.CompareAndSwap(false, true) {
|
||||
return // a watcher is already running for this instance
|
||||
}
|
||||
go func() {
|
||||
defer rec.provisionGuard.Store(false)
|
||||
log := d.log.With("instance_id", rec.InstanceID, "auto_provision", true)
|
||||
|
||||
// Only clustered worlds provision (and show the indicator). Resolve the
|
||||
// /cluster bind like clusterplayer.go; bail out for standalone servers.
|
||||
mctx, mcancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
mounts, _ := d.runtime.ContainerMounts(mctx, "panel-"+rec.InstanceID)
|
||||
mcancel()
|
||||
clustered := false
|
||||
for _, m := range mounts {
|
||||
if m.Destination == "/cluster" && m.Type == "bind" && m.Source != "" {
|
||||
clustered = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !clustered {
|
||||
return
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(provisionPollTimeout)
|
||||
delay := 5 * time.Second // first poll soon, to override a premature green "running"
|
||||
var lastDetail string
|
||||
bounced := false
|
||||
emit := func(detail string) {
|
||||
if detail == lastDetail {
|
||||
return
|
||||
}
|
||||
lastDetail = detail
|
||||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_RUNNING, 0, detail)
|
||||
}
|
||||
for {
|
||||
time.Sleep(delay)
|
||||
delay = provisionPollInterval
|
||||
if time.Now().After(deadline) {
|
||||
return // gave up; the entrypoint aligns on the next ordinary restart
|
||||
}
|
||||
ectx, cancel := context.WithTimeout(context.Background(), provisionExecTimeout)
|
||||
stdout, _, code, err := d.runtime.ExecCapture(ectx, rec.ContainerID, []string{"sh", "-c", provisionProbeScript})
|
||||
cancel()
|
||||
if err != nil || code != 0 {
|
||||
continue // mid-boot / exec hiccup — leave the agent's own status alone
|
||||
}
|
||||
switch strings.TrimSpace(string(stdout)) {
|
||||
case "CANON", "PROVISIONED":
|
||||
emit("running") // fully set up — clear the "setting up" indicator
|
||||
return
|
||||
case "NEEDS_ALIGN":
|
||||
emit(provisionDetail)
|
||||
if !bounced {
|
||||
bounced = true
|
||||
actx, acancel := context.WithTimeout(context.Background(), provisionExecTimeout)
|
||||
_, _, _, _ = d.runtime.ExecCapture(actx, rec.ContainerID, []string{"sh", "-c", provisionAttemptScript})
|
||||
acancel()
|
||||
log.Info("auto-provision: aligning new cluster world to canonical (one-time bounce)")
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID, Stream: "stdout", At: timestamppb.Now(),
|
||||
Line: "[panel] auto-provision: aligning the new world's decorations to the cluster (one-time restart)…",
|
||||
}},
|
||||
})
|
||||
rctx, rcancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
if err := d.runtime.Restart(rctx, rec.ContainerID, provisionRestartGrace); err != nil {
|
||||
log.Error("auto-provision restart failed (entrypoint aligns on the next ordinary restart)", "err", err)
|
||||
}
|
||||
rcancel()
|
||||
}
|
||||
continue // keep watching across the bounce until the align boot marks it provisioned
|
||||
default: // GENERATING / NOWORLD / ATTEMPTED — still setting up
|
||||
emit(provisionDetail)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// handleDelete tears down the instance record. If the container is still
|
||||
// running, it's stopped with the requested grace first. Optionally
|
||||
// removes every Docker volume labeled for this instance ("purge").
|
||||
//
|
||||
// Never deletes the host data_path directory — operators may want to
|
||||
// keep configs/logs that landed on the bind mount. If they want a full
|
||||
// wipe, they can delete the dir manually after.
|
||||
func (d *Dispatcher) handleDelete(ctx context.Context, corrID string, req *panelv1.InstanceDelete) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||||
|
||||
d.mu.Lock()
|
||||
rec, exists := d.instances[req.InstanceId]
|
||||
d.mu.Unlock()
|
||||
|
||||
// Cancel any in-flight steamcmd / updater goroutine FIRST. The
|
||||
// updater spins up its own sidecar container (panel-<id>-steamcmd-<n>)
|
||||
// that mounts the same data volumes. If we don't cancel it before
|
||||
// trying to remove volumes, the sidecar holds them open and the
|
||||
// volume-rm fails with "volume is in use" — operator sees the
|
||||
// instance stuck on "Deleting…" while a download finishes in the
|
||||
// background.
|
||||
d.cancelUpdater(req.InstanceId)
|
||||
// Whatever state the main container is in, any Files-tab helper that
|
||||
// was running against this instance is now orphaned. Clean it up
|
||||
// before touching anything else so its volume-holds don't interfere
|
||||
// with the main container's stop/remove.
|
||||
d.teardownFsHelper(req.InstanceId)
|
||||
|
||||
grace := time.Duration(req.GraceSeconds) * time.Second
|
||||
if grace == 0 {
|
||||
grace = 15 * time.Second
|
||||
}
|
||||
|
||||
// If we have a record, use its container_id directly. Otherwise try
|
||||
// to find the container by its canonical name (may exist from a
|
||||
// previous session that skipped rehydrate).
|
||||
containerID := ""
|
||||
if exists {
|
||||
containerID = rec.ContainerID
|
||||
} else {
|
||||
if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil {
|
||||
containerID = state.ContainerID
|
||||
}
|
||||
}
|
||||
|
||||
if containerID != "" {
|
||||
// Best-effort stop; ignore errors since container may already be stopped.
|
||||
_ = d.runtime.Stop(ctx, containerID, grace)
|
||||
removeErr := d.runtime.Remove(ctx, containerID)
|
||||
// d.runtime.Remove already swallows not-found, so a non-nil
|
||||
// removeErr means the remove genuinely failed. Before we tear down
|
||||
// any bookkeeping (in-memory record + sidecar metadata) we must be
|
||||
// SURE the container is gone — otherwise we orphan a live container
|
||||
// the controller still routes to, and the agent forgets it on the
|
||||
// next rehydrate. The classic trigger: a delete racing with agent
|
||||
// shutdown, where the Docker connection dies mid-remove
|
||||
// ("terminated signal received"). The remove errors, and the
|
||||
// follow-up existence check ALSO errors — fate unknown, not gone.
|
||||
if removeErr != nil {
|
||||
exists, existErr := d.runtime.ContainerExists(ctx, "panel-"+req.InstanceId)
|
||||
switch {
|
||||
case existErr != nil:
|
||||
// Couldn't even ask (daemon down / connection dropped /
|
||||
// ctx cancelled). Do NOT delete the sidecar — bail and let
|
||||
// the operator retry once the daemon is reachable. Leaving
|
||||
// the sidecar means the next rehydrate re-adopts the
|
||||
// instance instead of orphaning it.
|
||||
log.Error("delete: container remove failed and existence is unknown — aborting to avoid orphan",
|
||||
"remove_err", removeErr, "exists_err", existErr)
|
||||
d.replyError(corrID, req.InstanceId, "container_remove",
|
||||
fmt.Sprintf("container panel-%s remove failed and existence unverifiable: %s", req.InstanceId, removeErr.Error()))
|
||||
return
|
||||
case exists:
|
||||
// Container is confirmed still present. The controller must
|
||||
// learn — otherwise the DB row gets purged while a zombie
|
||||
// container lives on, and the next create with the same id
|
||||
// fails on a "name already in use" docker error.
|
||||
log.Error("delete: container still exists after remove", "err", removeErr)
|
||||
d.replyError(corrID, req.InstanceId, "container_remove",
|
||||
fmt.Sprintf("container panel-%s still exists after remove: %s", req.InstanceId, removeErr.Error()))
|
||||
return
|
||||
default:
|
||||
// Confirmed gone (e.g. autoremove fired between Stop and
|
||||
// Remove). Safe to proceed with bookkeeping cleanup.
|
||||
log.Warn("delete: remove returned error but container is confirmed gone", "err", removeErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Signal any live trackers / log streams to stop.
|
||||
if exists {
|
||||
if rec.TrackerCancel != nil {
|
||||
rec.TrackerCancel()
|
||||
}
|
||||
if rec.LogCancel != nil {
|
||||
rec.LogCancel()
|
||||
}
|
||||
}
|
||||
|
||||
// Purge volumes if requested.
|
||||
if req.PurgeVolumes {
|
||||
dr, ok := d.runtime.(*runtime.DockerRuntime)
|
||||
if !ok {
|
||||
log.Warn("delete: purge requested but runtime is not Docker; skipping volume cleanup")
|
||||
} else {
|
||||
// Sweep every container that mounts one of this instance's
|
||||
// volumes — running OR exited. Without this, a stranded
|
||||
// steamcmd / backup / fs-helper sidecar from a previous
|
||||
// agent crash keeps the volume "in use" and volume-rm
|
||||
// fails repeatedly. Operator sees the instance stuck
|
||||
// "Deleting…" forever.
|
||||
holders, err := dr.ListContainersHoldingInstanceVolumes(ctx, req.InstanceId)
|
||||
if err != nil {
|
||||
log.Warn("delete: scan for orphan sidecars failed", "err", err)
|
||||
}
|
||||
for _, hid := range holders {
|
||||
if err := dr.ForceRemoveContainer(ctx, hid); err != nil {
|
||||
log.Warn("delete: force-remove sidecar failed", "container_id", hid, "err", err)
|
||||
} else {
|
||||
log.Info("delete: removed orphan sidecar", "container_id", short(hid))
|
||||
}
|
||||
}
|
||||
vols, err := dr.ListVolumesByInstance(ctx, req.InstanceId)
|
||||
if err != nil {
|
||||
log.Warn("delete: list volumes failed", "err", err)
|
||||
}
|
||||
for _, v := range vols {
|
||||
// Force=true: there's no scenario where we want a panel
|
||||
// volume to survive a purge=true delete after we've
|
||||
// scrubbed every container that referenced it.
|
||||
if err := dr.RemoveVolume(ctx, v, true); err != nil {
|
||||
log.Warn("delete: remove volume failed", "volume", v, "err", err)
|
||||
d.replyError(corrID, req.InstanceId, "volume_remove", fmt.Sprintf("volume %s: %s", v, err.Error()))
|
||||
return
|
||||
}
|
||||
log.Info("delete: purged volume", "volume", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove in-memory record + sidecar metadata.
|
||||
d.mu.Lock()
|
||||
delete(d.instances, req.InstanceId)
|
||||
d.mu.Unlock()
|
||||
d.deleteMeta(req.InstanceId)
|
||||
|
||||
// Leave host data_path alone (see doc comment); operators can rm -rf
|
||||
// manually. We do NOT attempt os.RemoveAll here — too destructive to
|
||||
// do automatically on a shared multi-instance directory layout.
|
||||
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "deleted")
|
||||
d.replyOK(corrID)
|
||||
log.Info("instance deleted", "purge_volumes", req.PurgeVolumes)
|
||||
_ = os.Stat // kept imported for future host-data_path removal flag
|
||||
}
|
||||
|
||||
// sendDelete-specific correlation reply uses the existing replyOK/err pattern.
|
||||
var _ = timestamppb.Now // keep import hint while ctx-based future expansions land
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,526 @@
|
||||
package dispatch
|
||||
|
||||
// Per-player POI discovery query against the empyrion server's
|
||||
// global.db SQLite store. Spawns an alpine+sqlite sidecar with the
|
||||
// instance's saves volume mounted read-only, runs a parameterized
|
||||
// join query, returns the rows.
|
||||
//
|
||||
// Schema reference (DiscoveredPOIs / Entities / Playfields / SolarSystems
|
||||
// / LoginLogoff) was reverse-engineered from a live empyrion install on
|
||||
// 2026-04-30. Eleon's mod API does NOT expose per-player discovery —
|
||||
// Request_GlobalStructure_List returns the server-wide aggregate that
|
||||
// EAH shows. The DB has the finer per-player data the in-game map uses.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// sqlite3's CLI doesn't support `?` parameter binding (that's a
|
||||
// prepared-statement-only feature on the C API). We string-interpolate
|
||||
// the SteamID64 — already validated as 17 digits server-side, but we
|
||||
// also belt-and-suspenders re-validate at the shell layer (digits only).
|
||||
//
|
||||
// Two query modes:
|
||||
// "personal" — POIs the player personally first-walked-into
|
||||
// "faction" — POIs anyone in the player's faction has discovered
|
||||
// (matches the in-game shared map)
|
||||
const sqlPersonalTpl = `
|
||||
SELECT
|
||||
COALESCE(ss.name, '') AS solar_system,
|
||||
COALESCE(pf.name, '') AS playfield,
|
||||
COALESCE(poi.name, '') AS poi_name,
|
||||
COALESCE(poi.etype, 0) AS poi_type,
|
||||
COALESCE(dp.gametime, 0) AS game_time,
|
||||
COALESCE(ll.playername, '') AS discoverer_name,
|
||||
COALESCE(ll.playerid, '') AS discoverer_steam_id,
|
||||
COALESCE(dp.facgroup, 0) AS facgroup,
|
||||
COALESCE(dp.facid, 0) AS facid
|
||||
FROM DiscoveredPOIs dp
|
||||
JOIN Entities poi ON dp.poiid = poi.entityid
|
||||
LEFT JOIN Playfields pf ON dp.pfid = pf.pfid
|
||||
LEFT JOIN SolarSystems ss ON pf.ssid = ss.ssid
|
||||
LEFT JOIN LoginLogoff ll ON dp.entityid = ll.entityid
|
||||
WHERE dp.entityid IN (
|
||||
SELECT DISTINCT entityid FROM LoginLogoff WHERE playerid='%SID%'
|
||||
)
|
||||
ORDER BY dp.gametime DESC
|
||||
LIMIT 5000;
|
||||
`
|
||||
|
||||
const sqlFactionTpl = `
|
||||
SELECT
|
||||
COALESCE(ss.name, '') AS solar_system,
|
||||
COALESCE(pf.name, '') AS playfield,
|
||||
COALESCE(poi.name, '') AS poi_name,
|
||||
COALESCE(poi.etype, 0) AS poi_type,
|
||||
COALESCE(dp.gametime, 0) AS game_time,
|
||||
COALESCE(ll.playername, '') AS discoverer_name,
|
||||
COALESCE(ll.playerid, '') AS discoverer_steam_id,
|
||||
COALESCE(dp.facgroup, 0) AS facgroup,
|
||||
COALESCE(dp.facid, 0) AS facid
|
||||
FROM DiscoveredPOIs dp
|
||||
JOIN Entities poi ON dp.poiid = poi.entityid
|
||||
LEFT JOIN Playfields pf ON dp.pfid = pf.pfid
|
||||
LEFT JOIN SolarSystems ss ON pf.ssid = ss.ssid
|
||||
LEFT JOIN LoginLogoff ll ON dp.entityid = ll.entityid
|
||||
WHERE (dp.facgroup, dp.facid) IN (
|
||||
SELECT DISTINCT pe.facgroup, pe.facid
|
||||
FROM Entities pe
|
||||
WHERE pe.entityid IN (
|
||||
SELECT DISTINCT entityid FROM LoginLogoff WHERE playerid='%SID%'
|
||||
)
|
||||
)
|
||||
ORDER BY dp.gametime DESC
|
||||
LIMIT 5000;
|
||||
`
|
||||
|
||||
const sqlPlayerNameTpl = `
|
||||
SELECT playername FROM LoginLogoff
|
||||
WHERE playerid='%SID%'
|
||||
ORDER BY loginticks DESC
|
||||
LIMIT 1;
|
||||
`
|
||||
|
||||
func (d *Dispatcher) handleEmpyrionDiscoveries(corrID string, req *panelv1.EmpyrionDiscoveriesRequest) {
|
||||
if req == nil || req.InstanceId == "" || req.SteamId == "" {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "instance_id and steam_id required",
|
||||
})
|
||||
return
|
||||
}
|
||||
go d.runEmpyrionDiscoveriesJob(corrID, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) runEmpyrionDiscoveriesJob(corrID string, req *panelv1.EmpyrionDiscoveriesRequest) {
|
||||
mode := strings.ToLower(strings.TrimSpace(req.Mode))
|
||||
if mode == "" || mode == "personal" {
|
||||
mode = "personal"
|
||||
} else if mode != "faction" {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "mode must be 'personal' or 'faction'", Mode: req.Mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
dockerBin, err := exec.LookPath("docker")
|
||||
if err != nil {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "docker binary not found on agent host", Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Saves volume name follows the panel convention: panel-<id>-saves.
|
||||
savesVol := "panel-" + req.InstanceId + "-saves"
|
||||
// Pre-flight: volume must exist (instance exists on this agent).
|
||||
if err := exec.Command(dockerBin, "volume", "inspect", savesVol).Run(); err != nil {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: fmt.Sprintf("instance saves volume %q not found on this agent", savesVol),
|
||||
Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Validate steam_id is digits-only — defense in depth, controller
|
||||
// already checks 17-digit format. Belt-and-suspenders against any
|
||||
// future caller that bypasses the controller's validation.
|
||||
for _, c := range req.SteamId {
|
||||
if c < '0' || c > '9' {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "steam_id must be digits only", Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
tpl := sqlPersonalTpl
|
||||
if mode == "faction" {
|
||||
tpl = sqlFactionTpl
|
||||
}
|
||||
q := strings.ReplaceAll(tpl, "%SID%", req.SteamId)
|
||||
nameQ := strings.ReplaceAll(sqlPlayerNameTpl, "%SID%", req.SteamId)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// alpine+sqlite sidecar. We don't need to discover the savegame
|
||||
// folder name dynamically — the entrypoint creates exactly one
|
||||
// (via dedicated.yaml.SaveDirectory which defaults to "DediGame";
|
||||
// the panel's empyrion module pins this). We pick `*/global.db`
|
||||
// at the shell layer so renames don't break the query.
|
||||
//
|
||||
// Note: -2 disables stdin which avoids the `command timed out`
|
||||
// path that empty stdin can hit on some Alpine builds. We feed the
|
||||
// SQL via the third positional arg.
|
||||
rowsScript := `set -e
|
||||
apk add --no-cache sqlite >/dev/null 2>&1
|
||||
DB=$(ls /saves/Saves/Games/*/global.db 2>/dev/null | head -1)
|
||||
if [ -z "$DB" ]; then echo '{"err":"no global.db found in saves volume"}' >&2; exit 2; fi
|
||||
NAME=$(sqlite3 "$DB" "$1" 2>/dev/null || echo "")
|
||||
ROWS=$(sqlite3 -json "$DB" "$2" 2>&1)
|
||||
if [ -z "$ROWS" ]; then ROWS='[]'; fi
|
||||
NAME_JSON=$(printf '%s' "$NAME" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g')
|
||||
printf '{"player_name":"%s","rows":%s}\n' "$NAME_JSON" "$ROWS"
|
||||
`
|
||||
|
||||
out, err := exec.CommandContext(ctx, dockerBin,
|
||||
"run", "--rm",
|
||||
"-v", savesVol+":/saves:ro",
|
||||
"--entrypoint", "sh",
|
||||
"alpine:3.20", "-c", rowsScript, "_", nameQ, q,
|
||||
).Output()
|
||||
if err != nil {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "sqlite query failed: " + err.Error(),
|
||||
Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Parse the wrapper json {"player_name": "...", "rows": [...]}.
|
||||
type wireRow struct {
|
||||
SolarSystem string `json:"solar_system"`
|
||||
Playfield string `json:"playfield"`
|
||||
PoiName string `json:"poi_name"`
|
||||
PoiType int32 `json:"poi_type"`
|
||||
GameTime int64 `json:"game_time"`
|
||||
DiscovererName string `json:"discoverer_name"`
|
||||
DiscovererSteamID string `json:"discoverer_steam_id"`
|
||||
Facgroup int32 `json:"facgroup"`
|
||||
Facid int32 `json:"facid"`
|
||||
}
|
||||
var wrap struct {
|
||||
PlayerName string `json:"player_name"`
|
||||
Rows []wireRow `json:"rows"`
|
||||
}
|
||||
// Strip leading whitespace; sqlite3 -json sometimes prepends a BOM-ish blank line.
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &wrap); err != nil {
|
||||
// Tolerate "no rows" — sqlite3 -json prints `[]` (good) or empty
|
||||
// string when no DB (bad — caught above) or `null` for some
|
||||
// builds. Fall back to empty-result on parse failure.
|
||||
var rowsOnly []wireRow
|
||||
if err2 := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &rowsOnly); err2 == nil {
|
||||
wrap.Rows = rowsOnly
|
||||
} else {
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: false, Error: "parse sqlite output: " + err.Error(),
|
||||
Mode: mode, SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
pois := make([]*panelv1.DiscoveredPOI, 0, len(wrap.Rows))
|
||||
for _, r := range wrap.Rows {
|
||||
pois = append(pois, &panelv1.DiscoveredPOI{
|
||||
SolarSystem: r.SolarSystem,
|
||||
Playfield: r.Playfield,
|
||||
PoiName: r.PoiName,
|
||||
PoiType: r.PoiType,
|
||||
GameTime: r.GameTime,
|
||||
DiscovererName: r.DiscovererName,
|
||||
DiscovererSteamId: r.DiscovererSteamID,
|
||||
FactionGroup: r.Facgroup,
|
||||
FactionId: r.Facid,
|
||||
})
|
||||
}
|
||||
d.sendEmpyrionDiscoveriesResult(corrID, &panelv1.EmpyrionDiscoveriesResult{
|
||||
Ok: true,
|
||||
Pois: pois,
|
||||
Mode: mode,
|
||||
SteamId: req.SteamId,
|
||||
PlayerName: strings.Trim(wrap.PlayerName, "\""),
|
||||
})
|
||||
_ = errors.New
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendEmpyrionDiscoveriesResult(corrID string, res *panelv1.EmpyrionDiscoveriesResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_EmpyrionDiscoveriesResult{
|
||||
EmpyrionDiscoveriesResult: res,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Player Summary — stats + sessions + inventory in one query
|
||||
// ============================================================================
|
||||
|
||||
// playerSummaryScript is one big sh script run inside an alpine sidecar.
|
||||
// It opens global.db, resolves the player's entityid via LoginLogoff,
|
||||
// then runs four sqlite queries with -json output. The output is one
|
||||
// JSON object with named keys for each query — we marshal-decode it
|
||||
// directly into the proto result.
|
||||
//
|
||||
// The 4 queries:
|
||||
// 1. summary — Entities row + LoginLogoff name lookup + last position
|
||||
// 2. stats — PlayerStatistics row
|
||||
// 3. sessions — recent LoginLogoff rows
|
||||
// 4. inv — most recent PlayerInventory snapshot + items
|
||||
//
|
||||
// We accept that some of these may return [] (player has no stats row
|
||||
// yet, never logged in this savegame, etc.) — JSON parser handles it.
|
||||
func playerSummaryScript() string {
|
||||
return `set -e
|
||||
apk add --no-cache sqlite >/dev/null 2>&1
|
||||
DB=$(ls /saves/Saves/Games/*/global.db 2>/dev/null | head -1)
|
||||
if [ -z "$DB" ]; then echo '{"err":"no global.db found in saves volume"}' >&2; exit 2; fi
|
||||
SID="$1"
|
||||
LIMIT="${2:-50}"
|
||||
|
||||
# Resolve entityid for this Steam ID. Use the most recent LoginLogoff
|
||||
# row to handle character resets that re-bind the same Steam ID to a
|
||||
# new entityid.
|
||||
EID=$(sqlite3 "$DB" "SELECT entityid FROM LoginLogoff WHERE playerid='${SID}' ORDER BY loginticks DESC LIMIT 1;")
|
||||
if [ -z "$EID" ]; then
|
||||
printf '{"steam_id":"%s","entity_id":0,"player_name":"","stats":null,"sessions":[],"inventory":[]}\n' "$SID"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
NAME=$(sqlite3 "$DB" "SELECT playername FROM LoginLogoff WHERE entityid=${EID} ORDER BY loginticks DESC LIMIT 1;")
|
||||
FG=$(sqlite3 "$DB" "SELECT facgroup FROM Entities WHERE entityid=${EID};")
|
||||
FI=$(sqlite3 "$DB" "SELECT facid FROM Entities WHERE entityid=${EID};")
|
||||
|
||||
# Last position from PlayerData.
|
||||
PD=$(sqlite3 -json "$DB" "
|
||||
SELECT
|
||||
COALESCE(pf.name, '') AS last_playfield,
|
||||
COALESCE(pd.posx, 0) AS last_x,
|
||||
COALESCE(pd.posy, 0) AS last_y,
|
||||
COALESCE(pd.posz, 0) AS last_z,
|
||||
COALESCE(pd.credits, 0) AS credits
|
||||
FROM PlayerData pd
|
||||
LEFT JOIN Playfields pf ON pd.pfid = pf.pfid
|
||||
WHERE pd.entityid=${EID};
|
||||
")
|
||||
|
||||
STATS=$(sqlite3 -json "$DB" "
|
||||
SELECT
|
||||
COALESCE(killedenemies, 0) AS killed_enemies,
|
||||
COALESCE(killedallied, 0) AS killed_allied,
|
||||
COALESCE(killedanimals, 0) AS killed_animals,
|
||||
COALESCE(killeddrones, 0) AS killed_drones,
|
||||
COALESCE(killedplayers, 0) AS killed_players,
|
||||
COALESCE(killedalliedplayers, 0) AS killed_allied_players,
|
||||
COALESCE(died, 0) AS died,
|
||||
COALESCE(score, 0) AS score,
|
||||
COALESCE(blocksplaced, 0) AS blocks_placed,
|
||||
COALESCE(blocksdigged, 0) AS blocks_digged,
|
||||
COALESCE(walkedmeters, 0) AS walked_meters,
|
||||
COALESCE(jetpackmeters, 0) AS jetpack_meters,
|
||||
COALESCE(hvmeters, 0) AS hv_meters,
|
||||
COALESCE(svmeters, 0) AS sv_meters,
|
||||
COALESCE(cvmeters, 0) AS cv_meters,
|
||||
COALESCE(playtime, 0.0) AS playtime,
|
||||
COALESCE(travly, 0.0) AS light_years,
|
||||
COALESCE(travau, 0.0) AS astronomical_units
|
||||
FROM PlayerStatistics WHERE entityid=${EID};
|
||||
")
|
||||
|
||||
SESS=$(sqlite3 -json "$DB" "
|
||||
SELECT
|
||||
COALESCE(loginticks, 0) AS login_ticks,
|
||||
COALESCE(logoffticks, 0) AS logoff_ticks,
|
||||
COALESCE(playername, '') AS player_name,
|
||||
COALESCE(buildnr, 0) AS build_nr,
|
||||
COALESCE(ip, '') AS ip,
|
||||
COALESCE(os, '') AS os,
|
||||
COALESCE(gfx, '') AS gfx,
|
||||
COALESCE(cpu, '') AS cpu
|
||||
FROM LoginLogoff WHERE entityid=${EID}
|
||||
ORDER BY loginticks DESC LIMIT ${LIMIT};
|
||||
")
|
||||
|
||||
INV_PIID=$(sqlite3 "$DB" "SELECT piid FROM PlayerInventory WHERE entityid=${EID} ORDER BY gametime DESC LIMIT 1;")
|
||||
INV_GT=0
|
||||
INV='[]'
|
||||
if [ -n "$INV_PIID" ]; then
|
||||
INV_GT=$(sqlite3 "$DB" "SELECT gametime FROM PlayerInventory WHERE piid=${INV_PIID};")
|
||||
INV=$(sqlite3 -json "$DB" "
|
||||
SELECT item AS item_id, count
|
||||
FROM PlayerInventoryItems WHERE piid=${INV_PIID};
|
||||
")
|
||||
if [ -z "$INV" ]; then INV='[]'; fi
|
||||
fi
|
||||
|
||||
if [ -z "$STATS" ]; then STATS='[]'; fi
|
||||
if [ -z "$SESS" ]; then SESS='[]'; fi
|
||||
if [ -z "$PD" ]; then PD='[]'; fi
|
||||
|
||||
printf '{"steam_id":"%s","player_name":%s,"entity_id":%s,"faction_group":%s,"faction_id":%s,"position":%s,"stats":%s,"sessions":%s,"inventory_gametime":%s,"inventory":%s}\n' \
|
||||
"$SID" \
|
||||
"$(printf '%%s' "$NAME" | sqlite3 ':memory:' 'select json_quote(?1);' 2>/dev/null || echo '""')" \
|
||||
"${EID:-0}" "${FG:-0}" "${FI:-0}" \
|
||||
"$PD" "$STATS" "$SESS" "${INV_GT:-0}" "$INV"
|
||||
`
|
||||
}
|
||||
|
||||
func (d *Dispatcher) handleEmpyrionPlayerSummary(corrID string, req *panelv1.EmpyrionPlayerSummaryRequest) {
|
||||
if req == nil || req.InstanceId == "" || req.SteamId == "" {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "instance_id and steam_id required",
|
||||
})
|
||||
return
|
||||
}
|
||||
go d.runEmpyrionPlayerSummaryJob(corrID, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) runEmpyrionPlayerSummaryJob(corrID string, req *panelv1.EmpyrionPlayerSummaryRequest) {
|
||||
dockerBin, err := exec.LookPath("docker")
|
||||
if err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "docker binary not found on agent host", SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
savesVol := "panel-" + req.InstanceId + "-saves"
|
||||
if err := exec.Command(dockerBin, "volume", "inspect", savesVol).Run(); err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: fmt.Sprintf("instance saves volume %q not found on this agent", savesVol), SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
limit := req.MaxLoginRows
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, dockerBin,
|
||||
"run", "--rm",
|
||||
"-v", savesVol+":/saves:ro",
|
||||
"--entrypoint", "sh",
|
||||
"alpine:3.20", "-c", playerSummaryScript(), "_",
|
||||
req.SteamId, fmt.Sprintf("%d", limit),
|
||||
).Output()
|
||||
if err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "sqlite query failed: " + err.Error(), SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
type wireStats struct {
|
||||
KilledEnemies int64 `json:"killed_enemies"`
|
||||
KilledAllied int64 `json:"killed_allied"`
|
||||
KilledAnimals int64 `json:"killed_animals"`
|
||||
KilledDrones int64 `json:"killed_drones"`
|
||||
KilledPlayers int64 `json:"killed_players"`
|
||||
KilledAlliedPlayers int64 `json:"killed_allied_players"`
|
||||
Died int64 `json:"died"`
|
||||
Score int64 `json:"score"`
|
||||
BlocksPlaced int64 `json:"blocks_placed"`
|
||||
BlocksDigged int64 `json:"blocks_digged"`
|
||||
WalkedMeters int64 `json:"walked_meters"`
|
||||
JetpackMeters int64 `json:"jetpack_meters"`
|
||||
HvMeters int64 `json:"hv_meters"`
|
||||
SvMeters int64 `json:"sv_meters"`
|
||||
CvMeters int64 `json:"cv_meters"`
|
||||
Playtime float64 `json:"playtime"`
|
||||
LightYears float64 `json:"light_years"`
|
||||
AstroUnits float64 `json:"astronomical_units"`
|
||||
}
|
||||
type wireSession struct {
|
||||
LoginTicks int64 `json:"login_ticks"`
|
||||
LogoffTicks int64 `json:"logoff_ticks"`
|
||||
PlayerName string `json:"player_name"`
|
||||
BuildNr int64 `json:"build_nr"`
|
||||
IP string `json:"ip"`
|
||||
OS string `json:"os"`
|
||||
GFX string `json:"gfx"`
|
||||
CPU string `json:"cpu"`
|
||||
}
|
||||
type wireInvItem struct {
|
||||
ItemID int32 `json:"item_id"`
|
||||
Count int32 `json:"count"`
|
||||
}
|
||||
type wirePos struct {
|
||||
LastPlayfield string `json:"last_playfield"`
|
||||
LastX float64 `json:"last_x"`
|
||||
LastY float64 `json:"last_y"`
|
||||
LastZ float64 `json:"last_z"`
|
||||
Credits int64 `json:"credits"`
|
||||
}
|
||||
type wire struct {
|
||||
SteamID string `json:"steam_id"`
|
||||
PlayerName string `json:"player_name"`
|
||||
EntityID int64 `json:"entity_id"`
|
||||
FactionGroup int32 `json:"faction_group"`
|
||||
FactionID int32 `json:"faction_id"`
|
||||
Position []wirePos `json:"position"`
|
||||
Stats []wireStats `json:"stats"`
|
||||
Sessions []wireSession `json:"sessions"`
|
||||
InventoryGametime int64 `json:"inventory_gametime"`
|
||||
Inventory []wireInvItem `json:"inventory"`
|
||||
}
|
||||
var w wire
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &w); err != nil {
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: false, Error: "parse sqlite output: " + err.Error(), SteamId: req.SteamId,
|
||||
})
|
||||
return
|
||||
}
|
||||
res := &panelv1.EmpyrionPlayerSummaryResult{
|
||||
Ok: true,
|
||||
SteamId: w.SteamID,
|
||||
PlayerName: strings.Trim(w.PlayerName, "\""),
|
||||
EntityId: w.EntityID,
|
||||
FactionGroup: w.FactionGroup,
|
||||
FactionId: w.FactionID,
|
||||
InventoryGametime: w.InventoryGametime,
|
||||
}
|
||||
if len(w.Position) > 0 {
|
||||
res.LastPlayfield = w.Position[0].LastPlayfield
|
||||
res.LastX = w.Position[0].LastX
|
||||
res.LastY = w.Position[0].LastY
|
||||
res.LastZ = w.Position[0].LastZ
|
||||
res.Credits = w.Position[0].Credits
|
||||
}
|
||||
if len(w.Stats) > 0 {
|
||||
s := w.Stats[0]
|
||||
res.Stats = &panelv1.PlayerStatsRow{
|
||||
KilledEnemies: s.KilledEnemies, KilledAllied: s.KilledAllied,
|
||||
KilledAnimals: s.KilledAnimals, KilledDrones: s.KilledDrones,
|
||||
KilledPlayers: s.KilledPlayers, KilledAlliedPlayers: s.KilledAlliedPlayers,
|
||||
Died: s.Died, Score: s.Score,
|
||||
BlocksPlaced: s.BlocksPlaced, BlocksDigged: s.BlocksDigged,
|
||||
WalkedMeters: s.WalkedMeters, JetpackMeters: s.JetpackMeters,
|
||||
HvMeters: s.HvMeters, SvMeters: s.SvMeters, CvMeters: s.CvMeters,
|
||||
Playtime: s.Playtime, LightYears: s.LightYears, AstronomicalUnits: s.AstroUnits,
|
||||
}
|
||||
}
|
||||
for _, s := range w.Sessions {
|
||||
res.Sessions = append(res.Sessions, &panelv1.PlayerLoginRow{
|
||||
LoginTicks: s.LoginTicks, LogoffTicks: s.LogoffTicks,
|
||||
PlayerName: s.PlayerName, BuildNr: s.BuildNr,
|
||||
Ip: s.IP, Os: s.OS, Gfx: s.GFX, Cpu: s.CPU,
|
||||
})
|
||||
}
|
||||
for _, it := range w.Inventory {
|
||||
res.Inventory = append(res.Inventory, &panelv1.PlayerInventoryItem{
|
||||
ItemId: it.ItemID, Count: it.Count,
|
||||
})
|
||||
}
|
||||
d.sendEmpyrionPlayerSummaryResult(corrID, res)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendEmpyrionPlayerSummaryResult(corrID string, res *panelv1.EmpyrionPlayerSummaryResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_EmpyrionPlayerSummaryResult{
|
||||
EmpyrionPlayerSummaryResult: res,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package dispatch
|
||||
|
||||
// Agent-side Empyrion workshop-scenario installer. Runs entirely on the
|
||||
// local Docker daemon, so a multi-agent panel can install Reforged Eden 2
|
||||
// (and any other workshop scenario) into an Empyrion instance regardless
|
||||
// of which agent owns it.
|
||||
//
|
||||
// Flow:
|
||||
// 1. SteamCMD sidecar mounts panel-empyrion-workshop (an agent-local
|
||||
// cache volume, lazily created on first install) and downloads the
|
||||
// workshop item.
|
||||
// 2. Alpine helper sidecar mounts the workshop volume + the instance's
|
||||
// `panel-<INSTANCE>-game` volume and copies the scenario files into
|
||||
// Content/Scenarios/<scenario_name>/.
|
||||
// 3. A marker file (.panel-installed) is touched so the empyrion
|
||||
// entrypoint can confirm the install.
|
||||
//
|
||||
// Progress is streamed back to the controller as LogLine envelopes with
|
||||
// stream="scenario" so the panel UI's scenario job log shows what's
|
||||
// happening live.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
func (d *Dispatcher) handleEmpyrionScenarioInstall(corrID string, req *panelv1.EmpyrionScenarioInstallRequest) {
|
||||
if req == nil || req.JobId == "" || req.InstanceId == "" || req.WorkshopId == "" || req.ScenarioName == "" {
|
||||
d.sendEmpyrionScenarioResult(corrID, req.GetJobId(), false, "missing required fields", req.GetScenarioName())
|
||||
return
|
||||
}
|
||||
appID := req.AppId
|
||||
if appID == "" {
|
||||
appID = "530870"
|
||||
}
|
||||
go d.runEmpyrionScenarioJob(corrID, req, appID)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) runEmpyrionScenarioJob(corrID string, req *panelv1.EmpyrionScenarioInstallRequest, appID string) {
|
||||
// Redact the password from any line we forward upstream, since
|
||||
// SteamCMD CAN echo `+login user PASS` back in error paths.
|
||||
redact := func(s string) string {
|
||||
if req.SteamPassword == "" || len(req.SteamPassword) < 4 {
|
||||
return s
|
||||
}
|
||||
return strings.ReplaceAll(s, req.SteamPassword, "[REDACTED]")
|
||||
}
|
||||
emit := func(line string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: req.InstanceId,
|
||||
Stream: "scenario",
|
||||
At: timestamppb.Now(),
|
||||
Line: redact(line),
|
||||
}},
|
||||
})
|
||||
}
|
||||
finish := func(ok bool, msg string) {
|
||||
d.sendEmpyrionScenarioResult(corrID, req.JobId, ok, msg, req.ScenarioName)
|
||||
}
|
||||
|
||||
emit(fmt.Sprintf("[scenario] starting workshop download: app=%s item=%s scenario=%q", appID, req.WorkshopId, req.ScenarioName))
|
||||
|
||||
dockerBin, err := exec.LookPath("docker")
|
||||
if err != nil {
|
||||
finish(false, "docker binary not found on agent host")
|
||||
return
|
||||
}
|
||||
|
||||
// Phase 1 — workshop download. The volume name is per-agent (Docker
|
||||
// daemons don't share volumes); it's created lazily by the first run.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// Build the SteamCMD argv for an attempt — login portion is the only
|
||||
// variable. `+app_license_request` refreshes Steam's entitlement
|
||||
// cache before the workshop download (cheap fix for stale-license
|
||||
// "Access Denied" on owned games).
|
||||
buildArgs := func(loginArgs []string) []string {
|
||||
base := []string{
|
||||
"run", "--rm",
|
||||
"-v", "panel-steamcmd-auth:/root/.local/share/Steam",
|
||||
"-v", "panel-empyrion-workshop:/empyrion-workshop/steamapps/workshop",
|
||||
"steamcmd/steamcmd:latest",
|
||||
"+@sSteamCmdForcePlatformType", "windows",
|
||||
"+force_install_dir", "/empyrion-workshop",
|
||||
}
|
||||
base = append(base, loginArgs...)
|
||||
base = append(base,
|
||||
"+app_license_request", appID,
|
||||
"+workshop_download_item", appID, req.WorkshopId, "validate",
|
||||
"+quit",
|
||||
)
|
||||
return base
|
||||
}
|
||||
|
||||
var out string
|
||||
switch {
|
||||
case req.SteamUsername == "":
|
||||
emit("[scenario] no Steam credentials provided — trying anonymous (will fail for app workshops that require ownership)")
|
||||
out, err = runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", "anonymous"}), emit)
|
||||
default:
|
||||
// Try cached refresh-token login first (no password). If the
|
||||
// agent's panel-steamcmd-auth volume has a valid token from a
|
||||
// prior install, this skips Steam Guard entirely — no push
|
||||
// notification fires. Only when the cache is missing or expired
|
||||
// (Logon failure / no cached credentials in the SteamCMD output)
|
||||
// do we fall back to full password auth, which DOES require
|
||||
// approving a Steam Guard push but then re-primes the cache for
|
||||
// future runs.
|
||||
emit(fmt.Sprintf("[scenario] login attempt 1: %q via cached refresh token (no Steam Guard push if cache valid)", req.SteamUsername))
|
||||
cachedOut, cachedErr := runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", req.SteamUsername}), emit)
|
||||
needsPwd := cachedErr != nil ||
|
||||
strings.Contains(cachedOut, "Logon failure") ||
|
||||
strings.Contains(cachedOut, "No cached credentials") ||
|
||||
strings.Contains(cachedOut, "FAILED login") ||
|
||||
strings.Contains(cachedOut, "Login failure")
|
||||
if !needsPwd {
|
||||
out, err = cachedOut, cachedErr
|
||||
break
|
||||
}
|
||||
if req.SteamPassword == "" {
|
||||
finish(false, "Steam refresh-token cache is missing/expired on this agent and no password is available to re-prime it. Re-enter credentials via the Steam login modal.")
|
||||
return
|
||||
}
|
||||
emit(fmt.Sprintf("[scenario] login attempt 2: %q with password (Steam Guard push will fire — approve on phone)", req.SteamUsername))
|
||||
out, err = runWithLineEmit(ctx, dockerBin, buildArgs([]string{"+login", req.SteamUsername, req.SteamPassword}), emit)
|
||||
}
|
||||
out = redact(out)
|
||||
if err != nil {
|
||||
finish(false, "steamcmd: "+err.Error())
|
||||
return
|
||||
}
|
||||
// Steam reports "Access Denied" when the logged-in account can't
|
||||
// pull this workshop item. Common causes (in order of likelihood):
|
||||
// 1. Account doesn't own the app — most common
|
||||
// 2. License cache stale — we already issued +app_license_request
|
||||
// above, so if it still errors that's not it
|
||||
// 3. Workshop item is private / friends-only / restricted by author
|
||||
// 4. Workshop item requires the operator to "subscribe" to it via
|
||||
// the desktop Steam client at least once before SteamCMD can
|
||||
// pull it (some apps gate this)
|
||||
// 5. Family Library Sharing — the account borrows Empyrion via
|
||||
// sharing instead of owning a license; SteamCMD requires real
|
||||
// ownership, not borrowed licenses
|
||||
if strings.Contains(out, "ERROR! Download item") && strings.Contains(out, "Access Denied") {
|
||||
// Account confirmed-owner of the app but still denied — this is
|
||||
// almost always a publisher-side restriction on workshop_download_item
|
||||
// (Empyrion / Eleon is the canonical example: the desktop Steam
|
||||
// client uses a different API path that's allowed, but SteamCMD's
|
||||
// path is denied for this app's workshop). NOT a panel bug, NOT an
|
||||
// account bug. Manual upload is the only reliable path.
|
||||
ownsApp := strings.Contains(out, "already owned")
|
||||
var msg string
|
||||
if ownsApp {
|
||||
msg = fmt.Sprintf(
|
||||
"Workshop item %s denied even though account %q owns app %s (Steam confirmed: \"AppID %s already owned\"). "+
|
||||
"This is a publisher-side restriction on SteamCMD's workshop API — common for Empyrion (Eleon disables CLI workshop downloads; the desktop Steam client uses a different allowed path). "+
|
||||
"Workaround: subscribe to the item in the desktop Steam client (Workshop → Subscribe), let it download to %%STEAM%%/steamapps/workshop/content/%s/%s/, then upload that folder into the instance's panel-empyrion-game volume at Content/Scenarios/<scenario_name>/. The panel can't bypass this — Steam policy enforced server-side.",
|
||||
req.WorkshopId, req.SteamUsername, appID, appID, appID, req.WorkshopId,
|
||||
)
|
||||
} else {
|
||||
msg = fmt.Sprintf(
|
||||
"Steam returned Access Denied for workshop item %s after logging in as %q. "+
|
||||
"Likely causes: (a) account doesn't actually own app %s (Family Library Sharing borrows don't count — needs a purchase license); "+
|
||||
"(b) workshop item is private/friends-only/restricted by author; "+
|
||||
"(c) item requires desktop-Steam-client subscription first. "+
|
||||
"Workaround: download via desktop Steam client and drop Content/Scenarios/<folder> into the instance volume manually.",
|
||||
req.WorkshopId, req.SteamUsername, appID,
|
||||
)
|
||||
}
|
||||
finish(false, msg)
|
||||
return
|
||||
}
|
||||
if !strings.Contains(out, "Success. Downloaded item") {
|
||||
// Don't fail outright on a missing success marker — some
|
||||
// SteamCMD versions return without that exact string when the
|
||||
// item is already cached. Sanity-check the file actually
|
||||
// exists in the volume before deciding.
|
||||
probe := []string{"run", "--rm", "-v", "panel-empyrion-workshop:/ws", "alpine:3.20",
|
||||
"sh", "-c", fmt.Sprintf("test -d /ws/content/%s/%s && echo cached", appID, req.WorkshopId)}
|
||||
probeOut, _ := exec.CommandContext(ctx, dockerBin, probe...).CombinedOutput()
|
||||
if !strings.Contains(string(probeOut), "cached") {
|
||||
finish(false, "steamcmd did not download item — see log for details")
|
||||
return
|
||||
}
|
||||
emit("[scenario] workshop item already cached on this agent — skipping re-download")
|
||||
}
|
||||
|
||||
// Phase 2 — copy into instance volume.
|
||||
emit("[scenario] copying scenario files into instance volume…")
|
||||
gameVol := "panel-" + req.InstanceId + "-game"
|
||||
dst := "/dst/Content/Scenarios/" + req.ScenarioName
|
||||
|
||||
// Verify the instance volume exists; if not, the instance was
|
||||
// never created on this agent.
|
||||
probe := []string{"volume", "inspect", gameVol}
|
||||
if err := exec.CommandContext(ctx, dockerBin, probe...).Run(); err != nil {
|
||||
finish(false, fmt.Sprintf("instance volume %q not found on this agent — is the empyrion instance created here?", gameVol))
|
||||
return
|
||||
}
|
||||
|
||||
helperArgs := []string{
|
||||
"run", "--rm",
|
||||
"-v", "panel-empyrion-workshop:/src",
|
||||
"-v", gameVol + ":/dst",
|
||||
"alpine:3.20", "sh", "-c",
|
||||
fmt.Sprintf(
|
||||
"mkdir -p %s && rm -rf %s/* %s/.* 2>/dev/null; "+
|
||||
"cp -a /src/content/%s/%s/. %s/ && date > %s/.panel-installed && echo OK",
|
||||
shQuote(dst), shQuote(dst), shQuote(dst),
|
||||
appID, req.WorkshopId, shQuote(dst), shQuote(dst),
|
||||
),
|
||||
}
|
||||
out, err = runWithLineEmit(ctx, dockerBin, helperArgs, emit)
|
||||
if err != nil {
|
||||
finish(false, "scenario copy failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if !strings.Contains(out, "OK") {
|
||||
finish(false, "scenario copy did not confirm OK — see log for details")
|
||||
return
|
||||
}
|
||||
|
||||
emit(fmt.Sprintf("[scenario] %q installed into %s", req.ScenarioName, req.InstanceId))
|
||||
finish(true, "")
|
||||
}
|
||||
|
||||
// shQuote single-quotes a path for safe inclusion in a `sh -c` script.
|
||||
// Embedded single quotes are escaped via the standard '\'' trick. Used
|
||||
// for scenario_name which is operator input — even though the controller
|
||||
// already rejects path-traversal characters, defense-in-depth at the
|
||||
// agent layer is cheap.
|
||||
func shQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
// runWithLineEmit spawns docker, captures combined stdout/stderr,
|
||||
// emits each line as it arrives, and returns the full combined output
|
||||
// for post-mortem checks (success markers etc.) when the process exits.
|
||||
func runWithLineEmit(ctx context.Context, bin string, args []string, emit func(string)) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, bin, args...)
|
||||
pipe, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
cmd.Stderr = cmd.Stdout
|
||||
if err := cmd.Start(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var buf strings.Builder
|
||||
scanCh := make(chan struct{})
|
||||
go func() {
|
||||
defer close(scanCh)
|
||||
readBuf := make([]byte, 4096)
|
||||
var line strings.Builder
|
||||
for {
|
||||
n, err := pipe.Read(readBuf)
|
||||
if n > 0 {
|
||||
chunk := readBuf[:n]
|
||||
buf.Write(chunk)
|
||||
for _, b := range chunk {
|
||||
if b == '\n' {
|
||||
emit(strings.TrimRight(line.String(), "\r"))
|
||||
line.Reset()
|
||||
} else {
|
||||
line.WriteByte(b)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if line.Len() > 0 {
|
||||
emit(strings.TrimRight(line.String(), "\r"))
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
waitErr := cmd.Wait()
|
||||
<-scanCh
|
||||
if waitErr != nil {
|
||||
return buf.String(), waitErr
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendEmpyrionScenarioResult(corrID, jobID string, ok bool, errMsg, scenarioName string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_EmpyrionScenarioInstallResult{
|
||||
EmpyrionScenarioInstallResult: &panelv1.EmpyrionScenarioInstallResult{
|
||||
JobId: jobID,
|
||||
Ok: ok,
|
||||
Error: errMsg,
|
||||
ScenarioName: scenarioName,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,662 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// File management: operations are rooted at each instance's declared
|
||||
// BrowseableRoot inside the container when the container is running +
|
||||
// the runtime supports container-path ops (Docker). Otherwise we fall
|
||||
// back to the host-side DataPath bind mount. Either path is safe-joined
|
||||
// to prevent escape.
|
||||
|
||||
const (
|
||||
// fsMaxReadBytes caps a single file-read RPC. Big enough for full
|
||||
// world saves and most ARK/Empyrion scenario zips (a few-hundred MB
|
||||
// is common). True streaming would remove the cap entirely; until
|
||||
// then 4 GiB matches the controller-side upload cap.
|
||||
fsMaxReadBytes = 4 * 1024 * 1024 * 1024
|
||||
fsMaxListSize = 5000
|
||||
|
||||
// fsCopyTimeout bounds a single CopyToContainer / CopyFromContainer
|
||||
// call. A multi-GB upload over the loopback Docker socket can take
|
||||
// 30+ seconds, and an extract that ships the whole archive's
|
||||
// contents back to the container can be larger still — give it
|
||||
// real headroom so we don't surface spurious timeouts.
|
||||
fsCopyTimeout = 10 * time.Minute
|
||||
)
|
||||
|
||||
// ---- Path safety ----
|
||||
|
||||
// safeJoinHost joins rel onto root using OS path semantics, rejecting
|
||||
// anything that canonicalizes outside of root.
|
||||
func safeJoinHost(root, rel string) (string, error) {
|
||||
rel = strings.TrimPrefix(rel, "/")
|
||||
rel = strings.TrimPrefix(rel, "\\")
|
||||
if rel == "" {
|
||||
return root, nil
|
||||
}
|
||||
joined := filepath.Join(root, rel)
|
||||
absRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
absJoined, err := filepath.Abs(joined)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sep := string(filepath.Separator)
|
||||
if absJoined != absRoot && !strings.HasPrefix(absJoined+sep, absRoot+sep) {
|
||||
return "", fmt.Errorf("path escapes instance data directory")
|
||||
}
|
||||
return absJoined, nil
|
||||
}
|
||||
|
||||
// safeJoinContainer joins rel onto root using Linux container semantics
|
||||
// (always forward-slash). Returns the cleaned absolute path.
|
||||
func safeJoinContainer(root, rel string) (string, error) {
|
||||
rel = strings.TrimPrefix(rel, "/")
|
||||
if rel == "" {
|
||||
return root, nil
|
||||
}
|
||||
joined := path.Clean(path.Join(root, rel))
|
||||
if joined != root && !strings.HasPrefix(joined+"/", root+"/") {
|
||||
return "", fmt.Errorf("path escapes instance root")
|
||||
}
|
||||
return joined, nil
|
||||
}
|
||||
|
||||
// rootPaths returns the full list of allowed root paths for a record
|
||||
// (flattens BrowseableRoots → container paths, or falls back to the
|
||||
// singular BrowseableRoot). Always returns at least one entry.
|
||||
func rootPaths(rec *instanceRecord) []string {
|
||||
if len(rec.BrowseableRoots) > 0 {
|
||||
out := make([]string, 0, len(rec.BrowseableRoots))
|
||||
for _, r := range rec.BrowseableRoots {
|
||||
if r.Path != "" {
|
||||
out = append(out, r.Path)
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
if rec.BrowseableRoot != "" {
|
||||
return []string{rec.BrowseableRoot}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// safeJoinAny resolves a caller-supplied path against ANY of the module's
|
||||
// declared roots. Semantics:
|
||||
//
|
||||
// - If p starts with "/", it must be EQUAL TO or UNDER one of the
|
||||
// roots; we return it cleaned as-is. Lets the UI send container-
|
||||
// absolute paths when it knows the full layout (multi-root modules).
|
||||
// - Otherwise p is relative, resolved against `defaultRoot`. Same
|
||||
// safeJoinContainer behaviour as before.
|
||||
//
|
||||
// Returns an error if the path can't be mapped to any root or tries to
|
||||
// escape via `..`.
|
||||
func safeJoinAny(defaultRoot string, roots []string, p string) (string, error) {
|
||||
if strings.HasPrefix(p, "/") {
|
||||
cleaned := path.Clean(p)
|
||||
for _, r := range roots {
|
||||
if cleaned == r || strings.HasPrefix(cleaned+"/", r+"/") {
|
||||
return cleaned, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("path %q is not under any declared root", cleaned)
|
||||
}
|
||||
return safeJoinContainer(defaultRoot, p)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) lookupRecord(instanceID string) (*instanceRecord, error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
rec, ok := d.instances[instanceID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("instance %q not on this target", instanceID)
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// useContainerOps decides whether to go through the container runtime
|
||||
// (true) or fall back to the host bind mount (false). Any module that
|
||||
// declares a browseable root uses container ops — if the main container
|
||||
// isn't running at the moment, we spin up a helper sidecar (see
|
||||
// getFsTargetContainerID). This lets operators browse/edit files on
|
||||
// stopped instances.
|
||||
func (d *Dispatcher) useContainerOps(rec *instanceRecord) bool {
|
||||
return rec.BrowseableRoot != ""
|
||||
}
|
||||
|
||||
// ---- FS helper sidecar: file ops on stopped containers ----
|
||||
|
||||
// fsHelperImage is the base for the helper container. We use debian-slim
|
||||
// rather than alpine because our `ls --time-style=+%s` call relies on GNU
|
||||
// coreutils — busybox's ls doesn't accept --time-style. debian:12-slim is
|
||||
// already on disk for every panel-native game module, so no extra pull.
|
||||
const fsHelperImage = "debian:12-slim"
|
||||
|
||||
// fsHelperIdleTTL is how long a helper container is kept alive after its
|
||||
// last file-op access. Balance: long enough that click-around browsing
|
||||
// reuses the same helper without respawning; short enough that a forgotten
|
||||
// Files tab doesn't pin volumes forever.
|
||||
const fsHelperIdleTTL = 5 * time.Minute
|
||||
|
||||
// getFsTargetContainerID picks the container to route a file op through:
|
||||
// the main instance container if it's running, or a helper sidecar
|
||||
// (lazy-created + started on demand) if it isn't. The helper reuses the
|
||||
// same named volumes as the main container, so the operator sees the
|
||||
// same files they would see with the game running.
|
||||
func (d *Dispatcher) getFsTargetContainerID(ctx context.Context, rec *instanceRecord) (string, error) {
|
||||
mainName := "panel-" + rec.InstanceID
|
||||
st, err := d.runtime.InspectByName(ctx, mainName)
|
||||
if err == nil && st.Status == "running" {
|
||||
// Main is alive — tear down any lingering helper so volumes
|
||||
// aren't held by two containers simultaneously during live play.
|
||||
d.teardownFsHelper(rec.InstanceID)
|
||||
return st.ContainerID, nil
|
||||
}
|
||||
return d.ensureFsHelper(ctx, rec)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) ensureFsHelper(ctx context.Context, rec *instanceRecord) (string, error) {
|
||||
helperName := "panel-" + rec.InstanceID + "-fs"
|
||||
|
||||
// Reuse existing helper if it's still healthy.
|
||||
d.mu.Lock()
|
||||
h, ok := d.fsHelpers[rec.InstanceID]
|
||||
d.mu.Unlock()
|
||||
if ok {
|
||||
st, err := d.runtime.InspectByName(ctx, helperName)
|
||||
if err == nil && st.Status == "running" {
|
||||
d.mu.Lock()
|
||||
h.lastAccess = time.Now()
|
||||
d.mu.Unlock()
|
||||
return h.containerID, nil
|
||||
}
|
||||
// Stale record — container gone or in a bad state. Drop + rebuild.
|
||||
d.teardownFsHelper(rec.InstanceID)
|
||||
}
|
||||
|
||||
// Check for an orphaned helper container on Docker's side from a
|
||||
// previous agent run. When the agent restarts, in-memory fsHelpers
|
||||
// empties but the container persists. Adopt it if it's running,
|
||||
// remove it if not (Create will then succeed).
|
||||
if st, err := d.runtime.InspectByName(ctx, helperName); err == nil {
|
||||
if st.Status == "running" {
|
||||
d.log.Info("fs helper adopted from prior agent run",
|
||||
"instance_id", rec.InstanceID, "container_id", st.ContainerID)
|
||||
d.mu.Lock()
|
||||
d.fsHelpers[rec.InstanceID] = &fsHelper{containerID: st.ContainerID, lastAccess: time.Now()}
|
||||
d.mu.Unlock()
|
||||
return st.ContainerID, nil
|
||||
}
|
||||
d.log.Info("removing stale fs helper before recreate",
|
||||
"instance_id", rec.InstanceID, "container_id", st.ContainerID, "status", st.Status)
|
||||
rmCtx, rmCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = d.runtime.Remove(rmCtx, st.ContainerID)
|
||||
rmCancel()
|
||||
}
|
||||
|
||||
// Resolve the instance's volumes from its manifest so the helper sees
|
||||
// exactly what the game would.
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("module %q not loaded; can't mount helper volumes", rec.ModuleID)
|
||||
}
|
||||
volumes := agentmodule.ResolveVolumes(manifest, rec.InstanceID, rec.DataPath)
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: rec.InstanceID + "-fs",
|
||||
IsSidecar: true,
|
||||
Image: fsHelperImage,
|
||||
Command: []string{"sleep", "infinity"},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
containerID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create fs helper: %w", err)
|
||||
}
|
||||
if err := d.runtime.Start(ctx, containerID); err != nil {
|
||||
// Clean up the half-created container so next call doesn't see a
|
||||
// zombie from `Created` state.
|
||||
rmCtx, rmCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer rmCancel()
|
||||
_ = d.runtime.Remove(rmCtx, containerID)
|
||||
return "", fmt.Errorf("start fs helper: %w", err)
|
||||
}
|
||||
d.mu.Lock()
|
||||
d.fsHelpers[rec.InstanceID] = &fsHelper{containerID: containerID, lastAccess: time.Now()}
|
||||
d.mu.Unlock()
|
||||
d.log.Info("fs helper started", "instance_id", rec.InstanceID, "container_id", containerID)
|
||||
return containerID, nil
|
||||
}
|
||||
|
||||
// teardownFsHelper stops and removes the helper sidecar for an instance,
|
||||
// if one exists. Safe to call at any time — no-ops on absent helpers.
|
||||
// Called on main-container start (to free volumes) and on instance
|
||||
// delete (cleanup).
|
||||
func (d *Dispatcher) teardownFsHelper(instanceID string) {
|
||||
d.mu.Lock()
|
||||
h, ok := d.fsHelpers[instanceID]
|
||||
if ok {
|
||||
delete(d.fsHelpers, instanceID)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Stop(ctx, h.containerID, 2*time.Second)
|
||||
_ = d.runtime.Remove(ctx, h.containerID)
|
||||
d.log.Info("fs helper torn down", "instance_id", instanceID)
|
||||
}
|
||||
|
||||
// fsHelperReaper runs for the lifetime of the dispatcher and tears down
|
||||
// any helper that hasn't been touched in fsHelperIdleTTL. Scans every
|
||||
// minute — granular enough that abandoned Files tabs don't pin resources
|
||||
// for long, coarse enough that the scan itself is cheap.
|
||||
func (d *Dispatcher) fsHelperReaper() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
now := time.Now()
|
||||
d.mu.Lock()
|
||||
stale := make([]string, 0)
|
||||
for id, h := range d.fsHelpers {
|
||||
if now.Sub(h.lastAccess) > fsHelperIdleTTL {
|
||||
stale = append(stale, id)
|
||||
}
|
||||
}
|
||||
d.mu.Unlock()
|
||||
for _, id := range stale {
|
||||
d.teardownFsHelper(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- FsList ----
|
||||
|
||||
func (d *Dispatcher) handleFsList(corrID string, req *panelv1.FsListRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsListResult(corrID, nil, "", err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
entries, err := d.listInContainer(rec, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsListResult(corrID, nil, req.Path, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsListResult(corrID, entries, req.Path, "")
|
||||
return
|
||||
}
|
||||
// Host fallback — bind mount.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsListResult(corrID, nil, req.Path, "no file storage available (container not running and no host data path)")
|
||||
return
|
||||
}
|
||||
absPath, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsListResult(corrID, nil, req.Path, err.Error())
|
||||
return
|
||||
}
|
||||
entries, err := os.ReadDir(absPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
d.sendFsListResult(corrID, nil, req.Path, "not found")
|
||||
return
|
||||
}
|
||||
d.sendFsListResult(corrID, nil, req.Path, err.Error())
|
||||
return
|
||||
}
|
||||
out := make([]*panelv1.FsEntry, 0, len(entries))
|
||||
for i, e := range entries {
|
||||
if i >= fsMaxListSize {
|
||||
break
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, &panelv1.FsEntry{
|
||||
Name: e.Name(),
|
||||
Path: filepath.ToSlash(filepath.Join(req.Path, e.Name())),
|
||||
IsDir: e.IsDir(),
|
||||
Size: info.Size(),
|
||||
ModTime: timestamppb.New(info.ModTime()),
|
||||
})
|
||||
}
|
||||
d.sendFsListResult(corrID, out, filepath.ToSlash(req.Path), "")
|
||||
}
|
||||
|
||||
// listInContainer shells out to `ls -la --time-style=+%s <root/rel>` and
|
||||
// parses the output into FsEntry records.
|
||||
func (d *Dispatcher) listInContainer(rec *instanceRecord, rel string) ([]*panelv1.FsEntry, error) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// -L dereferences symlinks so that listing a `@ModName` link (which
|
||||
// points into the shared workshop tree) shows the mod's contents, not
|
||||
// the symlink itself. Without this, `ls -l` on a symlinked dir prints
|
||||
// the symlink as a single entry and the UI looks empty.
|
||||
stdout, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{
|
||||
"sh", "-c",
|
||||
"ls -lAL --time-style=+%s -- " + shellQuote(abs),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exec ls: %w", err)
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("ls exited %d", code)
|
||||
}
|
||||
return nil, fmt.Errorf("%s", msg)
|
||||
}
|
||||
return parseLsOutput(string(stdout), rel)
|
||||
}
|
||||
|
||||
// parseLsOutput parses `ls -lA --time-style=+%s` output. Each line:
|
||||
//
|
||||
// drwxr-xr-x 2 root root 4096 1735000000 somefile
|
||||
//
|
||||
// We split on any whitespace and take fields. The date field is a unix
|
||||
// timestamp (because of --time-style=+%s).
|
||||
func parseLsOutput(out, relRoot string) ([]*panelv1.FsEntry, error) {
|
||||
lines := strings.Split(strings.TrimSpace(out), "\n")
|
||||
entries := make([]*panelv1.FsEntry, 0, len(lines))
|
||||
for i, line := range lines {
|
||||
if i == 0 && strings.HasPrefix(line, "total ") {
|
||||
continue
|
||||
}
|
||||
// Split into at most 7 fields so filenames with spaces survive.
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 7 {
|
||||
continue
|
||||
}
|
||||
// fields[0]=mode 1=nlink 2=user 3=group 4=size 5=unixts 6+=name
|
||||
size, _ := strconv.ParseInt(fields[4], 10, 64)
|
||||
tsUnix, _ := strconv.ParseInt(fields[5], 10, 64)
|
||||
name := strings.Join(fields[6:], " ")
|
||||
// For symlinks "name -> target", keep just the name.
|
||||
if strings.Contains(name, " -> ") {
|
||||
name = strings.SplitN(name, " -> ", 2)[0]
|
||||
}
|
||||
if name == "" || name == "." || name == ".." {
|
||||
continue
|
||||
}
|
||||
isDir := strings.HasPrefix(fields[0], "d")
|
||||
entries = append(entries, &panelv1.FsEntry{
|
||||
Name: name,
|
||||
Path: path.Join(relRoot, name),
|
||||
IsDir: isDir,
|
||||
Size: size,
|
||||
ModTime: timestamppb.New(time.Unix(tsUnix, 0)),
|
||||
})
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// shellQuote wraps s in single quotes, escaping embedded quotes for
|
||||
// interpolation into `sh -c`.
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsListResult(corrID string, entries []*panelv1.FsEntry, p, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsListResult{
|
||||
FsListResult: &panelv1.FsListResult{Entries: entries, Path: p, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsRead ----
|
||||
|
||||
func (d *Dispatcher) handleFsRead(corrID string, req *panelv1.FsReadRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
data, err := d.runtime.CopyFileFromContainer(ctx, targetID, abs)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if len(data) > fsMaxReadBytes {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, fmt.Sprintf("file too large (%d bytes; max %d)", len(data), fsMaxReadBytes))
|
||||
return
|
||||
}
|
||||
d.sendFsReadResult(corrID, req.Path, data, "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, "no file storage available")
|
||||
return
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if info.IsDir() {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, "is a directory")
|
||||
return
|
||||
}
|
||||
if info.Size() > fsMaxReadBytes {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, fmt.Sprintf("file too large (%d bytes; max %d)", info.Size(), fsMaxReadBytes))
|
||||
return
|
||||
}
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
d.sendFsReadResult(corrID, req.Path, nil, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsReadResult(corrID, req.Path, data, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsReadResult(corrID, p string, content []byte, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsReadResult{
|
||||
FsReadResult: &panelv1.FsReadResult{Path: p, Content: content, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsWrite ----
|
||||
|
||||
func (d *Dispatcher) handleFsWrite(corrID string, req *panelv1.FsWriteRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
|
||||
if err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
// Make sure the parent dir exists. `mkdir -p` inside container.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
targetID, terr := d.getFsTargetContainerID(ctx, rec)
|
||||
if terr != nil {
|
||||
cancel()
|
||||
d.sendFsWriteResult(corrID, 0, terr.Error())
|
||||
return
|
||||
}
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(abs))})
|
||||
cancel()
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel2()
|
||||
if err := d.runtime.CopyFileToContainer(ctx2, targetID, abs, req.Content); err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsWriteResult(corrID, int64(len(req.Content)), "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsWriteResult(corrID, 0, "no file storage available")
|
||||
return
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(abs, req.Content, 0o644); err != nil {
|
||||
d.sendFsWriteResult(corrID, 0, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsWriteResult(corrID, int64(len(req.Content)), "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsWriteResult(corrID string, bytesWritten int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsWriteResult{
|
||||
FsWriteResult: &panelv1.FsWriteResult{BytesWritten: bytesWritten, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsDelete ----
|
||||
|
||||
func (d *Dispatcher) handleFsDelete(corrID string, req *panelv1.FsDeleteRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.Path)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if abs == rec.BrowseableRoot {
|
||||
d.sendFsDeleteResult(corrID, "refusing to delete instance root")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
flag := ""
|
||||
if req.Recursive {
|
||||
flag = "-rf"
|
||||
} else {
|
||||
flag = "-f"
|
||||
}
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"rm " + flag + " -- " + shellQuote(abs)})
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
d.sendFsDeleteResult(corrID, strings.TrimSpace(string(stderr)))
|
||||
return
|
||||
}
|
||||
d.sendFsDeleteResult(corrID, "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsDeleteResult(corrID, "no file storage available")
|
||||
return
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, req.Path)
|
||||
if err != nil {
|
||||
d.sendFsDeleteResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if abs == rec.DataPath {
|
||||
d.sendFsDeleteResult(corrID, "refusing to delete instance data root")
|
||||
return
|
||||
}
|
||||
var delErr error
|
||||
if req.Recursive {
|
||||
delErr = os.RemoveAll(abs)
|
||||
} else {
|
||||
delErr = os.Remove(abs)
|
||||
}
|
||||
if delErr != nil {
|
||||
d.sendFsDeleteResult(corrID, delErr.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsDeleteResult(corrID, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsDeleteResult(corrID, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsDeleteResult{
|
||||
FsDeleteResult: &panelv1.FsDeleteResult{Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
package dispatch
|
||||
|
||||
// Archive operations for the file browser:
|
||||
//
|
||||
// - FsExtract : expand a zip / tar / tar.gz / tar.bz2 archive that
|
||||
// already lives inside the instance's volume.
|
||||
// - FsCompress : zip up one or more paths inside the instance into a
|
||||
// destination .zip in the same volume.
|
||||
// - FsRename : safe `mv` that respects the browseable_root sandbox.
|
||||
//
|
||||
// Every container-side path goes through the existing fs helper sidecar
|
||||
// (see getFsTargetContainerID) so these all work on stopped instances.
|
||||
//
|
||||
// Implementation notes:
|
||||
// - Extract reads the archive bytes via CopyFileFromContainer, walks
|
||||
// the entries with the appropriate stdlib reader, and re-emits a
|
||||
// single tar stream that's piped to CopyTarToContainer in one API
|
||||
// call. That avoids N per-file writes.
|
||||
// - Compress walks each source via CopyDirFromContainer (tar stream
|
||||
// from Docker), and re-encodes the entries as zip. The final zip
|
||||
// bytes go back via CopyFileToContainer.
|
||||
// - We support zip + tar + tar.gz + tar.bz2 + tar.xz + 7z + rar.
|
||||
// 7z and rar use pure-Go libraries (bodgit/sevenzip,
|
||||
// nwaples/rardecode/v2) so no native deps. Encrypted rars and
|
||||
// solid 7z volumes are surfaced as errors, not silently skipped.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/archive"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// ---- FsExtract ----
|
||||
|
||||
func (d *Dispatcher) handleFsExtract(corrID string, req *panelv1.FsExtractRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if req.ArchivePath == "" {
|
||||
d.sendFsExtractResult(corrID, 0, 0, "archive_path is required")
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
d.extractInContainer(corrID, rec, req)
|
||||
return
|
||||
}
|
||||
d.extractOnHost(corrID, rec, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) extractInContainer(corrID string, rec *instanceRecord, req *panelv1.FsExtractRequest) {
|
||||
archiveAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.ArchivePath)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
destRel := req.DestDir
|
||||
if destRel == "" {
|
||||
// Default: same dir as the archive.
|
||||
destRel = path.Dir(req.ArchivePath)
|
||||
}
|
||||
destAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), destRel)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Pull the archive bytes back from the container. CopyFileFromContainer
|
||||
// internally tar-walks Docker's response and returns just the file body.
|
||||
archiveData, err := d.runtime.CopyFileFromContainer(ctx, targetID, archiveAbs)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, fmt.Errorf("read archive: %w", err).Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure the destination dir exists before we ship the tar stream.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(destAbs)})
|
||||
|
||||
// Build a tar stream of the archive's contents and pipe it to
|
||||
// CopyTarToContainer. Doing this in a single round-trip keeps the
|
||||
// wall-time of "extract a 500 MB archive with 4000 entries" manageable.
|
||||
pr, pw := io.Pipe()
|
||||
emitErr := make(chan error, 1)
|
||||
var entries, bytesOut int64
|
||||
go func() {
|
||||
n, b, err := archive.WriteAsTar(archiveData, req.ArchivePath, pw)
|
||||
entries, bytesOut = n, b
|
||||
_ = pw.CloseWithError(err)
|
||||
emitErr <- err
|
||||
}()
|
||||
if err := d.runtime.CopyTarToContainer(ctx, targetID, destAbs, pr); err != nil {
|
||||
// Pipe gets closed with our error from the goroutine; whichever
|
||||
// side surfaces first wins. We prefer the goroutine error since
|
||||
// it carries archive-format detail.
|
||||
if encErr := <-emitErr; encErr != nil {
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, encErr.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, fmt.Errorf("copy to container: %w", err).Error())
|
||||
return
|
||||
}
|
||||
if encErr := <-emitErr; encErr != nil {
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, encErr.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
// extractOnHost handles the rare host-fallback case (no container, no
|
||||
// helper) — uses os primitives. Same path-safety contract as everywhere
|
||||
// else: must stay under the instance's data path.
|
||||
func (d *Dispatcher) extractOnHost(corrID string, rec *instanceRecord, req *panelv1.FsExtractRequest) {
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsExtractResult(corrID, 0, 0, "no file storage available")
|
||||
return
|
||||
}
|
||||
archiveAbs, err := safeJoinHost(rec.DataPath, req.ArchivePath)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
destRel := req.DestDir
|
||||
if destRel == "" {
|
||||
destRel = filepath.Dir(req.ArchivePath)
|
||||
}
|
||||
destAbs, err := safeJoinHost(rec.DataPath, destRel)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(destAbs, 0o755); err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
data, err := os.ReadFile(archiveAbs)
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
entries, bytesOut, err := archive.Walk(data, req.ArchivePath, func(name string, mode int64, isDir bool, body io.Reader) error {
|
||||
safeName := archive.SafeEntryPath(name)
|
||||
if safeName == "" {
|
||||
return nil
|
||||
}
|
||||
out := filepath.Join(destAbs, filepath.FromSlash(safeName))
|
||||
if isDir {
|
||||
return os.MkdirAll(out, 0o755)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(mode&0o777))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(f, body); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
})
|
||||
if err != nil {
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsExtractResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsExtractResult(corrID string, entries, bytesOut int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsExtractResult{
|
||||
FsExtractResult: &panelv1.FsExtractResult{EntriesWritten: entries, BytesWritten: bytesOut, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsCompress ----
|
||||
|
||||
func (d *Dispatcher) handleFsCompress(corrID string, req *panelv1.FsCompressRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.Sources) == 0 {
|
||||
d.sendFsCompressResult(corrID, 0, 0, "at least one source path is required")
|
||||
return
|
||||
}
|
||||
if req.DestZipPath == "" {
|
||||
d.sendFsCompressResult(corrID, 0, 0, "dest_zip_path is required")
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
d.compressInContainer(corrID, rec, req)
|
||||
return
|
||||
}
|
||||
d.compressOnHost(corrID, rec, req)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) compressInContainer(corrID string, rec *instanceRecord, req *panelv1.FsCompressRequest) {
|
||||
destAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.DestZipPath)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var zipBuf bytes.Buffer
|
||||
zw := zip.NewWriter(&zipBuf)
|
||||
var entries, bytesOut int64
|
||||
|
||||
for _, src := range req.Sources {
|
||||
srcAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), src)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
// CopyDirFromContainer returns Docker's tar stream of srcAbs.
|
||||
// For files: a single-entry tar. For dirs: every file under it.
|
||||
// The base name of srcAbs is the prefix Docker uses, which we
|
||||
// keep in the zip so the user gets familiar names back.
|
||||
rc, err := d.runtime.CopyDirFromContainer(ctx, targetID, srcAbs)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, fmt.Errorf("copy from container %q: %w", src, err).Error())
|
||||
return
|
||||
}
|
||||
n, b, err := pipeTarIntoZip(rc, zw)
|
||||
_ = rc.Close()
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, entries+n, bytesOut+b, fmt.Errorf("encode %q: %w", src, err).Error())
|
||||
return
|
||||
}
|
||||
entries += n
|
||||
bytesOut += b
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, fmt.Errorf("zip close: %w", err).Error())
|
||||
return
|
||||
}
|
||||
// Drop the resulting zip into the volume.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(destAbs))})
|
||||
if err := d.runtime.CopyFileToContainer(ctx, targetID, destAbs, zipBuf.Bytes()); err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) compressOnHost(corrID string, rec *instanceRecord, req *panelv1.FsCompressRequest) {
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsCompressResult(corrID, 0, 0, "no file storage available")
|
||||
return
|
||||
}
|
||||
destAbs, err := safeJoinHost(rec.DataPath, req.DestZipPath)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(destAbs), 0o755); err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
f, err := os.Create(destAbs)
|
||||
if err != nil {
|
||||
d.sendFsCompressResult(corrID, 0, 0, err.Error())
|
||||
return
|
||||
}
|
||||
zw := zip.NewWriter(f)
|
||||
var entries, bytesOut int64
|
||||
for _, src := range req.Sources {
|
||||
srcAbs, err := safeJoinHost(rec.DataPath, src)
|
||||
if err != nil {
|
||||
zw.Close()
|
||||
f.Close()
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
base := filepath.Base(srcAbs)
|
||||
err = filepath.Walk(srcAbs, func(p string, info os.FileInfo, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
rel, err := filepath.Rel(srcAbs, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
zipName := base
|
||||
if rel != "." {
|
||||
zipName = filepath.ToSlash(filepath.Join(base, rel))
|
||||
}
|
||||
if info.IsDir() {
|
||||
if zipName == "" || zipName == "." {
|
||||
return nil
|
||||
}
|
||||
_, err = zw.Create(zipName + "/")
|
||||
return err
|
||||
}
|
||||
zf, err := zw.Create(zipName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rf, err := os.Open(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rf.Close()
|
||||
n, err := io.Copy(zf, rf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries++
|
||||
bytesOut += n
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
zw.Close()
|
||||
f.Close()
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
f.Close()
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsCompressResult(corrID, entries, bytesOut, "")
|
||||
}
|
||||
|
||||
// pipeTarIntoZip walks a docker-style tar stream and writes each
|
||||
// regular-file entry into the zip writer, preserving relative paths.
|
||||
// Directory entries are written as zero-byte zip directory markers.
|
||||
func pipeTarIntoZip(r io.Reader, zw *zip.Writer) (int64, int64, error) {
|
||||
tr := tar.NewReader(r)
|
||||
var entries, bytesOut, totalBytes int64
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
safe := archive.SafeEntryPath(hdr.Name)
|
||||
if safe == "" {
|
||||
continue
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if _, err := zw.Create(safe + "/"); err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
zf, err := zw.Create(safe)
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
n, err := io.Copy(zf, tr)
|
||||
if err != nil {
|
||||
return entries, bytesOut, err
|
||||
}
|
||||
entries++
|
||||
bytesOut += n
|
||||
totalBytes += n
|
||||
if totalBytes > archive.MaxTotalBytes {
|
||||
return entries, bytesOut, fmt.Errorf("compress: total uncompressed bytes exceed %d", archive.MaxTotalBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsCompressResult(corrID string, entries, bytesOut int64, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsCompressResult{
|
||||
FsCompressResult: &panelv1.FsCompressResult{EntriesWritten: entries, BytesWritten: bytesOut, Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ---- FsRename ----
|
||||
|
||||
func (d *Dispatcher) handleFsRename(corrID string, req *panelv1.FsRenameRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if req.FromPath == "" || req.ToPath == "" {
|
||||
d.sendFsRenameResult(corrID, "from_path and to_path are required")
|
||||
return
|
||||
}
|
||||
if d.useContainerOps(rec) {
|
||||
fromAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.FromPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
toAbs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), req.ToPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if fromAbs == rec.BrowseableRoot {
|
||||
d.sendFsRenameResult(corrID, "refusing to rename instance root")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
// mkdir parent so a "rename to subdir" implicitly creates it.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(toAbs))})
|
||||
_, stderr, code, err := d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mv -- " + shellQuote(fromAbs) + " " + shellQuote(toAbs)})
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if code != 0 {
|
||||
msg := strings.TrimSpace(string(stderr))
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("mv exited %d", code)
|
||||
}
|
||||
d.sendFsRenameResult(corrID, msg)
|
||||
return
|
||||
}
|
||||
d.sendFsRenameResult(corrID, "")
|
||||
return
|
||||
}
|
||||
// Host fallback.
|
||||
if rec.DataPath == "" {
|
||||
d.sendFsRenameResult(corrID, "no file storage available")
|
||||
return
|
||||
}
|
||||
fromAbs, err := safeJoinHost(rec.DataPath, req.FromPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
toAbs, err := safeJoinHost(rec.DataPath, req.ToPath)
|
||||
if err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if fromAbs == rec.DataPath {
|
||||
d.sendFsRenameResult(corrID, "refusing to rename instance root")
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(toAbs), 0o755); err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.Rename(fromAbs, toAbs); err != nil {
|
||||
d.sendFsRenameResult(corrID, err.Error())
|
||||
return
|
||||
}
|
||||
d.sendFsRenameResult(corrID, "")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendFsRenameResult(corrID, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsRenameResult{
|
||||
FsRenameResult: &panelv1.FsRenameResult{Error: errMsg},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package dispatch
|
||||
|
||||
// Chunked file upload — solves three problems at once:
|
||||
//
|
||||
// 1. Cloudflare's request-body cap (~100 MB free, 200 MB pro) silently
|
||||
// stalls "single-shot" uploads of large mod / scenario archives
|
||||
// that go through the panel's public hostname.
|
||||
// 2. The previous single-shot path buffered the entire body in memory
|
||||
// on both the controller and the agent, peaking at ~3× file size.
|
||||
// A 3 GB upload would thrash a smaller box.
|
||||
// 3. Single-shot uploads block the agent's bidi stream for the
|
||||
// duration of the transfer; chunks are small and interleaved.
|
||||
//
|
||||
// Wire protocol (all on the existing AgentEnvelope):
|
||||
//
|
||||
// FsWriteChunkRequest { upload_id, instance_id, path, offset, data,
|
||||
// total_size, is_final }
|
||||
// FsWriteChunkResult { bytes_received, total_received, finalized,
|
||||
// error }
|
||||
//
|
||||
// The agent owns one `uploadSession` per upload_id. Each session has an
|
||||
// `*os.File` open in its scratch dir. Chunks append in order. On
|
||||
// is_final the agent ships the file to the container via a streaming
|
||||
// CopyTarToContainer call (no full-file in-memory copy), then unlinks.
|
||||
// Stale sessions are reaped after uploadIdleTTL.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
const uploadIdleTTL = 30 * time.Minute
|
||||
|
||||
type uploadSession struct {
|
||||
mu sync.Mutex
|
||||
uploadID string
|
||||
instanceID string
|
||||
path string
|
||||
tempPath string
|
||||
f *os.File
|
||||
bytes int64
|
||||
total int64
|
||||
lastTouch time.Time
|
||||
}
|
||||
|
||||
func (d *Dispatcher) uploadScratchDir() string {
|
||||
dir := filepath.Join(d.dataRoot, ".panel-uploads")
|
||||
_ = os.MkdirAll(dir, 0o755)
|
||||
return dir
|
||||
}
|
||||
|
||||
// handleFsWriteChunk processes one chunk in the streaming-upload protocol.
|
||||
// First chunk creates the session; subsequent chunks append; is_final
|
||||
// triggers a streaming copy into the container and cleans up.
|
||||
func (d *Dispatcher) handleFsWriteChunk(corrID string, req *panelv1.FsWriteChunkRequest) {
|
||||
if req.UploadId == "" {
|
||||
d.sendChunkResult(corrID, 0, 0, false, "upload_id is required")
|
||||
return
|
||||
}
|
||||
if req.InstanceId == "" || req.Path == "" {
|
||||
d.sendChunkResult(corrID, 0, 0, false, "instance_id and path are required")
|
||||
return
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendChunkResult(corrID, 0, 0, false, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
sess, ok := d.uploads[req.UploadId]
|
||||
if !ok {
|
||||
// First chunk for this upload — create session + temp file.
|
||||
// Use the upload_id in the filename so a recovered scratch dir
|
||||
// is self-describing for forensic purposes.
|
||||
base := filepath.Join(d.uploadScratchDir(), "u-"+sanitizeUploadID(req.UploadId))
|
||||
f, err := os.OpenFile(base, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
d.mu.Unlock()
|
||||
d.sendChunkResult(corrID, 0, 0, false, fmt.Errorf("open temp: %w", err).Error())
|
||||
return
|
||||
}
|
||||
sess = &uploadSession{
|
||||
uploadID: req.UploadId,
|
||||
instanceID: req.InstanceId,
|
||||
path: req.Path,
|
||||
tempPath: base,
|
||||
f: f,
|
||||
total: req.TotalSize,
|
||||
lastTouch: time.Now(),
|
||||
}
|
||||
d.uploads[req.UploadId] = sess
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
sess.mu.Lock()
|
||||
defer sess.mu.Unlock()
|
||||
|
||||
// Reject stray chunks for a session that's already closed (final
|
||||
// chunk processed earlier — operator double-clicked, retry, etc.).
|
||||
if sess.f == nil {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, true, "")
|
||||
return
|
||||
}
|
||||
|
||||
// Allow operator to upload to a renamed path mid-stream? No — pin
|
||||
// the destination from the first chunk for safety. Subsequent
|
||||
// chunks with a different path get rejected.
|
||||
if sess.path != req.Path {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, false, "path differs from initial chunk")
|
||||
return
|
||||
}
|
||||
|
||||
// Append the chunk. The browser is expected to send chunks in
|
||||
// order; if offset doesn't match the running tail we err so the
|
||||
// frontend can fall back / retry rather than silently writing a
|
||||
// hole.
|
||||
if req.Offset != sess.bytes {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, false,
|
||||
fmt.Sprintf("chunk offset %d != expected %d (out-of-order chunk)", req.Offset, sess.bytes))
|
||||
return
|
||||
}
|
||||
n, err := sess.f.Write(req.Data)
|
||||
if err != nil {
|
||||
d.sendChunkResult(corrID, 0, sess.bytes, false, fmt.Errorf("write chunk: %w", err).Error())
|
||||
return
|
||||
}
|
||||
sess.bytes += int64(n)
|
||||
sess.lastTouch = time.Now()
|
||||
|
||||
if !req.IsFinal {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, "")
|
||||
return
|
||||
}
|
||||
|
||||
// Final chunk — close the temp file, ship it to the container.
|
||||
if err := sess.f.Sync(); err != nil {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, fmt.Errorf("fsync: %w", err).Error())
|
||||
return
|
||||
}
|
||||
if err := sess.f.Close(); err != nil {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, fmt.Errorf("close temp: %w", err).Error())
|
||||
return
|
||||
}
|
||||
sess.f = nil
|
||||
if sess.total > 0 && sess.bytes != sess.total {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false,
|
||||
fmt.Sprintf("size mismatch: received %d bytes, expected %d", sess.bytes, sess.total))
|
||||
return
|
||||
}
|
||||
|
||||
// Ship to the container (or host fallback) without re-reading the
|
||||
// whole file into memory — we hand Docker an io.Reader that
|
||||
// streams from disk, wrapped in a tar header on the fly.
|
||||
if err := d.deliverUploadedFile(rec, sess); err != nil {
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, false, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Tear down the session — temp file deleted, map slot freed.
|
||||
_ = os.Remove(sess.tempPath)
|
||||
d.mu.Lock()
|
||||
delete(d.uploads, sess.uploadID)
|
||||
d.mu.Unlock()
|
||||
|
||||
d.sendChunkResult(corrID, int64(n), sess.bytes, true, "")
|
||||
}
|
||||
|
||||
// deliverUploadedFile streams the temp file into the destination,
|
||||
// either via container CopyTarToContainer (running or helper sidecar)
|
||||
// or via a plain os.Rename for host-fallback instances. Designed so
|
||||
// the file's bytes never sit in memory in their entirety.
|
||||
func (d *Dispatcher) deliverUploadedFile(rec *instanceRecord, sess *uploadSession) error {
|
||||
if d.useContainerOps(rec) {
|
||||
abs, err := safeJoinAny(rec.BrowseableRoot, rootPaths(rec), sess.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), fsCopyTimeout)
|
||||
defer cancel()
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// mkdir -p the parent so the destination always exists.
|
||||
_, _, _, _ = d.runtime.ExecCapture(ctx, targetID, []string{"sh", "-c",
|
||||
"mkdir -p -- " + shellQuote(path.Dir(abs))})
|
||||
|
||||
// Open the temp file for read; build a tar wrapper on the fly
|
||||
// and feed Docker. CopyTarToContainer streams the body without
|
||||
// materializing it in memory.
|
||||
fr, err := os.Open(sess.tempPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fr.Close()
|
||||
fi, err := fr.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
tw := tar.NewWriter(pw)
|
||||
err := tw.WriteHeader(&tar.Header{
|
||||
Name: path.Base(abs),
|
||||
Mode: 0o644,
|
||||
Size: fi.Size(),
|
||||
})
|
||||
if err == nil {
|
||||
_, err = io.Copy(tw, fr)
|
||||
}
|
||||
if err == nil {
|
||||
err = tw.Close()
|
||||
}
|
||||
_ = pw.CloseWithError(err)
|
||||
errCh <- err
|
||||
}()
|
||||
if err := d.runtime.CopyTarToContainer(ctx, targetID, path.Dir(abs), pr); err != nil {
|
||||
if encErr := <-errCh; encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return fmt.Errorf("copy to container: %w", err)
|
||||
}
|
||||
if encErr := <-errCh; encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Host fallback — just move the temp file into place.
|
||||
if rec.DataPath == "" {
|
||||
return fmt.Errorf("no file storage available")
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, sess.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
// os.Rename is atomic on the same filesystem; if scratch lives on
|
||||
// a different fs we fall back to copy + delete.
|
||||
if err := os.Rename(sess.tempPath, abs); err == nil {
|
||||
return nil
|
||||
}
|
||||
src, err := os.Open(sess.tempPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
dst, err := os.OpenFile(abs, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
if _, err := io.Copy(dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadReaper closes + removes any session that hasn't been touched
|
||||
// in uploadIdleTTL. Runs forever; cheap enough to scan once a minute.
|
||||
func (d *Dispatcher) uploadReaper() {
|
||||
t := time.NewTicker(time.Minute)
|
||||
defer t.Stop()
|
||||
for range t.C {
|
||||
now := time.Now()
|
||||
d.mu.Lock()
|
||||
stale := []*uploadSession{}
|
||||
for id, s := range d.uploads {
|
||||
if now.Sub(s.lastTouch) > uploadIdleTTL {
|
||||
stale = append(stale, s)
|
||||
delete(d.uploads, id)
|
||||
}
|
||||
}
|
||||
d.mu.Unlock()
|
||||
for _, s := range stale {
|
||||
s.mu.Lock()
|
||||
if s.f != nil {
|
||||
_ = s.f.Close()
|
||||
s.f = nil
|
||||
}
|
||||
_ = os.Remove(s.tempPath)
|
||||
s.mu.Unlock()
|
||||
d.log.Info("upload session reaped", "upload_id", s.uploadID, "bytes", s.bytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sanitizeUploadID strips characters that could escape the scratch dir.
|
||||
// upload_id is generated by the browser (UUID-like) — defense in depth.
|
||||
func sanitizeUploadID(s string) string {
|
||||
out := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case (c >= 'a' && c <= 'z'), (c >= 'A' && c <= 'Z'), (c >= '0' && c <= '9'), c == '-', c == '_':
|
||||
out = append(out, c)
|
||||
}
|
||||
if len(out) > 64 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = []byte("anon")
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendChunkResult(corrID string, n, total int64, finalized bool, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_FsWriteChunkResult{
|
||||
FsWriteChunkResult: &panelv1.FsWriteChunkResult{
|
||||
BytesReceived: n,
|
||||
TotalReceived: total,
|
||||
Finalized: finalized,
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// hangGuard implements a belt-and-suspenders hang-detection guardrail for
|
||||
// ARK Survival Ascended (ark-sa) instances running under Wine 9.
|
||||
// The Wine 9 RCON listener can wedge under sustained traffic; the engine
|
||||
// also sometimes emits a "!!!HANG DETECTED!!!" line on its own.
|
||||
// This guard auto-restarts the docker container in either case, with a
|
||||
// cooldown to prevent flapping.
|
||||
// The root-cause fix (docker_exec_rcon adapter) shipped 2026-05-03; this
|
||||
// guard handles the rare residual cases.
|
||||
// It replaces the bash watchdog that previously ran on princess via systemd timer.
|
||||
|
||||
const (
|
||||
hangGuardWindow = 10 * time.Minute
|
||||
hangGuardFailThreshold = 5
|
||||
hangGuardCooldown = 10 * time.Minute
|
||||
hangGuardEngineMarker = "!!!HANG DETECTED!!!"
|
||||
)
|
||||
|
||||
// hangGuardState holds per-instance hang-detection state.
|
||||
// Embed by value in instanceRecord; zero value is ready to use.
|
||||
type hangGuardState struct {
|
||||
mu sync.Mutex
|
||||
consecutiveFails int
|
||||
windowStart time.Time
|
||||
lastRestart time.Time
|
||||
restarting atomic.Bool
|
||||
}
|
||||
|
||||
// arkHangGuardOnLogLine checks for engine-side hang detection tokens
|
||||
// in console log lines. Only applies to ark-sa instances.
|
||||
func (d *Dispatcher) arkHangGuardOnLogLine(rec *instanceRecord, line string) {
|
||||
if rec.ModuleID != "ark-sa" {
|
||||
return
|
||||
}
|
||||
if !strings.Contains(line, hangGuardEngineMarker) {
|
||||
return
|
||||
}
|
||||
// Engine self-detected a hang; trigger restart asynchronously.
|
||||
go d.arkHangGuardRestart(rec, fmt.Sprintf("engine emitted '%s'", hangGuardEngineMarker))
|
||||
}
|
||||
|
||||
// arkHangGuardOnRconResult tracks RCON call successes/failures for
|
||||
// hang detection. A threshold of consecutive failures triggers a restart.
|
||||
func (d *Dispatcher) arkHangGuardOnRconResult(rec *instanceRecord, success bool) {
|
||||
if rec.ModuleID != "ark-sa" {
|
||||
return
|
||||
}
|
||||
|
||||
rec.hangGuard.mu.Lock()
|
||||
defer rec.hangGuard.mu.Unlock()
|
||||
|
||||
if success {
|
||||
rec.hangGuard.consecutiveFails = 0
|
||||
rec.hangGuard.windowStart = time.Time{}
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if rec.hangGuard.windowStart.IsZero() || now.Sub(rec.hangGuard.windowStart) > hangGuardWindow {
|
||||
// Start a new window
|
||||
rec.hangGuard.windowStart = now
|
||||
rec.hangGuard.consecutiveFails = 1
|
||||
} else {
|
||||
rec.hangGuard.consecutiveFails++
|
||||
}
|
||||
|
||||
if rec.hangGuard.consecutiveFails >= hangGuardFailThreshold {
|
||||
go d.arkHangGuardRestart(rec, fmt.Sprintf("%d RCON failures in last %v", rec.hangGuard.consecutiveFails, hangGuardWindow))
|
||||
}
|
||||
}
|
||||
|
||||
// arkHangGuardRestart performs the actual container restart, respecting a
|
||||
// cooldown and using a single-flight pattern to prevent concurrent restarts.
|
||||
func (d *Dispatcher) arkHangGuardRestart(rec *instanceRecord, reason string) {
|
||||
// Single-flight: only one restart at a time per instance.
|
||||
if !rec.hangGuard.restarting.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
defer rec.hangGuard.restarting.Store(false)
|
||||
|
||||
// Lock only for cooldown check and counter reset. We do NOT set
|
||||
// lastRestart here — only after a successful runtime.Restart, so a
|
||||
// failed restart attempt doesn't lock out retries for the full
|
||||
// cooldown (would block the next legitimate hang signal). Resetting
|
||||
// consecutiveFails / windowStart up front is fine: the bounce is
|
||||
// either about to happen (success path will set lastRestart) or has
|
||||
// failed (caller logged + emitted; future signals will re-arm).
|
||||
rec.hangGuard.mu.Lock()
|
||||
if !rec.hangGuard.lastRestart.IsZero() && time.Since(rec.hangGuard.lastRestart) < hangGuardCooldown {
|
||||
remaining := hangGuardCooldown - time.Since(rec.hangGuard.lastRestart)
|
||||
d.log.Debug("hang-guard skipped (cooldown)",
|
||||
"instance_id", rec.InstanceID,
|
||||
"cooldown_remaining", remaining,
|
||||
)
|
||||
rec.hangGuard.mu.Unlock()
|
||||
return
|
||||
}
|
||||
rec.hangGuard.consecutiveFails = 0
|
||||
rec.hangGuard.windowStart = time.Time{}
|
||||
rec.hangGuard.mu.Unlock()
|
||||
|
||||
d.log.Warn("hang-guard restarting container",
|
||||
"instance_id", rec.InstanceID,
|
||||
"container_id", rec.ContainerID,
|
||||
"reason", reason,
|
||||
)
|
||||
|
||||
// Emit a panel log line announcing the restart.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] HANG-GUARD: " + reason,
|
||||
}},
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 2026-05-25: bumped grace 5s → 30s. The hang-guard's bounce can
|
||||
// fire mid-SaveWorld; the 5s window made it a near-guarantee that the
|
||||
// SIGKILL would land inside the SQLite-backed save's flush — same
|
||||
// torn-save pattern that the memory guardrail was producing. 30s lets
|
||||
// most in-flight saves finish before the engine dies; if the engine is
|
||||
// truly wedged it'll get SIGKILL'd after the 30s anyway. See
|
||||
// stats.go's emergencyStopGracePeriod for the matching rationale.
|
||||
if err := d.runtime.Restart(ctx, rec.ContainerID, 30*time.Second); err != nil {
|
||||
d.log.Error("hang-guard restart failed",
|
||||
"instance_id", rec.InstanceID,
|
||||
"err", err,
|
||||
)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] HANG-GUARD restart failed: " + err.Error(),
|
||||
}},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Successful restart — start the cooldown clock so we don't bounce
|
||||
// the same instance again for hangGuardCooldown. A failed restart
|
||||
// (above branch returned early) intentionally leaves lastRestart
|
||||
// untouched so the next hang signal can retry.
|
||||
rec.hangGuard.mu.Lock()
|
||||
rec.hangGuard.lastRestart = time.Now()
|
||||
rec.hangGuard.mu.Unlock()
|
||||
|
||||
d.log.Info("hang-guard restart issued",
|
||||
"instance_id", rec.InstanceID,
|
||||
)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] HANG-GUARD restart issued",
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package dispatch
|
||||
|
||||
// Tests for WI-05: heartbeats routed through the send queue (sendLoop as the
|
||||
// sole owner of stream.Send) and the single-flight, ctx-bound exit watcher.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// recordingSender captures every envelope the sendLoop delivers.
|
||||
type recordingSender struct {
|
||||
mu sync.Mutex
|
||||
envs []*panelv1.AgentEnvelope
|
||||
}
|
||||
|
||||
func (r *recordingSender) Send(e *panelv1.AgentEnvelope) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.envs = append(r.envs, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingSender) snapshot() []*panelv1.AgentEnvelope {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
out := make([]*panelv1.AgentEnvelope, len(r.envs))
|
||||
copy(out, r.envs)
|
||||
return out
|
||||
}
|
||||
|
||||
// TestSendHeartbeatGoesThroughQueue verifies SendHeartbeat enqueues onto
|
||||
// sendCh and the envelope reaches the sender via sendLoop — i.e. heartbeats
|
||||
// no longer need a direct stream.Send from the session goroutine.
|
||||
func TestSendHeartbeatGoesThroughQueue(t *testing.T) {
|
||||
d := newTestDispatcher()
|
||||
rs := &recordingSender{}
|
||||
d.SetSender(rs)
|
||||
|
||||
now := time.Now()
|
||||
d.SendHeartbeat(now)
|
||||
|
||||
deadline := time.After(2 * time.Second)
|
||||
for {
|
||||
for _, e := range rs.snapshot() {
|
||||
if hb := e.GetHeartbeat(); hb != nil {
|
||||
if got := hb.At.AsTime(); !got.Equal(now.UTC().Truncate(time.Nanosecond)) && got.Unix() != now.Unix() {
|
||||
t.Fatalf("heartbeat timestamp mismatch: got %v want %v", got, now)
|
||||
}
|
||||
return // delivered through the queue — pass
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("heartbeat never delivered through sendLoop")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeatIsDroppable pins the backpressure classification: a heartbeat
|
||||
// must be shed (not block the producer) when the buffer is full.
|
||||
func TestHeartbeatIsDroppable(t *testing.T) {
|
||||
env := &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_Heartbeat{Heartbeat: &panelv1.Heartbeat{}}}
|
||||
if !isDroppableEnvelope(env) {
|
||||
t.Fatal("heartbeat envelope must be droppable telemetry")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeExitRuntime implements just enough of runtime.Runtime for watchExit.
|
||||
// The embedded nil interface panics on any unexpected call — that's a test
|
||||
// failure signal, not a hazard.
|
||||
type fakeExitRuntime struct {
|
||||
runtime.Runtime
|
||||
mu sync.Mutex
|
||||
activeWaits int
|
||||
maxWaits int
|
||||
release chan struct{} // close to make Wait return exit code 0
|
||||
}
|
||||
|
||||
func (f *fakeExitRuntime) Wait(ctx context.Context, id string) (int, error) {
|
||||
f.mu.Lock()
|
||||
f.activeWaits++
|
||||
if f.activeWaits > f.maxWaits {
|
||||
f.maxWaits = f.activeWaits
|
||||
}
|
||||
f.mu.Unlock()
|
||||
defer func() {
|
||||
f.mu.Lock()
|
||||
f.activeWaits--
|
||||
f.mu.Unlock()
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return -1, ctx.Err()
|
||||
case <-f.release:
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeExitRuntime) InspectByName(ctx context.Context, name string) (runtime.RuntimeInstanceState, error) {
|
||||
return runtime.RuntimeInstanceState{Status: "exited"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeExitRuntime) counts() (active, max int) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.activeWaits, f.maxWaits
|
||||
}
|
||||
|
||||
func newExitWatchDispatcher(f *fakeExitRuntime) *Dispatcher {
|
||||
d := &Dispatcher{
|
||||
log: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
runtime: f,
|
||||
instances: map[string]*instanceRecord{},
|
||||
sendCh: make(chan *panelv1.AgentEnvelope, 256),
|
||||
}
|
||||
go d.sendLoop()
|
||||
return d
|
||||
}
|
||||
|
||||
// waitFor polls cond until true or the deadline elapses.
|
||||
func waitFor(t *testing.T, d time.Duration, cond func() bool, msg string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(d)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal(msg)
|
||||
}
|
||||
|
||||
// TestStartExitWatchSingleFlight re-arms the watcher many times (the shape of
|
||||
// Announce firing on every controller reconnect) and asserts only ONE
|
||||
// ContainerWait is ever in flight — the pre-fix behavior leaked one blocked
|
||||
// Wait (and its docker connection) per reconnect.
|
||||
func TestStartExitWatchSingleFlight(t *testing.T) {
|
||||
f := &fakeExitRuntime{release: make(chan struct{})}
|
||||
d := newExitWatchDispatcher(f)
|
||||
rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
|
||||
|
||||
for i := 0; i < 8; i++ {
|
||||
d.startExitWatch(rec)
|
||||
}
|
||||
// All superseded watchers must exit; exactly one survivor remains.
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 },
|
||||
"expected exactly 1 in-flight ContainerWait after re-arms")
|
||||
// The peak may briefly exceed 1 while a cancelled watcher unwinds, but it
|
||||
// must never approach the number of re-arms (pre-fix: 8 concurrent Waits).
|
||||
if _, max := f.counts(); max >= 8 {
|
||||
t.Fatalf("watchExit not single-flighted: peak concurrent waits = %d", max)
|
||||
}
|
||||
|
||||
// Cancelling the record's watcher (handleStop / Shutdown path) must
|
||||
// release the last Wait.
|
||||
d.mu.Lock()
|
||||
cancel := rec.ExitWatchCancel
|
||||
rec.ExitWatchCancel = nil
|
||||
d.mu.Unlock()
|
||||
cancel()
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 0 },
|
||||
"cancelled exit watcher did not release ContainerWait")
|
||||
}
|
||||
|
||||
// TestWatchExitCancelledEmitsNothing: a ctx-cancelled watcher must return
|
||||
// silently — no spurious stopped/crashed InstanceState for a container that
|
||||
// is still running.
|
||||
func TestWatchExitCancelledEmitsNothing(t *testing.T) {
|
||||
f := &fakeExitRuntime{release: make(chan struct{})}
|
||||
d := newExitWatchDispatcher(f)
|
||||
rs := &recordingSender{}
|
||||
d.SetSender(rs)
|
||||
rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
|
||||
|
||||
d.startExitWatch(rec)
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 }, "watcher never started")
|
||||
d.mu.Lock()
|
||||
cancel := rec.ExitWatchCancel
|
||||
d.mu.Unlock()
|
||||
cancel()
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 0 }, "watcher never exited")
|
||||
time.Sleep(50 * time.Millisecond) // allow any (wrong) emission to flush
|
||||
for _, e := range rs.snapshot() {
|
||||
if e.GetInstanceState() != nil {
|
||||
t.Fatalf("cancelled watchExit emitted InstanceState: %v", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchExitTerminalExit: when the container really exits (Wait returns
|
||||
// 0 and inspect says "exited"), the watcher emits a STOPPED state.
|
||||
func TestWatchExitTerminalExit(t *testing.T) {
|
||||
f := &fakeExitRuntime{release: make(chan struct{})}
|
||||
d := newExitWatchDispatcher(f)
|
||||
rs := &recordingSender{}
|
||||
d.SetSender(rs)
|
||||
rec := &instanceRecord{InstanceID: "x", ContainerID: "cid"}
|
||||
d.instances["x"] = rec
|
||||
|
||||
d.startExitWatch(rec)
|
||||
waitFor(t, 2*time.Second, func() bool { a, _ := f.counts(); return a == 1 }, "watcher never started")
|
||||
close(f.release)
|
||||
waitFor(t, 2*time.Second, func() bool {
|
||||
for _, e := range rs.snapshot() {
|
||||
if st := e.GetInstanceState(); st != nil && st.Status == panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}, "terminal exit never produced a STOPPED InstanceState")
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
)
|
||||
|
||||
// logCoalescer throttles repetitive progress-style log lines while letting
|
||||
// genuine content through unchanged.
|
||||
//
|
||||
// Why this exists: containers under install/update pressure (SteamCMD
|
||||
// download ticks, preallocation, LGSM step banners) can emit 40+ lines per
|
||||
// second. Forwarding every one to the controller saturates the gRPC send
|
||||
// window, fills the event bus's per-subscriber buffer (which silently drops
|
||||
// on overflow), and burns CPU re-rendering DOM. The browser only needs to
|
||||
// see the *latest* progress value, not every interstitial frame.
|
||||
//
|
||||
// Behaviour:
|
||||
// - Lines with progressKey() == "": emit immediately.
|
||||
// - Lines with a key: if it's been less than `interval` since the last
|
||||
// emit of that key, replace the currently-buffered line for that key.
|
||||
// A single background timer flushes any pending lines every `interval`.
|
||||
type logCoalescer struct {
|
||||
interval time.Duration
|
||||
emit func(stream runtime.LogStream, line string, at time.Time)
|
||||
|
||||
mu sync.Mutex
|
||||
pending map[string]*pendingLine
|
||||
lastAt map[string]time.Time
|
||||
timer *time.Timer
|
||||
stopped bool
|
||||
}
|
||||
|
||||
type pendingLine struct {
|
||||
stream runtime.LogStream
|
||||
line string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func newLogCoalescer(interval time.Duration, emit func(stream runtime.LogStream, line string, at time.Time)) *logCoalescer {
|
||||
c := &logCoalescer{
|
||||
interval: interval,
|
||||
emit: emit,
|
||||
pending: map[string]*pendingLine{},
|
||||
lastAt: map[string]time.Time{},
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// offer delivers a line to the coalescer. Safe to call concurrently, though
|
||||
// in practice the caller is the single log-scan goroutine.
|
||||
func (c *logCoalescer) offer(stream runtime.LogStream, line string, at time.Time) {
|
||||
key := progressKey(line)
|
||||
if key == "" {
|
||||
// Non-progress — emit immediately, but first flush any buffered
|
||||
// progress lines so they don't appear AFTER a later non-progress line.
|
||||
c.mu.Lock()
|
||||
c.flushLocked()
|
||||
c.mu.Unlock()
|
||||
c.emit(stream, line, at)
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.stopped {
|
||||
return
|
||||
}
|
||||
last := c.lastAt[key]
|
||||
if last.IsZero() || at.Sub(last) >= c.interval {
|
||||
// Enough quiet time — emit right away.
|
||||
c.emit(stream, line, at)
|
||||
c.lastAt[key] = at
|
||||
delete(c.pending, key)
|
||||
return
|
||||
}
|
||||
// Replace the buffered line for this key (we only care about the newest).
|
||||
c.pending[key] = &pendingLine{stream: stream, line: line, at: at}
|
||||
// Make sure the timer is armed. Single timer drains all pending keys.
|
||||
if c.timer == nil {
|
||||
c.timer = time.AfterFunc(c.interval, c.tick)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *logCoalescer) tick() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.flushLocked()
|
||||
// If more pending arrived during flush (or we expect more), re-arm the timer.
|
||||
if len(c.pending) > 0 && !c.stopped {
|
||||
c.timer = time.AfterFunc(c.interval, c.tick)
|
||||
} else {
|
||||
c.timer = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *logCoalescer) flushLocked() {
|
||||
now := time.Now()
|
||||
for key, p := range c.pending {
|
||||
c.emit(p.stream, p.line, p.at)
|
||||
c.lastAt[key] = now
|
||||
delete(c.pending, key)
|
||||
}
|
||||
}
|
||||
|
||||
// stop flushes any pending lines and prevents further work. Call from the
|
||||
// streamLogs goroutine's defer so a stopping instance doesn't leave buffered
|
||||
// progress frames undelivered.
|
||||
func (c *logCoalescer) stop() {
|
||||
c.mu.Lock()
|
||||
c.stopped = true
|
||||
if c.timer != nil {
|
||||
c.timer.Stop()
|
||||
c.timer = nil
|
||||
}
|
||||
c.flushLocked()
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// Regexes for lines we treat as part of a repeating progress stream.
|
||||
// Each group returns a stable key — same key = same progress counter.
|
||||
var (
|
||||
// SteamCMD: "Update state (0x61) downloading, progress: 47.95 (...)"
|
||||
// Also covers "verifying", "preallocating", "committing" stages.
|
||||
reSteamcmdProgress = regexp.MustCompile(`Update state \([^)]+\) (\w+), progress:`)
|
||||
// Generic "Progress: 64%" or "[ 64%] Installing ...".
|
||||
reGenericProgress = regexp.MustCompile(`^\s*(?:Progress:|\[\s*\d+%\s*\])`)
|
||||
// LGSM step banner: "Starting sdtdserver: <step>". vinanrra emits
|
||||
// several per step as the banner animates ([ .... ] → [ INFO ] → [ OK ]).
|
||||
reLGSMStarting = regexp.MustCompile(`^\s*Starting sdtdserver:`)
|
||||
reLGSMStopping = regexp.MustCompile(`^\s*Stopping sdtdserver:`)
|
||||
// Docker pull: "Downloading [=>] 12.3MB / 100MB"
|
||||
reDockerPull = regexp.MustCompile(`^\s*(?:Downloading|Extracting|Pulling)\s+[\[(]`)
|
||||
)
|
||||
|
||||
// progressKey returns a stable identifier for a progress-style line, or ""
|
||||
// if the line should be forwarded unchanged. The returned key collapses
|
||||
// every frame of the same counter onto a single slot so we keep only the
|
||||
// latest within each coalesce window.
|
||||
func progressKey(text string) string {
|
||||
if m := reSteamcmdProgress.FindStringSubmatch(text); m != nil {
|
||||
return "steamcmd:" + m[1]
|
||||
}
|
||||
if reGenericProgress.MatchString(text) {
|
||||
return "progress"
|
||||
}
|
||||
if reLGSMStarting.MatchString(text) {
|
||||
return "lgsm-starting"
|
||||
}
|
||||
if reLGSMStopping.MatchString(text) {
|
||||
return "lgsm-stopping"
|
||||
}
|
||||
if reDockerPull.MatchString(text) {
|
||||
return "docker-pull"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// instanceMeta is the on-disk sidecar written per instance after successful
|
||||
// Create. It lets the agent rehydrate its in-memory records after a restart,
|
||||
// so operator commands (stop, etc.) continue to work against instances that
|
||||
// outlived the agent process.
|
||||
//
|
||||
// Stored at <metaDir>/<instance_id>.json. Mode 0600 because it carries the
|
||||
// RCON password. Deleted on stop.
|
||||
type instanceMeta struct {
|
||||
InstanceID string `json:"instance_id"`
|
||||
ContainerID string `json:"container_id"`
|
||||
ModuleID string `json:"module_id"`
|
||||
// Branch is the normalized Steam branch the install came from (warm-seed
|
||||
// matching key). Empty on sidecars written before this feature.
|
||||
Branch string `json:"branch,omitempty"`
|
||||
DataPath string `json:"data_path"`
|
||||
BrowseableRoot string `json:"browseable_root,omitempty"`
|
||||
RCONAddr string `json:"rcon_addr,omitempty"`
|
||||
RCONPassword string `json:"rcon_password,omitempty"`
|
||||
ConfigValues map[string]string `json:"config_values,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (d *Dispatcher) metaPath(instanceID string) string {
|
||||
return filepath.Join(d.metaDir, instanceID+".json")
|
||||
}
|
||||
|
||||
func (d *Dispatcher) writeMeta(rec *instanceRecord) error {
|
||||
if d.metaDir == "" {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(d.metaDir, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir meta dir: %w", err)
|
||||
}
|
||||
m := instanceMeta{
|
||||
InstanceID: rec.InstanceID,
|
||||
ContainerID: rec.ContainerID,
|
||||
ModuleID: rec.ModuleID,
|
||||
Branch: rec.Branch,
|
||||
DataPath: rec.DataPath,
|
||||
BrowseableRoot: rec.BrowseableRoot,
|
||||
RCONAddr: rec.RCONAddr,
|
||||
RCONPassword: rec.RCONPassword,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
data, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal meta: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(d.metaPath(rec.InstanceID), data, 0o600); err != nil {
|
||||
return fmt.Errorf("write meta: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dispatcher) deleteMeta(instanceID string) {
|
||||
if d.metaDir == "" {
|
||||
return
|
||||
}
|
||||
if err := os.Remove(d.metaPath(instanceID)); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
d.log.Warn("delete meta failed", "instance_id", instanceID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) loadAllMeta() ([]instanceMeta, error) {
|
||||
if d.metaDir == "" {
|
||||
return nil, nil
|
||||
}
|
||||
entries, err := os.ReadDir(d.metaDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var out []instanceMeta
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
if filepath.Ext(name) != ".json" {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(d.metaDir, name))
|
||||
if err != nil {
|
||||
d.log.Warn("read meta failed", "file", name, "err", err)
|
||||
continue
|
||||
}
|
||||
var m instanceMeta
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
d.log.Warn("bad meta file", "file", name, "err", err)
|
||||
continue
|
||||
}
|
||||
if m.InstanceID == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// seedModsFromInstance clones the Mods/ folder of srcInstanceID (the cluster
|
||||
// master) onto dstInstanceID's install volume, REPLACING the destination's mod
|
||||
// set with the master's. Runs a throwaway debian sidecar that mounts both
|
||||
// instances' install volumes and tar-copies /src-game/Mods → /dst-game/Mods.
|
||||
//
|
||||
// This is the inverse of tryWarmSeedFromSibling (which copies the install but
|
||||
// EXCLUDES Mods): here we copy ONLY Mods, so a freshly-joined cluster member
|
||||
// ends up running exactly the master's mod set without an operator re-uploading
|
||||
// every mod by hand.
|
||||
//
|
||||
// Per-server STATE is preserved/excluded so a clone never leaks one server's
|
||||
// player data onto another:
|
||||
// - The master's RefugeBot homes.json (teleport homes, keyed by SteamID) is
|
||||
// EXCLUDED from the copy — it's per-server state, not mod config. This is
|
||||
// the exact file that leaked when an operator rsync'd Mods by hand.
|
||||
// - The destination's OWN homes.json is snapshotted before the wipe and
|
||||
// restored after, so the joining server keeps its own teleports.
|
||||
//
|
||||
// Both instances must live on THIS agent (same Docker host) — the sidecar
|
||||
// mounts local named volumes. If the master isn't found here the seed returns
|
||||
// an error and the caller treats it as best-effort (the server still boots with
|
||||
// whatever mods it already had).
|
||||
func (d *Dispatcher) seedModsFromInstance(ctx context.Context, srcInstanceID, dstInstanceID, dstDataPath string, manifest *modulepkg.Manifest) (int, error) {
|
||||
if manifest == nil {
|
||||
return 0, fmt.Errorf("mod-seed: nil manifest")
|
||||
}
|
||||
if srcInstanceID == dstInstanceID {
|
||||
return 0, fmt.Errorf("mod-seed: refusing to seed instance %q from itself", dstInstanceID)
|
||||
}
|
||||
installPath := ""
|
||||
if len(manifest.UpdateProviders) > 0 {
|
||||
installPath = manifest.UpdateProviders[0].InstallPath
|
||||
}
|
||||
if installPath == "" {
|
||||
installPath = manifest.Runtime.Docker.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return 0, fmt.Errorf("mod-seed: no install path for module %q", manifest.ID)
|
||||
}
|
||||
|
||||
// Resolve the master's data path from our local instance table.
|
||||
d.mu.Lock()
|
||||
srcRec, ok := d.instances[srcInstanceID]
|
||||
d.mu.Unlock()
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("mod-seed: master instance %q is not on this agent", srcInstanceID)
|
||||
}
|
||||
if srcRec.ModuleID != manifest.ID {
|
||||
return 0, fmt.Errorf("mod-seed: master %q is module %q, not %q", srcInstanceID, srcRec.ModuleID, manifest.ID)
|
||||
}
|
||||
|
||||
srcVolumes := agentmodule.ResolveVolumes(manifest, srcInstanceID, srcRec.DataPath)
|
||||
dstVolumes := agentmodule.ResolveVolumes(manifest, dstInstanceID, dstDataPath)
|
||||
srcGameVol := volumeNameForPath(srcVolumes, installPath)
|
||||
dstGameVol := volumeNameForPath(dstVolumes, installPath)
|
||||
if srcGameVol == "" || dstGameVol == "" {
|
||||
return 0, fmt.Errorf("mod-seed: install_path %q not a named volume on both instances", installPath)
|
||||
}
|
||||
|
||||
// tar, not rsync — debian:12-slim ships no rsync. Replace /dst-game/Mods
|
||||
// from /src-game/Mods, excluding the master's per-server homes.json, and
|
||||
// preserving the destination's own homes.json across the wipe.
|
||||
const seedScript = `set -e
|
||||
echo "mod-seed: cloning master Mods set (excl per-server homes.json)"
|
||||
HOMES=/dst-game/Mods/zzzzzzzRefugeBot/homes.json
|
||||
if [ -f "$HOMES" ]; then cp -a "$HOMES" /tmp/dst-homes.json; echo "mod-seed: preserved destination homes.json"; fi
|
||||
mkdir -p /dst-game/Mods
|
||||
find /dst-game/Mods -mindepth 1 -delete
|
||||
( cd /src-game/Mods && tar --exclude='./zzzzzzzRefugeBot/homes.json' -cf - . ) | ( cd /dst-game/Mods && tar -xf - )
|
||||
if [ -f /tmp/dst-homes.json ]; then mkdir -p /dst-game/Mods/zzzzzzzRefugeBot; cp -a /tmp/dst-homes.json "$HOMES"; echo "mod-seed: restored destination homes.json"; fi
|
||||
echo "mod-seed: $(ls /dst-game/Mods 2>/dev/null | wc -l) mods now present"
|
||||
echo SEED_OK
|
||||
`
|
||||
|
||||
sidecarID := fmt.Sprintf("%s-modseed", dstInstanceID)
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: "debian:12-slim",
|
||||
Command: []string{"bash", "-c", seedScript},
|
||||
RestartPolicy: "no",
|
||||
Volumes: []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true},
|
||||
{Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"},
|
||||
},
|
||||
}
|
||||
|
||||
d.log.Info("mod-seed: starting sidecar", "src", srcInstanceID, "dst", dstInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol)
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("mod-seed: create sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return 0, fmt.Errorf("mod-seed: start sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
var modCount int
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.log.Info("mod-seed log", "instance_id", dstInstanceID, "line", line)
|
||||
// Parse the sidecar's "mod-seed: N mods now present" echo so the
|
||||
// caller (and the cluster-sync UI) can report how many mods landed.
|
||||
if strings.Contains(line, "mods now present") {
|
||||
fields := strings.Fields(line)
|
||||
for i, f := range fields {
|
||||
if f == "mods" && i > 0 {
|
||||
if n, err := strconv.Atoi(fields[i-1]); err == nil {
|
||||
modCount = n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("mod-seed: wait sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return 0, fmt.Errorf("mod-seed: sidecar exited %d", exitCode)
|
||||
}
|
||||
d.log.Info("mod-seed: success", "src", srcInstanceID, "dst", dstInstanceID, "mods", modCount)
|
||||
return modCount, nil
|
||||
}
|
||||
|
||||
// handleSeedMods runs the mod-seed sidecar on demand — no container recreate,
|
||||
// no restart. Clones src_instance_id's Mods/ onto instance_id (excluding the
|
||||
// master's per-server homes.json, preserving the member's own). The member
|
||||
// keeps running on its currently-loaded mods; the freshly-copied files take
|
||||
// effect on its NEXT restart. Backs the cluster "Sync mods to members" action,
|
||||
// where the master pushes its current mod set to every other member at once.
|
||||
func (d *Dispatcher) handleSeedMods(ctx context.Context, corrID string, req *panelv1.SeedModsRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "src", req.SrcInstanceId, "correlation_id", corrID)
|
||||
reply := func(ok bool, modCount int, errMsg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_SeedModsResult{
|
||||
SeedModsResult: &panelv1.SeedModsResult{
|
||||
InstanceId: req.InstanceId,
|
||||
Ok: ok,
|
||||
ModCount: int32(modCount),
|
||||
Error: errMsg,
|
||||
},
|
||||
},
|
||||
})
|
||||
if !ok {
|
||||
log.Warn("mod-seed RPC failed", "err", errMsg)
|
||||
}
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
reply(false, 0, err.Error())
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
reply(false, 0, "module "+rec.ModuleID+" not in registry")
|
||||
return
|
||||
}
|
||||
modCount, err := d.seedModsFromInstance(ctx, req.SrcInstanceId, req.InstanceId, rec.DataPath, manifest)
|
||||
if err != nil {
|
||||
reply(false, 0, err.Error())
|
||||
return
|
||||
}
|
||||
log.Info("mod-seed RPC complete", "mods", modCount)
|
||||
reply(true, modCount, "")
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/pkg/regionmedic"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// Region Medic agent handlers — scan a 7DTD instance's active-world Region
|
||||
// directory for .7rg corruption, and heal one region from the newest clean
|
||||
// panel backup. Volume files are reached through the runtime's container-path
|
||||
// ops (the agent runs unprivileged, so it cannot touch /var/lib/docker/volumes
|
||||
// directly). The controller orchestrates stop→heal→start; handleRegionHeal
|
||||
// itself only performs the file swap and refuses to run on a live container.
|
||||
|
||||
// activeRegionDir resolves the container-absolute Region directory of the
|
||||
// instance's active world by reading GameWorld/GameName from the live
|
||||
// serverconfig.xml. Works on a stopped container (CopyFileFromContainer does).
|
||||
func (d *Dispatcher) activeRegionDir(ctx context.Context, container string) (string, error) {
|
||||
raw, err := d.runtime.CopyFileFromContainer(ctx, container, "/game-saves/serverconfig.xml")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read serverconfig.xml: %w", err)
|
||||
}
|
||||
world := xmlProp(string(raw), "GameWorld")
|
||||
game := xmlProp(string(raw), "GameName")
|
||||
if world == "" || game == "" {
|
||||
return "", fmt.Errorf("serverconfig.xml missing GameWorld/GameName (got %q/%q)", world, game)
|
||||
}
|
||||
// HOME is pinned to /game-saves by the 7dtd entrypoint.
|
||||
return path.Join("/game-saves/.local/share/7DaysToDie/Saves", world, game, "Region"), nil
|
||||
}
|
||||
|
||||
// xmlProp pulls the value from a 7DTD serverconfig line:
|
||||
//
|
||||
// <property name="GameWorld" value="Navezgane" />
|
||||
//
|
||||
// Tolerant of attribute order and single/double quotes; skips commented lines.
|
||||
func xmlProp(content, name string) string {
|
||||
for _, raw := range strings.Split(content, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "<!--") {
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(line, `name="`+name+`"`) && !strings.Contains(line, `name='`+name+`'`) {
|
||||
continue
|
||||
}
|
||||
i := strings.Index(line, "value=")
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
rest := line[i+len("value="):]
|
||||
if rest == "" {
|
||||
continue
|
||||
}
|
||||
q := rest[0]
|
||||
rest = rest[1:]
|
||||
if end := strings.IndexByte(rest, q); end >= 0 {
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// listRegionDirNames returns the file names in the container's Region dir.
|
||||
// Uses ExecCapture when the container is running; nil + error otherwise (the
|
||||
// scan path runs while the instance is up).
|
||||
func (d *Dispatcher) listRegionDirNames(ctx context.Context, container, regionDir string) ([]string, error) {
|
||||
stdout, _, _, err := d.runtime.ExecCapture(ctx, container, []string{"ls", "-1", regionDir})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var names []string
|
||||
for _, n := range strings.Split(string(stdout), "\n") {
|
||||
if n = strings.TrimSpace(n); n != "" {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// handleRegionScan reports corruption evidence for an instance's active world:
|
||||
// error_backup salvage files clustered into regions, plus structural validation
|
||||
// of each affected region's .7rg file.
|
||||
func (d *Dispatcher) handleRegionScan(ctx context.Context, corrID string, req *panelv1.RegionScanRequest) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "not_found", Message: err.Error()}})
|
||||
return
|
||||
}
|
||||
container := "panel-" + rec.InstanceID
|
||||
regionDir, err := d.activeRegionDir(ctx, container)
|
||||
if err != nil {
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "resolve_region_dir", Message: err.Error()}})
|
||||
return
|
||||
}
|
||||
names, err := d.listRegionDirNames(ctx, container, regionDir)
|
||||
if err != nil {
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{RegionDir: regionDir, Error: &panelv1.Error{Code: "list_region_dir", Message: err.Error()}})
|
||||
return
|
||||
}
|
||||
|
||||
// Cluster error_backups by region; note which region files exist.
|
||||
type agg struct {
|
||||
files int
|
||||
present bool
|
||||
}
|
||||
clusters := map[regionmedic.RegionCoord]*agg{}
|
||||
fileSet := map[regionmedic.RegionCoord]bool{}
|
||||
total := 0
|
||||
for _, n := range names {
|
||||
if cx, cz, ok := regionmedic.ParseErrorBackup(n); ok {
|
||||
total++
|
||||
rc := regionmedic.RegionForChunk(cx, cz)
|
||||
a := clusters[rc]
|
||||
if a == nil {
|
||||
a = &agg{}
|
||||
clusters[rc] = a
|
||||
}
|
||||
a.files++
|
||||
continue
|
||||
}
|
||||
if rc, ok := regionmedic.ParseRegionFileName(n); ok {
|
||||
fileSet[rc] = true
|
||||
}
|
||||
}
|
||||
|
||||
var affected []*panelv1.AffectedRegionMsg
|
||||
for rc, a := range clusters {
|
||||
msg := &panelv1.AffectedRegionMsg{
|
||||
Region: rc.String(),
|
||||
ErrorBackups: int32(a.files),
|
||||
FilePresent: fileSet[rc],
|
||||
}
|
||||
// Validate the affected region file (read via container-path op).
|
||||
if fileSet[rc] {
|
||||
if b, rerr := d.runtime.CopyFileFromContainer(ctx, container, path.Join(regionDir, rc.FileName())); rerr == nil {
|
||||
rep := regionmedic.ValidateRegionBytes(b, false)
|
||||
msg.FileCorrupt = !rep.Healthy()
|
||||
msg.BadChunks = int32(len(rep.BadChunks))
|
||||
}
|
||||
}
|
||||
affected = append(affected, msg)
|
||||
}
|
||||
sort.Slice(affected, func(i, j int) bool { return affected[i].ErrorBackups > affected[j].ErrorBackups })
|
||||
|
||||
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{
|
||||
ErrorBackups: int32(total),
|
||||
Affected: affected,
|
||||
RegionDir: regionDir,
|
||||
})
|
||||
}
|
||||
|
||||
// handleRegionHeal swaps one region file for the newest clean backup copy.
|
||||
// It REQUIRES the instance to be stopped (the controller stops it first) so the
|
||||
// running engine can't overwrite the swap. It snapshots the corrupt original
|
||||
// next to the region file before overwriting. Orchestration (stop/start) is the
|
||||
// controller's job.
|
||||
func (d *Dispatcher) handleRegionHeal(ctx context.Context, corrID string, req *panelv1.RegionHealRequest) {
|
||||
fail := func(code, msg string) {
|
||||
d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{Region: req.Region, Error: &panelv1.Error{Code: code, Message: msg}})
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
fail("not_found", err.Error())
|
||||
return
|
||||
}
|
||||
region, ok := regionmedic.ParseRegionFileName(regionFileArg(req.Region))
|
||||
if !ok {
|
||||
fail("bad_region", fmt.Sprintf("invalid region %q", req.Region))
|
||||
return
|
||||
}
|
||||
container := "panel-" + rec.InstanceID
|
||||
|
||||
// Refuse to swap a region on a LIVE container (it would be overwritten on
|
||||
// the next save). ExecCapture succeeds only when the container is running.
|
||||
if _, _, _, exErr := d.runtime.ExecCapture(ctx, container, []string{"true"}); exErr == nil {
|
||||
fail("instance_running", "instance must be stopped before healing a region")
|
||||
return
|
||||
}
|
||||
|
||||
regionDir, err := d.activeRegionDir(ctx, container)
|
||||
if err != nil {
|
||||
fail("resolve_region_dir", err.Error())
|
||||
return
|
||||
}
|
||||
liveFile := path.Join(regionDir, region.FileName())
|
||||
liveBytes, err := d.runtime.CopyFileFromContainer(ctx, container, liveFile)
|
||||
if err != nil {
|
||||
fail("read_live_region", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Find the newest clean backup copy of this region (backups are local to
|
||||
// the agent at <backupDir>/<instance>/).
|
||||
backups, err := regionmedic.ListPanelBackups(path.Join(d.backupDir, rec.InstanceID))
|
||||
if err != nil {
|
||||
fail("list_backups", err.Error())
|
||||
return
|
||||
}
|
||||
entry := regionmedic.SaveRelEntry(regionDir, region.FileName())
|
||||
best, _, err := regionmedic.FindLastGoodRegion(backups, entry, true, time.Time{})
|
||||
if err != nil {
|
||||
fail("find_backup", err.Error())
|
||||
return
|
||||
}
|
||||
if !best.Found || !best.Clean() {
|
||||
fail("no_clean_backup", "no backup contains a clean copy of this region")
|
||||
return
|
||||
}
|
||||
|
||||
// Snapshot the corrupt original alongside the region file (no new subdir, so
|
||||
// CopyFileToContainer needs no parent-dir creation on a stopped container).
|
||||
stamp := time.Now().UTC().Format("20060102-150405")
|
||||
snapName := region.FileName() + ".medic-corrupt-" + stamp
|
||||
snapPath := path.Join(regionDir, snapName)
|
||||
if err := d.runtime.CopyFileToContainer(ctx, container, snapPath, liveBytes); err != nil {
|
||||
fail("snapshot", err.Error())
|
||||
return
|
||||
}
|
||||
// Swap in the clean copy (overwrites the live region file in place).
|
||||
if err := d.runtime.CopyFileToContainer(ctx, container, liveFile, best.Bytes); err != nil {
|
||||
fail("write_region", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{
|
||||
Region: region.String(),
|
||||
SourceBackup: best.Backup.ID,
|
||||
BytesWritten: int64(len(best.Bytes)),
|
||||
SnapshotPath: snapPath,
|
||||
Healed: true,
|
||||
})
|
||||
}
|
||||
|
||||
// regionFileArg normalizes "r.X.Z" or "r.X.Z.7rg" to a .7rg filename.
|
||||
func regionFileArg(s string) string {
|
||||
if strings.HasSuffix(s, ".7rg") {
|
||||
return s
|
||||
}
|
||||
return s + ".7rg"
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendRegionScanResult(corrID string, res *panelv1.RegionScanResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_RegionScanResult{RegionScanResult: res},
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendRegionHealResult(corrID string, res *panelv1.RegionHealResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_RegionHealResult{RegionHealResult: res},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
)
|
||||
|
||||
func medicEnvOr(k, def string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// Medic host paths on the agent box (figaro, for 7DTD). Overridable via env so
|
||||
// the same agent binary works if 7DTD ever moves hosts.
|
||||
func regionRollingRoot() string {
|
||||
return medicEnvOr("PANEL_REGION_ROLLING_DIR", "/home/refuge/region-rolling")
|
||||
}
|
||||
func regionMedicBin() string {
|
||||
return medicEnvOr("PANEL_REGION_MEDIC_BIN", "/home/refuge/panel/bin/region-medic")
|
||||
}
|
||||
func discordTokenFile() string {
|
||||
return medicEnvOr("PANEL_DISCORD_TOKEN_FILE", "/home/refuge/panel/discord-bot-token")
|
||||
}
|
||||
|
||||
func medicFileExists(p string) bool {
|
||||
st, err := os.Stat(p)
|
||||
return err == nil && !st.IsDir()
|
||||
}
|
||||
|
||||
// runRegionMedicOnStart runs the autoheal CLI in a throwaway debian sidecar while
|
||||
// the 7DTD container is STILL STOPPED, just before the agent starts it:
|
||||
// - validate every .7rg in the active world's Region dir,
|
||||
// - heal corrupt ones from the newest clean rolling snapshot, then the panel
|
||||
// tar backups as a fallback,
|
||||
// - snapshot the now-clean Region dir to a fresh rolling slot,
|
||||
// - post a Discord summary (naming the server) if anything healed.
|
||||
//
|
||||
// The per-server medic config (enable / rolling-keep / Discord channel) lives in
|
||||
// the SAVES volume at region-medic.json — written by the panel's medic-config
|
||||
// endpoint via FsWrite (NO container recreate) and read by the CLI here in the
|
||||
// sidecar. The agent therefore always launches the sidecar for 7DTD and lets the
|
||||
// CLI decide; an absent/empty config just means defaults + no Discord.
|
||||
//
|
||||
// Best-effort by contract: any failure is logged + swallowed, never blocking the
|
||||
// start. 10-min hard cap so a wedged sidecar/daemon can't stall a boot.
|
||||
func (d *Dispatcher) runRegionMedicOnStart(ctx context.Context, rec *instanceRecord) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
log := d.log.With("instance_id", rec.InstanceID, "medic", true)
|
||||
medicBin := regionMedicBin()
|
||||
if !medicFileExists(medicBin) {
|
||||
log.Warn("region-medic: binary missing, skipping pre-start heal", "bin", medicBin)
|
||||
return
|
||||
}
|
||||
savesVol := fmt.Sprintf("panel-%s-saves", rec.InstanceID)
|
||||
rollingDir := path.Join(regionRollingRoot(), rec.InstanceID)
|
||||
_ = os.MkdirAll(rollingDir, 0o755)
|
||||
backupsDir := path.Join(d.backupDir, rec.InstanceID)
|
||||
|
||||
cmd := []string{"/rm", "autoheal",
|
||||
"-saves-root", "/saves",
|
||||
"-rolling-dir", "/rolling",
|
||||
"-keep", "2", // default; overridden by region-medic.json in the saves volume
|
||||
"-label", rec.InstanceID,
|
||||
"-apply",
|
||||
}
|
||||
vols := []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: savesVol, ContainerPath: "/saves"},
|
||||
{Type: "bind", HostPath: rollingDir, ContainerPath: "/rolling"},
|
||||
{Type: "bind", HostPath: medicBin, ContainerPath: "/rm", ReadOnly: true},
|
||||
}
|
||||
// debian:12-slim ships no CA bundle, so the CLI's HTTPS POST to Discord can't
|
||||
// verify the TLS cert ("x509: certificate signed by unknown authority"). Mount
|
||||
// the host's CA certs read-only — Go's x509 finds ca-certificates.crt there.
|
||||
if _, err := os.Stat("/etc/ssl/certs"); err == nil {
|
||||
vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: "/etc/ssl/certs", ContainerPath: "/etc/ssl/certs", ReadOnly: true})
|
||||
}
|
||||
// Panel tar backups are the heal fallback when no rolling slot has a clean
|
||||
// copy (e.g. very first boot, before any snapshot). Only mount if present.
|
||||
if st, err := os.Stat(backupsDir); err == nil && st.IsDir() {
|
||||
cmd = append(cmd, "-backups-dir", "/backups")
|
||||
vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: backupsDir, ContainerPath: "/backups", ReadOnly: true})
|
||||
}
|
||||
// The token is mounted whenever it exists; the CLI only posts if the saves
|
||||
// config names a channel. No -discord-channel flag — the CLI reads it from
|
||||
// /saves/region-medic.json.
|
||||
if tf := discordTokenFile(); medicFileExists(tf) {
|
||||
cmd = append(cmd, "-discord-token-file", "/discord-token")
|
||||
vols = append(vols, runtime.VolumeSpec{Type: "bind", HostPath: tf, ContainerPath: "/discord-token", ReadOnly: true})
|
||||
}
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: rec.InstanceID + "-medic",
|
||||
IsSidecar: true,
|
||||
Image: "debian:12-slim",
|
||||
Command: cmd,
|
||||
RestartPolicy: "no",
|
||||
Volumes: vols,
|
||||
}
|
||||
|
||||
log.Info("region-medic: pre-start scan+heal")
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
log.Warn("region-medic: create sidecar failed", "err", err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
log.Warn("region-medic: start sidecar failed", "err", err)
|
||||
return
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
log.Info("region-medic", "line", line)
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
log.Warn("region-medic: wait sidecar failed", "err", err)
|
||||
return
|
||||
}
|
||||
if exitCode != 0 {
|
||||
log.Warn("region-medic: sidecar exited non-zero", "code", exitCode)
|
||||
return
|
||||
}
|
||||
log.Info("region-medic: pre-start heal complete")
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// handleRenderConfig re-renders the module's declared config_files into the
|
||||
// instance's host DataPath — the same modulepkg.Render call handleCreate
|
||||
// makes, minus the container recreate. For modules that bind-mount the
|
||||
// rendered output into the container (7dtd's serverconfig.xml.rendered),
|
||||
// os.WriteFile rewrites the host file in place (same inode), so the running
|
||||
// container sees the new content immediately and the entrypoint applies it
|
||||
// on the NEXT boot. The container itself is never restarted.
|
||||
//
|
||||
// Containers created BEFORE a config_file's bind-mount was added to the
|
||||
// module manifest don't have the bind at all (docker pins mounts at create)
|
||||
// — their /game-saves/<file> is a plain file inside the saves volume. For
|
||||
// those we self-verify: read the file back through the container, and if it
|
||||
// doesn't match the fresh render, copy the rendered bytes in directly
|
||||
// (works running or stopped; the game only reads config at boot).
|
||||
func (d *Dispatcher) handleRenderConfig(ctx context.Context, corrID string, req *panelv1.InstanceRenderConfigRequest) {
|
||||
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
||||
fail := func(msg string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_InstanceRenderConfigResult{
|
||||
InstanceRenderConfigResult: &panelv1.InstanceRenderConfigResult{Error: msg},
|
||||
},
|
||||
})
|
||||
log.Warn("render config failed", "err", msg)
|
||||
}
|
||||
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
fail(err.Error())
|
||||
return
|
||||
}
|
||||
if rec.DataPath == "" {
|
||||
fail("instance has no data path (pre-feature sidecar?) — recreate once to adopt")
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
fail("module " + rec.ModuleID + " not in registry")
|
||||
return
|
||||
}
|
||||
|
||||
// Same merge handleCreate uses: generated secrets first, user values on
|
||||
// top (empties dropped → template `or` defaults kick in).
|
||||
values, err := mergeValuesAndSecrets(manifest, req.ConfigValues)
|
||||
if err != nil {
|
||||
fail("secrets: " + err.Error())
|
||||
return
|
||||
}
|
||||
// Branch-aware: a 3.0 instance renders serverconfig.v3.xml.tmpl, ≤2.6
|
||||
// keeps the default. Resolve from the instance's persisted branch (set at
|
||||
// create), falling back to the merged config_values' provider_id so a
|
||||
// durable config-render still picks the right template if the record
|
||||
// predates branch persistence.
|
||||
branch := rec.Branch
|
||||
if branch == "" {
|
||||
branch = resolveCreateBranch(manifest, values)
|
||||
}
|
||||
if err := modulepkg.RenderForBranch(manifest, rec.DataPath, values, branch); err != nil {
|
||||
fail("render: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Self-verify through the container; fall back to a direct copy into
|
||||
// the volume for pre-bind containers. The mapping host-path → container
|
||||
// path comes from the manifest's volume specs.
|
||||
volumes := agentmodule.ResolveVolumes(manifest, rec.InstanceID, rec.DataPath)
|
||||
container := "panel-" + rec.InstanceID
|
||||
var rendered []string
|
||||
for _, cf := range manifest.ConfigFiles {
|
||||
if cf.TemplateForBranch(branch) == "" {
|
||||
continue
|
||||
}
|
||||
rendered = append(rendered, cf.Path)
|
||||
hostPath := filepath.Join(rec.DataPath, cf.Path)
|
||||
ctrPath := ""
|
||||
for _, v := range volumes {
|
||||
if v.Type == "bind" && v.HostPath == hostPath {
|
||||
ctrPath = v.ContainerPath
|
||||
break
|
||||
}
|
||||
}
|
||||
if ctrPath == "" {
|
||||
continue // file isn't mounted into the container; host render is all there is
|
||||
}
|
||||
want, rerr := os.ReadFile(hostPath)
|
||||
if rerr != nil {
|
||||
fail("read rendered " + cf.Path + ": " + rerr.Error())
|
||||
return
|
||||
}
|
||||
got, gerr := d.runtime.CopyFileFromContainer(ctx, container, ctrPath)
|
||||
if gerr == nil && bytes.Equal(got, want) {
|
||||
continue // live bind-mount delivered the new render
|
||||
}
|
||||
// Pre-bind container (or unreadable): write the volume copy directly.
|
||||
if cerr := d.runtime.CopyFileToContainer(ctx, container, ctrPath, want); cerr != nil {
|
||||
fail("container has no live bind for " + cf.Path + " and direct copy failed: " + cerr.Error())
|
||||
return
|
||||
}
|
||||
log.Info("rendered config copied into pre-bind container volume", "file", cf.Path, "container_path", ctrPath)
|
||||
}
|
||||
log.Info("re-rendered config files (durable, applies next boot)", "files", rendered)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_InstanceRenderConfigResult{
|
||||
InstanceRenderConfigResult: &panelv1.InstanceRenderConfigResult{RenderedFiles: rendered},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// blockingSender simulates a controller that has stopped draining the stream:
|
||||
// every Send blocks until released. This is the backpressure condition that
|
||||
// used to wedge the whole agent when sendEnv called Send inline.
|
||||
type blockingSender struct {
|
||||
release chan struct{}
|
||||
mu sync.Mutex
|
||||
sent int
|
||||
}
|
||||
|
||||
func (b *blockingSender) Send(*panelv1.AgentEnvelope) error {
|
||||
<-b.release // block until the test lets it through
|
||||
b.mu.Lock()
|
||||
b.sent++
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func logEnv() *panelv1.AgentEnvelope {
|
||||
return &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{Line: "x"}}}
|
||||
}
|
||||
|
||||
func stateEnv() *panelv1.AgentEnvelope {
|
||||
return &panelv1.AgentEnvelope{Payload: &panelv1.AgentEnvelope_InstanceState{InstanceState: &panelv1.InstanceStateUpdate{InstanceId: "i"}}}
|
||||
}
|
||||
|
||||
// newTestDispatcher builds a Dispatcher with just the send machinery wired —
|
||||
// enough to exercise sendEnv / sendLoop without a real runtime.
|
||||
func newTestDispatcher() *Dispatcher {
|
||||
d := &Dispatcher{sendCh: make(chan *panelv1.AgentEnvelope, 4096)}
|
||||
go d.sendLoop()
|
||||
return d
|
||||
}
|
||||
|
||||
// TestSendEnvDropsTelemetryUnderBackpressure is the core regression guard for
|
||||
// the VEIN-exposed wedge: with the sender blocked, a flood of log envelopes far
|
||||
// exceeding the buffer must NOT block the producer. Pre-fix, sendEnv called
|
||||
// Send inline and the producer (and the whole agent) froze here.
|
||||
func TestSendEnvDropsTelemetryUnderBackpressure(t *testing.T) {
|
||||
d := newTestDispatcher()
|
||||
bs := &blockingSender{release: make(chan struct{})}
|
||||
d.SetSender(bs)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
// 50k log frames — 12x the 4096 buffer. Must all return promptly.
|
||||
for i := 0; i < 50000; i++ {
|
||||
d.sendEnv(logEnv())
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Producer completed without blocking on the stalled sender. Pass.
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("sendEnv blocked under backpressure — the wedge regressed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCriticalEnvelopeNotDropped verifies an instance-state envelope is queued
|
||||
// (not silently dropped) even when telemetry is being shed. We fill the buffer
|
||||
// with the sender blocked, then confirm a critical send blocks until drained
|
||||
// (i.e. it's preserved, not discarded).
|
||||
func TestCriticalEnvelopeNotDropped(t *testing.T) {
|
||||
d := &Dispatcher{sendCh: make(chan *panelv1.AgentEnvelope, 2)}
|
||||
// No drainer started yet — fill the buffer.
|
||||
d.sendEnv(logEnv())
|
||||
d.sendEnv(logEnv())
|
||||
// Buffer is full (cap 2). A telemetry frame must drop (non-blocking).
|
||||
dropDone := make(chan struct{})
|
||||
go func() { d.sendEnv(logEnv()); close(dropDone) }()
|
||||
select {
|
||||
case <-dropDone:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("telemetry send blocked on full buffer — should have dropped")
|
||||
}
|
||||
// A critical frame must NOT drop: it blocks until space frees. Start a
|
||||
// drainer to free space and confirm the critical send then completes.
|
||||
critDone := make(chan struct{})
|
||||
go func() { d.sendEnv(stateEnv()); close(critDone) }()
|
||||
select {
|
||||
case <-critDone:
|
||||
t.Fatal("critical send returned before buffer had space — it was dropped")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// Correctly still blocking. Now drain and it should complete.
|
||||
}
|
||||
go d.sendLoop()
|
||||
select {
|
||||
case <-critDone:
|
||||
// Drained and delivered. Pass.
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("critical send never completed after drain")
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrentProducersNoRace runs many producers through sendEnv at once.
|
||||
// Pre-fix, each called stream.Send directly — concurrent Sends on one gRPC
|
||||
// stream are a data race. The single-drainer design serializes them. Run with
|
||||
// -race to catch a regression.
|
||||
func TestConcurrentProducersNoRace(t *testing.T) {
|
||||
d := newTestDispatcher()
|
||||
bs := &blockingSender{release: make(chan struct{})}
|
||||
close(bs.release) // never block — just count
|
||||
d.SetSender(bs)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for p := 0; p < 32; p++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 1000; i++ {
|
||||
d.sendEnv(logEnv())
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
// Give the drainer a moment to flush.
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
// No assertion on exact count (some may still be in flight / dropped);
|
||||
// the point is the -race detector finds no concurrent Send.
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Per-instance stats polling. Docker streams a JSON frame ~once per
|
||||
// second; we throttle to one upstream emit per statsEmitInterval so the
|
||||
// event bus isn't flooded. CPU% uses the frame's cur−previous delta
|
||||
// directly (Docker includes precpu_stats on streamed frames).
|
||||
|
||||
// 3s is snappy enough to watch RAM climb during ARK world-load (12-15 GB
|
||||
// in under a minute) without flooding the SSE bus. Older 10s value made
|
||||
// the dashboard feel laggy when memory was actually moving fast.
|
||||
const statsEmitInterval = 3 * time.Second
|
||||
|
||||
// ARK SA memory guardrails — two-stage so we can flush saves cleanly
|
||||
// before having to SIGKILL.
|
||||
//
|
||||
// Healthy ARK SA world: 10-15 GB resident. 16-30 GB during save-flush
|
||||
// peaks is normal on a populated cluster. 45 GB is the warning level;
|
||||
// 60 GB is the hard kill that protects the host from OOM (which would
|
||||
// take down all sibling ARKs + postgres + the panel itself).
|
||||
//
|
||||
// 2026-05-25 history: the previous single-threshold 40 GB → 5s SIGKILL
|
||||
// path corrupted at least three SQLite-backed `.ark` saves over the
|
||||
// 2026-05-19 → 2026-05-25 window because the SIGKILL hit while a
|
||||
// SaveWorld was mid-flush (FAtlasSaveManager::LoadOperationSql crash on
|
||||
// next boot — null page read in the half-written SQLite store).
|
||||
//
|
||||
// The two-stage guard:
|
||||
//
|
||||
// At arkRAMWarnLimit (45 GB) — fired AT MOST ONCE per container life:
|
||||
// 1. Broadcast a 60s heads-up to in-game players via RCON
|
||||
// 2. SaveWorld (lets ARK flush cleanly)
|
||||
// 3. Wait 60s for the save + warning window
|
||||
// 4. runtime.Stop with 120s grace (clean shutdown path)
|
||||
// The container then stays Exited; operator restarts when ready, no
|
||||
// torn save, no progress loss beyond what the SaveWorld captured.
|
||||
//
|
||||
// At arkRAMHardLimit (60 GB) — fires if the warn path didn't help:
|
||||
// SIGKILL via runtime.Stop with 30s grace. Save MAY be torn here;
|
||||
// this is the host-protection layer, accepted as last-resort.
|
||||
//
|
||||
// The warn level fires at most once per container life via the
|
||||
// rec.memGuardWarned atomic — once we've committed to a graceful stop,
|
||||
// the hard limit isn't allowed to also fire and interrupt the SaveWorld.
|
||||
const arkRAMWarnLimit uint64 = 45 * 1024 * 1024 * 1024
|
||||
const arkRAMHardLimit uint64 = 60 * 1024 * 1024 * 1024
|
||||
|
||||
// gracefulStopGracePeriod is the docker-stop grace given to the
|
||||
// graceful warn path. ARK's SaveWorld can take 30-60s on a populated
|
||||
// world; 120s is enough headroom for the flush + the watchdog's DoExit.
|
||||
const gracefulStopGracePeriod = 120 * time.Second
|
||||
|
||||
// emergencyStopGracePeriod is the docker-stop grace for the 60 GB hard
|
||||
// path. 30s (was 5s) gives the engine a real shot at flushing in-flight
|
||||
// state before SIGKILL — the host-OOM risk that motivated the original
|
||||
// 5s grace doesn't materialize this fast in practice.
|
||||
const emergencyStopGracePeriod = 30 * time.Second
|
||||
|
||||
// dockerStatsFrame is the minimum subset of Docker's /containers/<id>/stats
|
||||
// response we need. Kept as a local shape so we don't depend on which
|
||||
// version of github.com/docker/docker renamed the struct this week.
|
||||
type dockerStatsFrame struct {
|
||||
CPUStats struct {
|
||||
// IMPORTANT: every nested field needs an explicit json tag.
|
||||
// Go's unmarshal is case-insensitive but does NOT translate
|
||||
// CamelCase ↔ snake_case. Without `json:"total_usage"`, the
|
||||
// field reads as 0 because Docker emits the snake_case key
|
||||
// and we'd be looking for "TotalUsage" / "totalusage". This
|
||||
// bug shipped for months — every container reported 0% CPU.
|
||||
CPUUsage struct {
|
||||
TotalUsage uint64 `json:"total_usage"`
|
||||
} `json:"cpu_usage"`
|
||||
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
||||
OnlineCPUs uint32 `json:"online_cpus"`
|
||||
} `json:"cpu_stats"`
|
||||
PreCPUStats struct {
|
||||
CPUUsage struct {
|
||||
TotalUsage uint64 `json:"total_usage"`
|
||||
} `json:"cpu_usage"`
|
||||
SystemCPUUsage uint64 `json:"system_cpu_usage"`
|
||||
} `json:"precpu_stats"`
|
||||
MemoryStats struct {
|
||||
Usage uint64 `json:"usage"`
|
||||
Limit uint64 `json:"limit"`
|
||||
// Docker's `usage` includes file-system page cache, which can
|
||||
// balloon to multi-GB on read-heavy workloads (ARK SA's mod +
|
||||
// asset reads inflate it to 25 GB on a 12 GB RSS workload).
|
||||
// `docker stats` CLI subtracts cache/inactive_file to match
|
||||
// what operators expect. We mirror that.
|
||||
// - cgroups v1: usage - stats.cache
|
||||
// - cgroups v2: usage - stats.inactive_file
|
||||
// Both keys are in the same nested map; whichever is present
|
||||
// is the one our cgroup version exposes.
|
||||
Stats struct {
|
||||
Cache uint64 `json:"cache"` // cgroups v1
|
||||
InactiveFile uint64 `json:"inactive_file"` // cgroups v2
|
||||
RSS uint64 `json:"rss"` // cgroups v1
|
||||
Anon uint64 `json:"anon"` // cgroups v2 (active anon)
|
||||
} `json:"stats"`
|
||||
} `json:"memory_stats"`
|
||||
Networks map[string]struct {
|
||||
RxBytes uint64 `json:"rx_bytes"`
|
||||
TxBytes uint64 `json:"tx_bytes"`
|
||||
} `json:"networks"`
|
||||
PidsStats struct {
|
||||
Current uint32 `json:"current"`
|
||||
} `json:"pids_stats"`
|
||||
}
|
||||
|
||||
func (d *Dispatcher) streamStats(ctx context.Context, rec *instanceRecord) {
|
||||
rc, err := d.runtime.StatsStream(ctx, rec.ContainerID)
|
||||
if err != nil {
|
||||
d.log.Warn("stats stream open failed", "instance_id", rec.InstanceID, "err", err)
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
d.log.Info("stats stream opened", "instance_id", rec.InstanceID)
|
||||
emitCount := 0
|
||||
defer func() { d.log.Info("stats stream closed", "instance_id", rec.InstanceID, "emitted", emitCount) }()
|
||||
|
||||
dec := json.NewDecoder(rc)
|
||||
lastEmit := time.Time{}
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var f dockerStatsFrame
|
||||
if err := dec.Decode(&f); err != nil {
|
||||
if err != io.EOF && ctx.Err() == nil {
|
||||
d.log.Debug("stats decode ended", "instance_id", rec.InstanceID, "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Since(lastEmit) < statsEmitInterval {
|
||||
continue
|
||||
}
|
||||
memUsed := d.sendStats(rec.InstanceID, &f)
|
||||
lastEmit = time.Now()
|
||||
emitCount++
|
||||
// ARK runaway-memory guardrails. Two stages: warn (45 GB) tries a
|
||||
// graceful flush; hard (60 GB) SIGKILLs to protect the host.
|
||||
// Both paths set rec.stopping inside their handler so subsequent
|
||||
// samples short-circuit there.
|
||||
if rec.ModuleID == "ark-sa" {
|
||||
if memUsed > arkRAMHardLimit {
|
||||
go d.emergencyRestart(rec, fmt.Sprintf(
|
||||
"ARK RAM hit %.1f GB (hard limit %d GB) — bouncing to protect host",
|
||||
float64(memUsed)/1024/1024/1024, arkRAMHardLimit/1024/1024/1024))
|
||||
return
|
||||
}
|
||||
if memUsed > arkRAMWarnLimit && rec.memGuardWarned.CompareAndSwap(false, true) {
|
||||
go d.gracefulMemoryRestart(rec, fmt.Sprintf(
|
||||
"ARK RAM hit %.1f GB (warn limit %d GB) — flushing save + graceful restart",
|
||||
float64(memUsed)/1024/1024/1024, arkRAMWarnLimit/1024/1024/1024))
|
||||
// Don't return — keep streaming so the hard limit can still
|
||||
// fire if RAM keeps climbing past 60 GB during the 60s flush
|
||||
// window. The memGuardWarned latch prevents the warn path
|
||||
// from re-arming, but the hard path is the host-protection
|
||||
// backstop and must remain live.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendStats(instanceID string, f *dockerStatsFrame) uint64 {
|
||||
// CPU%: delta total / delta system * online_cpus * 100.
|
||||
// First frame's precpu_stats is zero so cpuPct comes out 0 — fine.
|
||||
var cpuPct float64
|
||||
cpuDelta := int64(f.CPUStats.CPUUsage.TotalUsage) - int64(f.PreCPUStats.CPUUsage.TotalUsage)
|
||||
sysDelta := int64(f.CPUStats.SystemCPUUsage) - int64(f.PreCPUStats.SystemCPUUsage)
|
||||
hostCpus := f.CPUStats.OnlineCPUs
|
||||
if sysDelta > 0 && cpuDelta > 0 {
|
||||
cpus := float64(hostCpus)
|
||||
if cpus < 1 {
|
||||
cpus = 1
|
||||
}
|
||||
cpuPct = float64(cpuDelta) / float64(sysDelta) * cpus * 100.0
|
||||
}
|
||||
|
||||
var rx, tx uint64
|
||||
for _, n := range f.Networks {
|
||||
rx += n.RxBytes
|
||||
tx += n.TxBytes
|
||||
}
|
||||
|
||||
// "Real" memory: subtract file cache (cgroups v1) or inactive_file
|
||||
// (cgroups v2). Falls through to raw usage when neither is reported
|
||||
// (Windows containers, exotic runtimes). This is what `docker stats`
|
||||
// shows and what operators expect — the inflated ARK numbers (~25 GB
|
||||
// vs the actual ~12 GB RSS) were the cache-included variant.
|
||||
memUsed := f.MemoryStats.Usage
|
||||
if f.MemoryStats.Stats.InactiveFile > 0 && f.MemoryStats.Stats.InactiveFile <= memUsed {
|
||||
memUsed -= f.MemoryStats.Stats.InactiveFile
|
||||
} else if f.MemoryStats.Stats.Cache > 0 && f.MemoryStats.Stats.Cache <= memUsed {
|
||||
memUsed -= f.MemoryStats.Stats.Cache
|
||||
}
|
||||
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_InstanceStats{
|
||||
InstanceStats: &panelv1.InstanceStatsUpdate{
|
||||
InstanceId: instanceID,
|
||||
CpuPercent: cpuPct,
|
||||
MemUsedBytes: memUsed,
|
||||
MemLimitBytes: f.MemoryStats.Limit,
|
||||
NetRxBytes: rx,
|
||||
NetTxBytes: tx,
|
||||
Pids: f.PidsStats.Current,
|
||||
HostCpus: hostCpus,
|
||||
At: timestamppb.Now(),
|
||||
},
|
||||
},
|
||||
})
|
||||
return memUsed
|
||||
}
|
||||
|
||||
// gracefulMemoryRestart is the 45 GB warn path. Tries to land a clean
|
||||
// SaveWorld before bouncing the container so the next boot doesn't face
|
||||
// a torn SQLite save. Sequence:
|
||||
//
|
||||
// 1. Latch rec.stopping so other paths back off.
|
||||
// 2. Emit a panel log line + INSTANCE_STATUS_STOPPING state.
|
||||
// 3. RCON Broadcast: "[Panel] high memory — saving and restarting in 60s"
|
||||
// 4. RCON SaveWorld
|
||||
// 5. Sleep 60s (gives players warning + engine time to write the save)
|
||||
// 6. Cancel tracker/log goroutines, runtime.Restart with 120s grace.
|
||||
//
|
||||
// Uses Restart NOT Stop: a plain Stop on an `unless-stopped` container
|
||||
// is honored by docker as an operator stop — the container then stays
|
||||
// Exited until someone manually restarts it. That regression took
|
||||
// Ragnarok offline for 17 hours after the 02:02 PDT warn fire on
|
||||
// 2026-05-25 (see HANDOFF "Last meaningfully updated"). Restart bounces
|
||||
// the container as a transient event so it comes back up automatically;
|
||||
// the operator just sees a brief blip + the "[panel] MEMORY-WARN" line.
|
||||
//
|
||||
// If any RCON step fails (engine wedged, RCON socket dead), fall through
|
||||
// to the restart anyway — we still want the engine down + back up. The
|
||||
// 120s docker grace gives the engine a real shot at a clean DoExit even
|
||||
// when RCON is the broken thing.
|
||||
func (d *Dispatcher) gracefulMemoryRestart(rec *instanceRecord, reason string) {
|
||||
if !rec.stopping.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
d.log.Warn("ark memory warn — graceful restart", "instance_id", rec.InstanceID, "reason", reason)
|
||||
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] MEMORY-WARN: " + reason,
|
||||
}},
|
||||
})
|
||||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPING, 0, "graceful memory restart: "+reason)
|
||||
|
||||
// Best-effort RCON broadcast + SaveWorld. Use a short per-call
|
||||
// timeout so a wedged RCON socket doesn't block the whole graceful
|
||||
// path. tracker.Exec redials internally, so a closed socket from
|
||||
// idle-disconnect won't burn the attempt.
|
||||
if rec.Tracker != nil {
|
||||
bctx, bcancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if _, err := rec.Tracker.Exec(bctx, "Broadcast [Panel] High memory detected. Server saving and restarting in 60 seconds."); err != nil {
|
||||
d.log.Warn("rcon broadcast failed during graceful memory restart", "instance_id", rec.InstanceID, "err", err)
|
||||
}
|
||||
bcancel()
|
||||
|
||||
sctx, scancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if _, err := rec.Tracker.Exec(sctx, "SaveWorld"); err != nil {
|
||||
d.log.Warn("rcon SaveWorld failed during graceful memory restart", "instance_id", rec.InstanceID, "err", err)
|
||||
}
|
||||
scancel()
|
||||
}
|
||||
|
||||
// Wait the warning window — players see the broadcast, engine
|
||||
// finishes writing the save. Sleeping in the goroutine is fine;
|
||||
// the stats loop continues to monitor for the 60 GB hard limit
|
||||
// in case RAM keeps climbing during the wait.
|
||||
time.Sleep(60 * time.Second)
|
||||
|
||||
// Cancel ancillary goroutines so they don't fight the bounce.
|
||||
// They'll be re-armed by watchExit when the restart settles.
|
||||
d.mu.Lock()
|
||||
if rec.TrackerCancel != nil {
|
||||
rec.TrackerCancel()
|
||||
rec.TrackerCancel = nil
|
||||
}
|
||||
if rec.LogCancel != nil {
|
||||
rec.LogCancel()
|
||||
rec.LogCancel = nil
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gracefulStopGracePeriod+30*time.Second)
|
||||
defer cancel()
|
||||
if err := d.runtime.Restart(ctx, rec.ContainerID, gracefulStopGracePeriod); err != nil {
|
||||
d.log.Error("graceful memory restart runtime failure", "instance_id", rec.InstanceID, "err", err)
|
||||
// Restart failed — surface to operator so they know to investigate.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] MEMORY-WARN restart failed: " + err.Error(),
|
||||
}},
|
||||
})
|
||||
}
|
||||
// Clear both the stopping and memGuardWarned latches so future
|
||||
// memory-warn fires can re-arm on this container's next life.
|
||||
rec.stopping.Store(false)
|
||||
rec.memGuardWarned.Store(false)
|
||||
}
|
||||
|
||||
// emergencyRestart bounces a runaway container outside the normal
|
||||
// handleStop path. Difference from handleStop: 30s grace (not 60s),
|
||||
// runs even when no operator request is in flight, and emits a clear
|
||||
// log line so the dashboard's Console tab shows WHY this happened.
|
||||
//
|
||||
// Uses Restart NOT Stop — see gracefulMemoryRestart for the rationale.
|
||||
// A plain Stop on `unless-stopped` containers parks them Exited until
|
||||
// manually started; the panel operator has no way to know they need to
|
||||
// click Start. Bouncing the container keeps the service available, lets
|
||||
// the engine recover into a fresh process with no leak state, and the
|
||||
// "[panel] AUTO-KILL" log line is preserved for after-the-fact diagnosis.
|
||||
// If the underlying leak is persistent, RAM will climb again and the
|
||||
// guard will fire again — that's the operator's signal to investigate.
|
||||
//
|
||||
// 2026-05-25: bumped from 5s → 30s grace. The 5s window guaranteed
|
||||
// SIGKILL mid-save when this fired during a SaveWorld, and that's
|
||||
// what was producing the FAtlasSaveManager corruption pattern. 30s
|
||||
// is enough for the engine to finish a save in flight without giving
|
||||
// up host-OOM protection.
|
||||
//
|
||||
// Idempotent via rec.stopping: the stats loop fires this from a goroutine
|
||||
// every 3s while RAM is over the limit, but the second+ invocations
|
||||
// short-circuit at the stopping check so we don't pile up Stop calls
|
||||
// against a container that's already heading down.
|
||||
func (d *Dispatcher) emergencyRestart(rec *instanceRecord, reason string) {
|
||||
if !rec.stopping.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
d.log.Warn("emergency restart", "instance_id", rec.InstanceID, "reason", reason)
|
||||
|
||||
// Surface to operator via log line + state update.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] AUTO-KILL: " + reason,
|
||||
}},
|
||||
})
|
||||
d.sendInstanceState(rec.InstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPING, 0, "auto-kill: "+reason)
|
||||
|
||||
// Cancel ancillary goroutines so they don't fight the bounce.
|
||||
// They'll be re-armed by watchExit when the restart settles.
|
||||
d.mu.Lock()
|
||||
if rec.TrackerCancel != nil {
|
||||
rec.TrackerCancel()
|
||||
rec.TrackerCancel = nil
|
||||
}
|
||||
if rec.LogCancel != nil {
|
||||
rec.LogCancel()
|
||||
rec.LogCancel = nil
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), emergencyStopGracePeriod+30*time.Second)
|
||||
defer cancel()
|
||||
if err := d.runtime.Restart(ctx, rec.ContainerID, emergencyStopGracePeriod); err != nil {
|
||||
d.log.Error("emergency restart runtime failure", "instance_id", rec.InstanceID, "err", err)
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "stderr",
|
||||
At: timestamppb.Now(),
|
||||
Line: "[panel] AUTO-KILL restart failed: " + err.Error(),
|
||||
}},
|
||||
})
|
||||
}
|
||||
// Clear both latches so future signals can re-arm on the bounced container.
|
||||
rec.stopping.Store(false)
|
||||
rec.memGuardWarned.Store(false)
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/updater"
|
||||
"github.com/dbledeez/panel/pkg/steamvdf"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Update-available check (WI-14). CHECK-ONLY: reads the installed
|
||||
// buildid from the instance's Steam appmanifest ACF and the latest
|
||||
// buildid for its branch via `steamcmd +app_info_print` — it never
|
||||
// starts an update or touches the install.
|
||||
|
||||
// appInfoCacheTTL is how long a fetched branch map is reused before a
|
||||
// fresh steamcmd run. app_info runs cost a sidecar spin-up (~10-30 s),
|
||||
// so checks across many instances of the same game share one fetch.
|
||||
const appInfoCacheTTL = 15 * time.Minute
|
||||
|
||||
type appInfoCacheEntry struct {
|
||||
branches map[string]string
|
||||
fetchedAt time.Time
|
||||
}
|
||||
|
||||
var (
|
||||
appInfoCacheMu sync.Mutex
|
||||
appInfoCache = map[string]appInfoCacheEntry{} // key: app_id
|
||||
// appInfoFetchMu single-flights concurrent fetches per process; a
|
||||
// coarse lock is fine — checks are rare and operator-initiated.
|
||||
appInfoFetchMu sync.Mutex
|
||||
)
|
||||
|
||||
func (d *Dispatcher) handleUpdateCheck(corrID string, req *panelv1.UpdateCheckRequest) {
|
||||
fail := func(msg string) {
|
||||
d.sendUpdateCheckResult(corrID, &panelv1.UpdateCheckResult{
|
||||
InstanceId: req.InstanceId, Error: msg,
|
||||
})
|
||||
}
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
fail(err.Error())
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
fail("module not in registry")
|
||||
return
|
||||
}
|
||||
// Same provider selection as handleUpdate: by id when supplied, else
|
||||
// first in the manifest.
|
||||
var spec *modulepkg_UpdateProvider
|
||||
for i := range manifest.UpdateProviders {
|
||||
p := &manifest.UpdateProviders[i]
|
||||
if req.ProviderId == "" || p.ID == req.ProviderId {
|
||||
spec = p
|
||||
break
|
||||
}
|
||||
}
|
||||
if spec == nil {
|
||||
fail(fmt.Sprintf("no provider %q on module %s", req.ProviderId, rec.ModuleID))
|
||||
return
|
||||
}
|
||||
if spec.Kind != "steamcmd" {
|
||||
fail(fmt.Sprintf("update check supports steamcmd providers only (provider %q is %q)", spec.ID, spec.Kind))
|
||||
return
|
||||
}
|
||||
if spec.AppID == "" {
|
||||
fail("provider has no app_id")
|
||||
return
|
||||
}
|
||||
|
||||
// The check spins a steamcmd sidecar (tens of seconds) — never block
|
||||
// the dispatch read loop.
|
||||
go func() {
|
||||
branch := spec.Beta
|
||||
branchKey := branch
|
||||
if branchKey == "" {
|
||||
branchKey = "public"
|
||||
}
|
||||
res := &panelv1.UpdateCheckResult{
|
||||
InstanceId: req.InstanceId,
|
||||
AppId: spec.AppID,
|
||||
Branch: branchKey,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// Installed side: appmanifest ACF under the provider's install
|
||||
// root. Prefer betakey/buildid straight from the manifest file.
|
||||
if acf, rerr := d.readInstalledACF(ctx, rec, spec.InstallPath, spec.AppID); rerr != nil {
|
||||
res.Error = "read appmanifest: " + rerr.Error()
|
||||
} else if m, perr := steamvdf.ParseAppManifest(string(acf)); perr != nil {
|
||||
res.Error = "parse appmanifest: " + perr.Error()
|
||||
} else {
|
||||
res.InstalledBuildid = m.BuildID
|
||||
if m.BetaKey != "" {
|
||||
// Trust the on-disk truth over the manifest's declared
|
||||
// beta — this is exactly the "bounced between branches"
|
||||
// case the check exists to expose.
|
||||
res.Branch = m.BetaKey
|
||||
branchKey = m.BetaKey
|
||||
}
|
||||
}
|
||||
|
||||
// Latest side: branch map via steamcmd app_info (15-min cache).
|
||||
branches, cached, ferr := d.latestBranches(ctx, rec.InstanceID, spec.AppID, req.Refresh)
|
||||
if ferr != nil {
|
||||
if res.Error != "" {
|
||||
res.Error += "; "
|
||||
}
|
||||
res.Error += "fetch latest: " + ferr.Error()
|
||||
} else {
|
||||
res.Cached = cached
|
||||
if bid, ok := branches[branchKey]; ok {
|
||||
res.LatestBuildid = bid
|
||||
} else if res.Error == "" {
|
||||
res.Error = fmt.Sprintf("branch %q not in Steam branches map (%d branches known)", branchKey, len(branches))
|
||||
}
|
||||
}
|
||||
|
||||
res.UpdateAvailable = res.InstalledBuildid != "" && res.LatestBuildid != "" &&
|
||||
res.InstalledBuildid != res.LatestBuildid
|
||||
d.log.Info("update check",
|
||||
"instance_id", req.InstanceId, "app_id", spec.AppID, "branch", branchKey,
|
||||
"installed", res.InstalledBuildid, "latest", res.LatestBuildid,
|
||||
"update_available", res.UpdateAvailable, "cached", res.Cached, "err", res.Error)
|
||||
d.sendUpdateCheckResult(corrID, res)
|
||||
}()
|
||||
}
|
||||
|
||||
// latestBranches returns the branch→buildid map for appID, from cache
|
||||
// when fresh (unless refresh), otherwise via a steamcmd sidecar run.
|
||||
func (d *Dispatcher) latestBranches(ctx context.Context, instanceID, appID string, refresh bool) (map[string]string, bool, error) {
|
||||
if !refresh {
|
||||
appInfoCacheMu.Lock()
|
||||
e, ok := appInfoCache[appID]
|
||||
appInfoCacheMu.Unlock()
|
||||
if ok && time.Since(e.fetchedAt) < appInfoCacheTTL {
|
||||
return e.branches, true, nil
|
||||
}
|
||||
}
|
||||
appInfoFetchMu.Lock()
|
||||
defer appInfoFetchMu.Unlock()
|
||||
// Re-check under the fetch lock — a concurrent check may have just
|
||||
// filled the cache while we waited.
|
||||
if !refresh {
|
||||
appInfoCacheMu.Lock()
|
||||
e, ok := appInfoCache[appID]
|
||||
appInfoCacheMu.Unlock()
|
||||
if ok && time.Since(e.fetchedAt) < appInfoCacheTTL {
|
||||
return e.branches, true, nil
|
||||
}
|
||||
}
|
||||
branches, err := updater.FetchAppInfoBranches(ctx, d.runtime, instanceID, appID, func(line string) {
|
||||
d.log.Debug("app_info", "instance_id", instanceID, "line", line)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
appInfoCacheMu.Lock()
|
||||
appInfoCache[appID] = appInfoCacheEntry{branches: branches, fetchedAt: time.Now()}
|
||||
appInfoCacheMu.Unlock()
|
||||
return branches, false, nil
|
||||
}
|
||||
|
||||
// readInstalledACF reads steamapps/appmanifest_<appID>.acf under the
|
||||
// provider's install root, using the same container-first/host-fallback
|
||||
// mechanics as the FsRead RPC (fs helper sidecar works on stopped
|
||||
// instances too).
|
||||
func (d *Dispatcher) readInstalledACF(ctx context.Context, rec *instanceRecord, installPath, appID string) ([]byte, error) {
|
||||
rel := path.Join("steamapps", fmt.Sprintf("appmanifest_%s.acf", appID))
|
||||
if d.useContainerOps(rec) {
|
||||
root := installPath
|
||||
if root == "" {
|
||||
root = rec.BrowseableRoot
|
||||
}
|
||||
abs, err := safeJoinAny(root, append(rootPaths(rec), root), path.Join(root, rel))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetID, err := d.getFsTargetContainerID(ctx, rec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d.runtime.CopyFileFromContainer(ctx, targetID, abs)
|
||||
}
|
||||
if rec.DataPath == "" {
|
||||
return nil, fmt.Errorf("no file storage available")
|
||||
}
|
||||
abs, err := safeJoinHost(rec.DataPath, rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return os.ReadFile(abs)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendUpdateCheckResult(corrID string, res *panelv1.UpdateCheckResult) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_UpdateCheckResult{UpdateCheckResult: res},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/updater"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Use a type alias so the file's handler signatures don't bleed the
|
||||
// modulepkg import into their exposed type names.
|
||||
type modulepkg_UpdateProvider = modulepkg.UpdateProvider
|
||||
|
||||
// registerUpdater stores the cancel func for the in-flight updater of
|
||||
// an instance. Returns (slot, true) when registration succeeds, or
|
||||
// (nil, false) when another updater is already running for the same
|
||||
// instance.
|
||||
//
|
||||
// We refuse instead of preempting because preempting races the docker
|
||||
// daemon's name cleanup: cancelling the prior context returns instantly,
|
||||
// but the prior steamcmd sidecar (e.g. panel-<id>-steamcmd-1) takes a
|
||||
// few seconds to actually exit and be removed. A new updater that
|
||||
// charges in immediately hits a "name already in use" Docker error and,
|
||||
// worse, leaves SteamCMD's staging directory mid-finalize (state 0x602)
|
||||
// — files stranded under <install>/steamapps/downloading/<appid>/.
|
||||
//
|
||||
// Concrete failure mode that prompted this guard:
|
||||
// - panelctl create palworld-test → auto-triggers first update.
|
||||
// - Operator (impatiently) runs panelctl update palworld-test 13s later.
|
||||
// - Old behavior cancelled the auto-update mid-validate, then the new
|
||||
// run failed name-conflict, then SteamCMD looped on exit-8 / 0x602.
|
||||
//
|
||||
// The delete path uses cancelUpdater (different entry), so this guard
|
||||
// doesn't block teardown.
|
||||
func (d *Dispatcher) registerUpdater(instanceID string, cancel context.CancelFunc) (*updaterSlot, bool) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if existing := d.updaters[instanceID]; existing != nil {
|
||||
return nil, false
|
||||
}
|
||||
slot := &updaterSlot{cancel: cancel}
|
||||
d.updaters[instanceID] = slot
|
||||
return slot, true
|
||||
}
|
||||
|
||||
// unregisterUpdater clears the registration ONLY IF the current entry
|
||||
// is still our slot — pointer compare guards against a later updater
|
||||
// having taken over.
|
||||
func (d *Dispatcher) unregisterUpdater(instanceID string, slot *updaterSlot) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.updaters[instanceID] == slot {
|
||||
delete(d.updaters, instanceID)
|
||||
}
|
||||
}
|
||||
|
||||
// cancelUpdater stops any in-flight updater for this instance and
|
||||
// removes the registration. Called from the delete path so the sidecar
|
||||
// gets torn down before we try to remove the volume.
|
||||
func (d *Dispatcher) cancelUpdater(instanceID string) {
|
||||
d.mu.Lock()
|
||||
slot := d.updaters[instanceID]
|
||||
delete(d.updaters, instanceID)
|
||||
d.mu.Unlock()
|
||||
if slot != nil && slot.cancel != nil {
|
||||
slot.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// Update handler. Controller sends ControllerEnvelope_Update; agent:
|
||||
//
|
||||
// 1. Looks up the instance + module manifest.
|
||||
// 2. Picks the update provider by id (or defaults to the first one).
|
||||
// 3. Emits a synchronous UpdateResult{accepted=true} so the panel can
|
||||
// un-gray the button and show "update started".
|
||||
// 4. Spawns a goroutine that runs the provider, streaming each progress
|
||||
// line as a LogLine{stream="update"}. Final sentinel line
|
||||
// "update:ok" or "update:error:<msg>" tells the UI we're done.
|
||||
//
|
||||
// Progress is not correlation-routed because the UI subscribes to the
|
||||
// normal event bus and picks up LogLines directly.
|
||||
|
||||
func (d *Dispatcher) handleUpdate(corrID string, req *panelv1.UpdateRequest) {
|
||||
d.handleUpdateWithHook(corrID, req, nil)
|
||||
}
|
||||
|
||||
// handleUpdateWithHook is the internal form that optionally fires onComplete
|
||||
// once the provider run finishes (success or failure). Used by the auto-
|
||||
// update-on-create path to chain a Start after the first-time download and
|
||||
// to clear UI state if the update errors out. The normal controller-driven
|
||||
// Update flow passes nil so the hook stays opt-in.
|
||||
func (d *Dispatcher) handleUpdateWithHook(corrID string, req *panelv1.UpdateRequest, onComplete func(err error)) {
|
||||
rec, err := d.lookupRecord(req.InstanceId)
|
||||
if err != nil {
|
||||
d.sendUpdateResult(corrID, false, err.Error(), "", "")
|
||||
return
|
||||
}
|
||||
manifest, ok := d.modules.Get(rec.ModuleID)
|
||||
if !ok {
|
||||
d.sendUpdateResult(corrID, false, "module not in registry", "", "")
|
||||
return
|
||||
}
|
||||
|
||||
// Special pseudo-provider "_basemods": short-circuits to the third-
|
||||
// party base-mod installer (Allocs + PrismaCore) instead of running
|
||||
// the module's normal updater. Reuses the Update RPC plumbing so the
|
||||
// UI gets the same "updating" pulse + error surface as a real update.
|
||||
// Reasoning for the underscore prefix: real provider ids never start
|
||||
// with one (manifest schema convention), so collisions are impossible.
|
||||
if req.ProviderId == "_basemods" {
|
||||
d.sendUpdateResult(corrID, true, "", "_basemods", "basemods")
|
||||
d.log.Info("install-base-mods: kicked", "instance_id", req.InstanceId)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
err := d.installBaseMods(ctx, req.InstanceId, manifest, rec.DataPath)
|
||||
if err != nil {
|
||||
d.log.Error("install-base-mods: failed", "instance_id", req.InstanceId, "err", err)
|
||||
} else {
|
||||
d.log.Info("install-base-mods: done", "instance_id", req.InstanceId)
|
||||
}
|
||||
if onComplete != nil {
|
||||
onComplete(err)
|
||||
}
|
||||
}()
|
||||
return
|
||||
}
|
||||
|
||||
// Pick provider: by id if supplied, else first in manifest.
|
||||
var spec *modulepkg_UpdateProvider
|
||||
for i := range manifest.UpdateProviders {
|
||||
p := &manifest.UpdateProviders[i]
|
||||
if req.ProviderId == "" || p.ID == req.ProviderId {
|
||||
spec = providerAdapter(p)
|
||||
break
|
||||
}
|
||||
}
|
||||
if spec == nil {
|
||||
d.sendUpdateResult(corrID, false, fmt.Sprintf("no provider %q on module %s", req.ProviderId, rec.ModuleID), "", "")
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := updater.Get(spec.Kind)
|
||||
if err != nil {
|
||||
d.sendUpdateResult(corrID, false, err.Error(), spec.ID, spec.Kind)
|
||||
return
|
||||
}
|
||||
|
||||
// Reject if another updater is already running on this instance —
|
||||
// see registerUpdater for why we refuse instead of preempting.
|
||||
//
|
||||
// Timeout bumped 30 → 90 min (2026-05-04) — Steam CDN can rate-limit
|
||||
// anonymous-login pulls down to 1-3 MB/s, and a 17 GB game (7DTD,
|
||||
// ARK SA) at 5 MB/s = 56 min, easily over 30. Killing the sidecar
|
||||
// at 30 min leaves a partial install + a confused operator. 90 min
|
||||
// covers the realistic worst case with margin; truly hung steamcmd
|
||||
// still gets killed eventually so we don't burn an updater slot
|
||||
// forever.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Minute)
|
||||
slot, ok := d.registerUpdater(req.InstanceId, cancel)
|
||||
if !ok {
|
||||
cancel()
|
||||
d.sendUpdateResult(corrID, false, "another update is already in progress for this instance", spec.ID, spec.Kind)
|
||||
d.log.Info("update rejected: already running", "instance_id", req.InstanceId, "provider_id", spec.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// Accept synchronously — run async.
|
||||
d.sendUpdateResult(corrID, true, "", spec.ID, spec.Kind)
|
||||
d.log.Info("update started", "instance_id", req.InstanceId, "provider_id", spec.ID, "kind", spec.Kind)
|
||||
|
||||
// Surface the in-flight update as a server-authoritative status.
|
||||
// Without this, the panel's status pill stayed at whatever it was
|
||||
// before the update (often "stopped" or worse, "crashed" if the
|
||||
// previous run had hard-exited) — operators saw "crashed" while
|
||||
// SteamCMD was actually downloading several GB. Emitting UPDATING
|
||||
// here gives every dashboard viewer the same accurate "installing"
|
||||
// pulse, regardless of which client kicked off the update.
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, "updating:"+spec.Kind)
|
||||
|
||||
go func() {
|
||||
defer cancel()
|
||||
defer d.unregisterUpdater(req.InstanceId, slot)
|
||||
sink := func(line string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: rec.InstanceID,
|
||||
Stream: "update",
|
||||
At: timestamppb.Now(),
|
||||
Line: line,
|
||||
}},
|
||||
})
|
||||
}
|
||||
sink(fmt.Sprintf("---- update started: provider=%s kind=%s ----", spec.ID, spec.Kind))
|
||||
uc := &updater.Context{
|
||||
InstanceID: rec.InstanceID,
|
||||
Manifest: manifest,
|
||||
ContainerID: rec.ContainerID,
|
||||
BrowseableRoot: rec.BrowseableRoot,
|
||||
DataPath: rec.DataPath,
|
||||
Runtime: d.runtime,
|
||||
Log: sink,
|
||||
// Steam login (if any) is forwarded by the controller for
|
||||
// modules that set `requires_steam_login`. Fields are empty
|
||||
// for anonymous-login apps — steamcmd.go falls back to that.
|
||||
SteamUsername: req.SteamUsername,
|
||||
SteamPassword: req.SteamPassword,
|
||||
}
|
||||
// ctx + cancel + slot were registered before the goroutine launched
|
||||
// (see registerUpdater above). The defer at the top of the goroutine
|
||||
// handles cleanup; the dispatcher's delete path can still abort us
|
||||
// via cancelUpdater.
|
||||
if err := prov.Update(ctx, uc, spec); err != nil {
|
||||
sink(fmt.Sprintf("update:error: %s", err.Error()))
|
||||
d.log.Warn("update failed", "instance_id", req.InstanceId, "err", err)
|
||||
// Clear the UPDATING pill — surface the failure as CRASHED with
|
||||
// a structured detail so operators see "install_failed: …" in
|
||||
// the UI rather than the pill being stuck on UPDATING forever.
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_CRASHED, -1, "install_failed: "+err.Error())
|
||||
if onComplete != nil {
|
||||
onComplete(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
sink("update:ok")
|
||||
d.log.Info("update complete", "instance_id", req.InstanceId, "provider_id", spec.ID)
|
||||
// Update succeeded — drop UPDATING back to STOPPED. handleStart's
|
||||
// own state emission supersedes if an auto-start follows. Without
|
||||
// this, a manually-triggered update left the pill on UPDATING
|
||||
// indefinitely until the next start/stop cycle.
|
||||
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "installed")
|
||||
if onComplete != nil {
|
||||
onComplete(nil)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendUpdateResult(corrID string, accepted bool, errMsg, providerID, providerKind string) {
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
CorrelationId: corrID,
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_UpdateResult{
|
||||
UpdateResult: &panelv1.UpdateResult{
|
||||
Accepted: accepted,
|
||||
Error: errMsg,
|
||||
ProviderId: providerID,
|
||||
ProviderKind: providerKind,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// providerAdapter is a pass-through that pins us to the modulepkg type
|
||||
// without needing to add another import alias at the top of this file.
|
||||
func providerAdapter(p *modulepkg_UpdateProvider) *modulepkg_UpdateProvider { return p }
|
||||
@@ -0,0 +1,331 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// tryWarmSeedFromSibling looks for another instance of the same module
|
||||
// already on this agent with a populated install volume; if found, copies
|
||||
// that volume verbatim into the new instance's install volume via a
|
||||
// throwaway debian sidecar. Saves operators 30+ minutes of redundant
|
||||
// SteamCMD downloads when standing up a second instance of a game.
|
||||
//
|
||||
// Returns (true, nil) on a successful seed — caller should skip the
|
||||
// updater and treat the install as already-complete.
|
||||
// Returns (false, nil) when there's no candidate sibling — caller falls
|
||||
// through to the normal updater.
|
||||
// Returns (false, err) on operational failure of the seed itself —
|
||||
// caller should also fall through (we'd rather over-download than leave
|
||||
// the user without a working install).
|
||||
//
|
||||
// Symlink handling: install volumes typically contain symlinks back into
|
||||
// the per-instance saves volume (e.g. /game/serverconfig.xml ->
|
||||
// /game-saves/serverconfig.xml on 7DTD, set up by the entrypoint at first
|
||||
// boot). Those symlinks would be DANGLING in the new instance because
|
||||
// the new saves volume is empty. We strip broken symlinks after copy
|
||||
// (`find -xtype l -delete`) so the entrypoint sees a clean state and
|
||||
// recreates them on its first boot.
|
||||
//
|
||||
// Saves bootstrap: if the sibling's saves volume contains a
|
||||
// serverconfig.xml-style file at a known location, we copy it to the
|
||||
// new saves volume so the entrypoint has a starting config (operator
|
||||
// can edit afterwards). Otherwise the entrypoint falls back to the
|
||||
// game's shipped default. Done per-module by the seedSavesPaths list.
|
||||
func (d *Dispatcher) tryWarmSeedFromSibling(ctx context.Context, newInstanceID string, manifest *modulepkg.Manifest, newDataPath string) (bool, error) {
|
||||
if manifest == nil || len(manifest.UpdateProviders) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
provider := manifest.UpdateProviders[0]
|
||||
installPath := provider.InstallPath
|
||||
if installPath == "" {
|
||||
installPath = manifest.Runtime.Docker.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Find a sibling: same agent, same module, NOT this instance, with a
|
||||
// container that exists (so we know the volume was at least populated
|
||||
// once). Prefer a stopped or running instance over a half-baked one.
|
||||
//
|
||||
// VERSION SAFETY: only reuse an install from a sibling on the SAME Steam
|
||||
// branch as this new instance. Copying e.g. a 3.0 (latest_experimental)
|
||||
// install into a server the operator created as 2.6 — or vice versa —
|
||||
// produces a binary/world mismatch that corrupts the save. The new
|
||||
// instance's record was tagged with its branch at Create (resolveCreateBranch),
|
||||
// before this runs. Siblings predating the feature have an empty branch and
|
||||
// are treated as the module's DEFAULT branch (branchOrDefault) — for 7DTD
|
||||
// that's v2.6, so the existing 2.6 fleet is still a valid seed source for a
|
||||
// new 2.6 server.
|
||||
d.mu.Lock()
|
||||
wantBranch := branchOrDefault("", manifest)
|
||||
if nr, ok := d.instances[newInstanceID]; ok {
|
||||
wantBranch = branchOrDefault(nr.Branch, manifest)
|
||||
}
|
||||
var srcID, srcDataPath string
|
||||
for id, rec := range d.instances {
|
||||
if id == newInstanceID {
|
||||
continue
|
||||
}
|
||||
if rec.ModuleID != manifest.ID {
|
||||
continue
|
||||
}
|
||||
if branchOrDefault(rec.Branch, manifest) != wantBranch {
|
||||
continue // different game version — not a safe seed source
|
||||
}
|
||||
// Best-effort: if we have multiple on this branch, the first one wins.
|
||||
// Could be improved later by picking the most-recently-updated.
|
||||
srcID = id
|
||||
srcDataPath = rec.DataPath
|
||||
break
|
||||
}
|
||||
d.mu.Unlock()
|
||||
if srcID == "" {
|
||||
d.log.Debug("warm-seed: no same-branch sibling instance found", "module", manifest.ID, "instance", newInstanceID, "want_branch", wantBranch)
|
||||
return false, nil
|
||||
}
|
||||
d.log.Info("warm-seed: matched sibling on branch", "module", manifest.ID, "instance", newInstanceID, "src", srcID, "branch", wantBranch)
|
||||
|
||||
srcVolumes := agentmodule.ResolveVolumes(manifest, srcID, srcDataPath)
|
||||
dstVolumes := agentmodule.ResolveVolumes(manifest, newInstanceID, newDataPath)
|
||||
|
||||
srcGameVol := volumeNameForPath(srcVolumes, installPath)
|
||||
dstGameVol := volumeNameForPath(dstVolumes, installPath)
|
||||
if srcGameVol == "" || dstGameVol == "" {
|
||||
return false, fmt.Errorf("warm-seed: install_path %q not in module volumes", installPath)
|
||||
}
|
||||
|
||||
// Find a "saves-style" sibling volume to bootstrap from. Heuristic:
|
||||
// any non-install named volume whose container path contains "save"
|
||||
// or "data". Most modules have one. If none, we still seed the install
|
||||
// volume; the entrypoint may still need to bootstrap saves itself.
|
||||
srcSavesVol, srcSavesPath := findSavesVolume(srcVolumes, installPath)
|
||||
dstSavesVol, _ := findSavesVolume(dstVolumes, installPath)
|
||||
|
||||
// Build the seed sidecar. debian:12-slim is small, has bash + find +
|
||||
// cp + mkdir, and is already cached on every panel-agent host (the
|
||||
// fs-helper sidecar uses it).
|
||||
// Why a tar pipe instead of cp -a: we want fine-grained exclusions for
|
||||
// per-instance state that shouldn't be inherited from the sibling.
|
||||
//
|
||||
// Mods/ split:
|
||||
// BASE mods (TFP_*, 0_TFP_*, Allocs_*, PrismaCore, xMarkers) are
|
||||
// load-bearing — TFP_Harmony is the runtime patcher, the server
|
||||
// won't even boot without it; Allocs is the live map renderer the
|
||||
// panel maps tab depends on; PrismaCore is the staff command surface
|
||||
// refugebotserver expects. We KEEP these.
|
||||
//
|
||||
// CUSTOM mods (AGF-VP-*, RefugeBot, AsylumRoboticInbox, anything
|
||||
// else) are server-specific — operators usually want a clean slate
|
||||
// on a new server so the panel's mod manager / their own upload
|
||||
// decides what's installed. We DROP these.
|
||||
//
|
||||
// steamapps/userdata, steamapps/downloading: per-Steam-account state
|
||||
// + half-fetched chunks. Always excluded.
|
||||
//
|
||||
// Approach: tar everything EXCEPT Mods + downloading + userdata, then
|
||||
// selectively copy back the base-mod entries from the sibling.
|
||||
// The Mods/ split + base-mod completeness gate below are 7DTD-SPECIFIC:
|
||||
// TFP_Harmony/Allocs/PrismaCore are load-bearing 7DTD server mods, and the
|
||||
// "missing base mods → exit 42" check re-triggers a full SteamCMD validate
|
||||
// to refetch them. Every OTHER module (palworld, soulmask, valheim, …) has
|
||||
// no Mods/ dir at all, so running this block against them ALWAYS hit the
|
||||
// completeness gate and exit-42'd — turning a valid sibling seed into a
|
||||
// spurious "incomplete sibling" failure. For non-7DTD modules we copy the
|
||||
// whole install verbatim (minus per-instance Steam state) and skip the gate.
|
||||
//
|
||||
// Root cause of the 2026-07-10 gamehost outage: Palworld + Soulmask checkouts
|
||||
// warm-seeded from a sibling, hit this 7DTD gate, exit-42'd, and fell through
|
||||
// to a fresh download — which was then canceled by a racing env-config
|
||||
// recreate (fixed separately in the controller). See changelog 2026-07-10.
|
||||
seedScript := `set -e
|
||||
echo "warm-seed: copying install volume contents (excl. Mods/, steamapps/userdata, steamapps/downloading)"
|
||||
find /dst-game -mindepth 1 -delete
|
||||
( cd /src-game && tar --exclude='./Mods' --exclude='./steamapps/userdata' --exclude='./steamapps/downloading' -cf - . ) | ( cd /dst-game && tar -xf - )
|
||||
`
|
||||
if manifest.ID == "7dtd" {
|
||||
seedScript += `mkdir -p /dst-game/Mods
|
||||
echo "warm-seed: copying BASE mods (TFP_*, Allocs_*, PrismaCore, Xample_MarkersMod) — load-bearing for the server"
|
||||
for mod in /src-game/Mods/TFP_* /src-game/Mods/0_TFP_* /src-game/Mods/Allocs_* /src-game/Mods/PrismaCore /src-game/Mods/Xample_* /src-game/Mods/*Markers* /src-game/Mods/xMarkers /src-game/Mods/xmarkers; do
|
||||
case "$mod" in *"*"*) continue ;; esac
|
||||
if [ -e "$mod" ]; then
|
||||
cp -a "$mod" /dst-game/Mods/
|
||||
echo " + $(basename $mod)"
|
||||
fi
|
||||
done
|
||||
`
|
||||
}
|
||||
seedScript += `echo "warm-seed: stripping dangling symlinks (saves-volume references)"
|
||||
find /dst-game -xtype l -delete
|
||||
`
|
||||
if manifest.ID == "7dtd" {
|
||||
seedScript += `# Completeness check (7DTD only) — fail the warmseed if any LOAD-BEARING base
|
||||
# mod is missing on dst. The caller falls back to a full steamcmd validate which
|
||||
# will re-fetch the TFP/xMarkers mods from the dedicated-server depot.
|
||||
missing=""
|
||||
for required in 0_TFP_Harmony TFP_WebServer; do
|
||||
[ -d "/dst-game/Mods/$required" ] || missing="$missing $required"
|
||||
done
|
||||
if [ -n "$missing" ]; then
|
||||
echo "warm-seed: incomplete sibling — missing base mods:$missing"
|
||||
exit 42
|
||||
fi
|
||||
`
|
||||
}
|
||||
if srcSavesVol != "" && dstSavesVol != "" {
|
||||
// Copy ONLY the rendered config file (and the .local/share parent
|
||||
// for HOME-based games). Don't copy the world data — the user wants
|
||||
// a fresh save on this new instance, not the sibling's.
|
||||
seedScript += `echo "warm-seed: bootstrapping saves volume from sibling"
|
||||
[ -s /src-saves/serverconfig.xml ] && cp /src-saves/serverconfig.xml /dst-saves/serverconfig.xml || true
|
||||
mkdir -p /dst-saves/.local/share 2>/dev/null || true
|
||||
# 7DTD HOME tree — entrypoint mkdir's the deeper path on boot, but creating
|
||||
# the parent here lets the install symlink resolve to a valid empty dir
|
||||
# even before the first start.
|
||||
for d in /src-saves/.local/share/7DaysToDie /src-saves/.local/share/Empyrion; do
|
||||
if [ -d "$d" ]; then
|
||||
mkdir -p "/dst-saves/$(echo $d | sed 's|/src-saves/||')"
|
||||
fi
|
||||
done
|
||||
`
|
||||
}
|
||||
seedScript += `echo SEED_OK`
|
||||
|
||||
sidecarID := fmt.Sprintf("%s-warmseed", newInstanceID)
|
||||
volumes := []runtime.VolumeSpec{
|
||||
{Type: "volume", VolumeName: srcGameVol, ContainerPath: "/src-game", ReadOnly: true},
|
||||
{Type: "volume", VolumeName: dstGameVol, ContainerPath: "/dst-game"},
|
||||
}
|
||||
if srcSavesVol != "" && dstSavesVol != "" {
|
||||
volumes = append(volumes,
|
||||
runtime.VolumeSpec{Type: "volume", VolumeName: srcSavesVol, ContainerPath: "/src-saves", ReadOnly: true},
|
||||
runtime.VolumeSpec{Type: "volume", VolumeName: dstSavesVol, ContainerPath: "/dst-saves"},
|
||||
)
|
||||
}
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: "debian:12-slim",
|
||||
Command: []string{"bash", "-c", seedScript},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
|
||||
// Surface what's happening — without this the card just shows a static
|
||||
// "installing" pulse with no console output during a multi-GB sibling copy,
|
||||
// which looks hung.
|
||||
d.sendInstanceState(newInstanceID, panelv1.InstanceStatus_INSTANCE_STATUS_UPDATING, 0, fmt.Sprintf("pulling game files from existing install (%s) — no SteamCMD download needed", srcID))
|
||||
d.log.Info("warm-seed: starting sidecar", "src", srcID, "dst", newInstanceID, "src_vol", srcGameVol, "dst_vol", dstGameVol, "saves_seed", srcSavesPath != "")
|
||||
contID, err := d.runtime.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("warm-seed: create sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = d.runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := d.runtime.Start(ctx, contID); err != nil {
|
||||
return false, fmt.Errorf("warm-seed: start sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
d.log.Info("warm-seed log", "instance_id", newInstanceID, "line", line)
|
||||
// Mirror the sidecar's friendly progress echoes ("warm-seed: copying
|
||||
// install volume…", "+ <mod>") onto the instance console so the
|
||||
// operator can watch the "pulling from existing install" progress.
|
||||
d.sendEnv(&panelv1.AgentEnvelope{
|
||||
SentAt: timestamppb.Now(),
|
||||
Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
|
||||
InstanceId: newInstanceID, Stream: "install", At: timestamppb.Now(), Line: line,
|
||||
}},
|
||||
})
|
||||
})
|
||||
}()
|
||||
exitCode, err := d.runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("warm-seed: wait sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return false, fmt.Errorf("warm-seed: sidecar exited %d", exitCode)
|
||||
}
|
||||
d.log.Info("warm-seed: success", "src", srcID, "dst", newInstanceID)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// volumeNameForPath returns the named-volume name whose container path
|
||||
// matches `path` exactly, or "" if no match. Bind mounts are skipped —
|
||||
// we only seed named volumes (host path is operator-managed).
|
||||
func volumeNameForPath(volumes []runtime.VolumeSpec, path string) string {
|
||||
for _, v := range volumes {
|
||||
if v.Type == "volume" && v.VolumeName != "" && v.ContainerPath == path {
|
||||
return v.VolumeName
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// findSavesVolume returns the (volumeName, containerPath) of the first
|
||||
// named-volume whose container path looks like a saves/data dir, ignoring
|
||||
// the install volume. Used to bootstrap the new instance's saves with a
|
||||
// rendered config from the sibling.
|
||||
func findSavesVolume(volumes []runtime.VolumeSpec, installPath string) (string, string) {
|
||||
for _, v := range volumes {
|
||||
if v.Type != "volume" || v.VolumeName == "" {
|
||||
continue
|
||||
}
|
||||
if v.ContainerPath == installPath {
|
||||
continue
|
||||
}
|
||||
lower := strings.ToLower(v.ContainerPath)
|
||||
if strings.Contains(lower, "save") || strings.Contains(lower, "data") {
|
||||
return v.VolumeName, v.ContainerPath
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// normalizeBranch collapses a provider `beta` value into a stable warm-seed
|
||||
// matching key: the empty beta (the moving public/stable branch) becomes
|
||||
// "public"; any explicit beta (e.g. "v2.6", "latest_experimental") is returned
|
||||
// trimmed as-is. Two installs are warm-seed compatible iff their normalized
|
||||
// branches are equal.
|
||||
func normalizeBranch(beta string) string {
|
||||
b := strings.TrimSpace(beta)
|
||||
if b == "" {
|
||||
return "public"
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// branchOrDefault returns the normalized branch for a record's Branch value,
|
||||
// treating an empty value (instances created before branch-tagging existed) as
|
||||
// the module's DEFAULT branch — UpdateProviders[0].Beta normalized. For 7DTD
|
||||
// the default is v2.6, so the pre-existing 2.6 fleet remains a valid warm-seed
|
||||
// source for a new 2.6 server. Returns "public" when the manifest has no
|
||||
// providers.
|
||||
func branchOrDefault(branch string, m *modulepkg.Manifest) string {
|
||||
if strings.TrimSpace(branch) != "" {
|
||||
return normalizeBranch(branch)
|
||||
}
|
||||
if m != nil && len(m.UpdateProviders) > 0 {
|
||||
return normalizeBranch(m.UpdateProviders[0].Beta)
|
||||
}
|
||||
return "public"
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Package module (agent-side) resolves a manifest + InstanceCreate RPC
|
||||
// into a runtime.InstanceSpec ready for the Docker / host runtime to launch.
|
||||
package module
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// ResolveDocker produces an InstanceSpec for Docker-mode launch of the given
|
||||
// module manifest and RPC create request. It substitutes $DATA_PATH in volume
|
||||
// targets and maps declared ports against the request's port overrides.
|
||||
func ResolveDocker(manifest *modulepkg.Manifest, req *panelv1.InstanceCreate) (runtime.InstanceSpec, error) {
|
||||
if manifest == nil {
|
||||
return runtime.InstanceSpec{}, errors.New("manifest is nil")
|
||||
}
|
||||
if manifest.Runtime.Docker == nil {
|
||||
return runtime.InstanceSpec{}, fmt.Errorf("module %q has no docker runtime", manifest.ID)
|
||||
}
|
||||
if req.DataPath == "" {
|
||||
return runtime.InstanceSpec{}, errors.New("req.data_path is required")
|
||||
}
|
||||
|
||||
docker := manifest.Runtime.Docker
|
||||
|
||||
env := map[string]string{}
|
||||
for k, v := range docker.Env {
|
||||
env[k] = expand(v, req.DataPath)
|
||||
}
|
||||
// Per-instance config_values layer: if the operator (or controller
|
||||
// feature code) set a value for an env key the module already
|
||||
// declares, use that value instead of the manifest default. Only
|
||||
// keys that ALREADY exist in docker.Env get overridden — we don't
|
||||
// let config_values smuggle in unrelated env vars. Used today by
|
||||
// the ARK cluster feature to override CLUSTER_ID per-instance.
|
||||
for k, v := range req.ConfigValues {
|
||||
if _, ok := env[k]; ok {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
volumes := make([]runtime.VolumeSpec, 0, len(docker.Volumes))
|
||||
for _, v := range docker.Volumes {
|
||||
vs := runtime.VolumeSpec{
|
||||
ContainerPath: v.Container,
|
||||
ReadOnly: v.ReadOnly,
|
||||
}
|
||||
kind := v.Type
|
||||
if kind == "" {
|
||||
// Infer from which field was populated.
|
||||
if v.Name != "" {
|
||||
kind = "volume"
|
||||
} else {
|
||||
kind = "bind"
|
||||
}
|
||||
}
|
||||
switch kind {
|
||||
case "volume":
|
||||
vs.Type = "volume"
|
||||
vs.VolumeName = expandInstance(v.Name, req.InstanceId)
|
||||
default:
|
||||
vs.Type = "bind"
|
||||
vs.HostPath = expand(v.Target, req.DataPath)
|
||||
}
|
||||
// Apply per-instance mount override if the controller asked for
|
||||
// one by container path. An absolute path (Unix `/...` or
|
||||
// Windows `C:\...`) is a bind mount host path; anything else is
|
||||
// a Docker named volume. Used by the ARK cluster feature to
|
||||
// swap the default per-instance cluster volume for a shared
|
||||
// host-visible `arkcluster/<id>/` bind mount. `$AGENT_DATA_ROOT`
|
||||
// has already been expanded by handleCreate.
|
||||
if ov, ok := req.MountOverrides[v.Container]; ok && ov != "" {
|
||||
if filepath.IsAbs(ov) {
|
||||
vs.Type = "bind"
|
||||
vs.HostPath = ov
|
||||
vs.VolumeName = ""
|
||||
} else {
|
||||
vs.Type = "volume"
|
||||
vs.VolumeName = ov
|
||||
vs.HostPath = ""
|
||||
}
|
||||
}
|
||||
volumes = append(volumes, vs)
|
||||
}
|
||||
|
||||
portOverrides := indexPortOverrides(req.Ports)
|
||||
ports := make([]runtime.PortSpec, 0, len(manifest.Ports))
|
||||
for _, p := range manifest.Ports {
|
||||
containerPort := uint16(p.Default)
|
||||
hostPort := containerPort
|
||||
if ov, ok := portOverrides[p.Name]; ok {
|
||||
if ov.ContainerPort != 0 {
|
||||
containerPort = uint16(ov.ContainerPort)
|
||||
}
|
||||
if ov.HostPort != 0 {
|
||||
hostPort = uint16(ov.HostPort)
|
||||
}
|
||||
}
|
||||
// Internal ports (e.g. 7DTD's telnet RCON on 8081) must still be
|
||||
// reachable from the Target agent — it runs on the same host — so we
|
||||
// bind them to 127.0.0.1 instead of 0.0.0.0. Public ports bind to
|
||||
// all interfaces (HostIP left empty).
|
||||
hostIP := ""
|
||||
if p.Internal {
|
||||
hostIP = "127.0.0.1"
|
||||
}
|
||||
ports = append(ports, runtime.PortSpec{
|
||||
ContainerPort: containerPort,
|
||||
HostPort: hostPort,
|
||||
Proto: strings.ToLower(p.Proto),
|
||||
HostIP: hostIP,
|
||||
})
|
||||
}
|
||||
|
||||
// Preserve nil when the manifest doesn't override — passing []string{} to
|
||||
// Docker ContainerCreate replaces the image's default entrypoint/cmd with
|
||||
// empty, which errors out as "no command specified".
|
||||
var entrypoint, command []string
|
||||
if len(docker.Entrypoint) > 0 {
|
||||
entrypoint = append([]string{}, docker.Entrypoint...)
|
||||
}
|
||||
if len(docker.Command) > 0 {
|
||||
command = append([]string{}, docker.Command...)
|
||||
}
|
||||
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: req.InstanceId,
|
||||
Image: docker.Image,
|
||||
Entrypoint: entrypoint,
|
||||
Command: command,
|
||||
Env: env,
|
||||
User: docker.User,
|
||||
Volumes: volumes,
|
||||
Ports: ports,
|
||||
NetworkMode: docker.NetworkMode,
|
||||
BuildContext: manifest.Dir, // module's directory holds the Dockerfile
|
||||
}
|
||||
// Auto-enable OpenStdin whenever the module's RCON adapter is stdio —
|
||||
// it's the one feature the adapter needs from the runtime, and making
|
||||
// it implicit saves a second knob every stdio-based module would have
|
||||
// to remember to set.
|
||||
if manifest.RCON != nil && manifest.RCON.Adapter == "stdio" {
|
||||
spec.OpenStdin = true
|
||||
}
|
||||
if lim := req.Limits; lim != nil {
|
||||
spec.Resources = runtime.ResourceLimits{
|
||||
CPUShares: lim.CpuShares,
|
||||
MemoryBytes: lim.MemBytes,
|
||||
}
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
// expand substitutes $DATA_PATH in the given string.
|
||||
func expand(s, dataPath string) string {
|
||||
return strings.ReplaceAll(s, "$DATA_PATH", dataPath)
|
||||
}
|
||||
|
||||
// expandInstance substitutes $INSTANCE_ID in the given string. Used for
|
||||
// named-volume templating so each instance gets its own volume.
|
||||
func expandInstance(s, instanceID string) string {
|
||||
return strings.ReplaceAll(s, "$INSTANCE_ID", instanceID)
|
||||
}
|
||||
|
||||
// ResolveVolumes translates manifest volumes into runtime.VolumeSpec form
|
||||
// for a specific instance. Exposed so sidecar launchers (SteamCMD updater,
|
||||
// future backup-runner, etc.) can mount the same volumes the main
|
||||
// container uses, putting files where the main container expects them.
|
||||
func ResolveVolumes(manifest *modulepkg.Manifest, instanceID, dataPath string) []runtime.VolumeSpec {
|
||||
if manifest == nil || manifest.Runtime.Docker == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]runtime.VolumeSpec, 0, len(manifest.Runtime.Docker.Volumes))
|
||||
for _, v := range manifest.Runtime.Docker.Volumes {
|
||||
vs := runtime.VolumeSpec{
|
||||
ContainerPath: v.Container,
|
||||
ReadOnly: v.ReadOnly,
|
||||
}
|
||||
kind := v.Type
|
||||
if kind == "" {
|
||||
if v.Name != "" {
|
||||
kind = "volume"
|
||||
} else {
|
||||
kind = "bind"
|
||||
}
|
||||
}
|
||||
switch kind {
|
||||
case "volume":
|
||||
vs.Type = "volume"
|
||||
vs.VolumeName = expandInstance(v.Name, instanceID)
|
||||
default:
|
||||
vs.Type = "bind"
|
||||
vs.HostPath = expand(v.Target, dataPath)
|
||||
}
|
||||
out = append(out, vs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// indexPortOverrides builds a by-name lookup of PortMap entries from the RPC.
|
||||
func indexPortOverrides(ports []*panelv1.PortMap) map[string]*panelv1.PortMap {
|
||||
out := map[string]*panelv1.PortMap{}
|
||||
for _, p := range ports {
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
out[p.Name] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BattlEyeDialer speaks the BattlEye RCON protocol used by DayZ, Arma 2/3,
|
||||
// Squad, Rising Storm 2, and a handful of other BE-protected games.
|
||||
// Protocol reference: https://www.battleye.com/downloads/BERConProtocol.txt
|
||||
//
|
||||
// Wire format (UDP, all frames start with "BE" + CRC32 + 0xFF + type + payload):
|
||||
//
|
||||
// 0x42 'B'
|
||||
// 0x45 'E'
|
||||
// uint32 crc32 — little-endian CRC32 of bytes from offset 6 onward
|
||||
// 0xFF — packet-header sentinel
|
||||
// uint8 type — 0x00 login, 0x01 command, 0x02 server message
|
||||
// payload — varies by type
|
||||
//
|
||||
// Auth (type 0x00): client sends its admin password; server replies with
|
||||
// a single-byte 0x01 = success, 0x00 = failure.
|
||||
//
|
||||
// Commands (type 0x01): client sends a 1-byte sequence id + the command
|
||||
// text; server echoes the same seq id with the response text.
|
||||
//
|
||||
// Server-pushed messages (type 0x02): the server streams chat/join/leave
|
||||
// events. Client MUST ack each one by echoing the seq id with an empty
|
||||
// payload or the server will disconnect.
|
||||
type BattlEyeDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (BattlEyeDialer) Name() string { return "be_rcon" }
|
||||
|
||||
const (
|
||||
bePacketHeader = 0xFF
|
||||
bePacketLogin = 0x00
|
||||
bePacketCmd = 0x01
|
||||
bePacketServer = 0x02
|
||||
)
|
||||
|
||||
// Dial performs the login handshake and returns a ready-to-use BE RCON client.
|
||||
func (BattlEyeDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("be rcon dial: host and port are required")
|
||||
}
|
||||
connectTimeout := opts.ConnectTimeout
|
||||
if connectTimeout == 0 {
|
||||
connectTimeout = 5 * time.Second
|
||||
}
|
||||
addr := &net.UDPAddr{IP: net.ParseIP(opts.Host), Port: opts.Port}
|
||||
if addr.IP == nil {
|
||||
// Allow hostnames too.
|
||||
resolved, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", opts.Host, opts.Port))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve %s: %w", opts.Host, err)
|
||||
}
|
||||
addr = resolved
|
||||
}
|
||||
conn, err := net.DialUDP("udp", nil, addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("be rcon dial %s:%d: %w", opts.Host, opts.Port, err)
|
||||
}
|
||||
|
||||
c := &beClient{conn: conn, addr: addr}
|
||||
if err := c.login(opts.Password, connectTimeout); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
// Kick off the keep-alive goroutine — BattlEye disconnects clients
|
||||
// that don't send traffic for ~45s. We ping every 30s.
|
||||
go c.keepAlive()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type beClient struct {
|
||||
conn *net.UDPConn
|
||||
addr *net.UDPAddr
|
||||
|
||||
seqMu sync.Mutex
|
||||
seq uint8 // next command sequence id
|
||||
|
||||
execMu sync.Mutex
|
||||
closed atomic.Bool
|
||||
// stopKA fires when Close is called; keep-alive loop watches it.
|
||||
stopKA chan struct{}
|
||||
}
|
||||
|
||||
// login sends the auth packet and waits for a 1-byte success response.
|
||||
func (c *beClient) login(password string, timeout time.Duration) error {
|
||||
pkt := buildBEPacket(bePacketLogin, []byte(password))
|
||||
if _, err := c.conn.Write(pkt); err != nil {
|
||||
return fmt.Errorf("be rcon login write: %w", err)
|
||||
}
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(timeout))
|
||||
buf := make([]byte, 256)
|
||||
n, err := c.conn.Read(buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("be rcon login read: %w", err)
|
||||
}
|
||||
payloadType, payload, ok := parseBEPacket(buf[:n])
|
||||
if !ok {
|
||||
return errors.New("be rcon login: malformed response")
|
||||
}
|
||||
if payloadType != bePacketLogin {
|
||||
return fmt.Errorf("be rcon login: unexpected packet type 0x%02x", payloadType)
|
||||
}
|
||||
if len(payload) < 1 {
|
||||
return errors.New("be rcon login: empty response payload")
|
||||
}
|
||||
if payload[0] != 0x01 {
|
||||
return errors.New("be rcon login: bad password")
|
||||
}
|
||||
c.stopKA = make(chan struct{})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exec runs one command and returns the server's response text. BattlEye
|
||||
// doesn't multi-frame short responses; long ones arrive as multiple packets
|
||||
// with the same seq id + a 1-byte 'part_count / part_index' header. We
|
||||
// concatenate parts in order.
|
||||
func (c *beClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
if c.closed.Load() {
|
||||
return "", errors.New("be rcon: client closed")
|
||||
}
|
||||
c.execMu.Lock()
|
||||
defer c.execMu.Unlock()
|
||||
|
||||
c.seqMu.Lock()
|
||||
seq := c.seq
|
||||
c.seq++
|
||||
c.seqMu.Unlock()
|
||||
|
||||
body := make([]byte, 0, len(cmd)+1)
|
||||
body = append(body, seq)
|
||||
body = append(body, []byte(cmd)...)
|
||||
pkt := buildBEPacket(bePacketCmd, body)
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
deadline = time.Now().Add(10 * time.Second)
|
||||
}
|
||||
_ = c.conn.SetDeadline(deadline)
|
||||
defer c.conn.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
|
||||
if _, err := c.conn.Write(pkt); err != nil {
|
||||
return "", fmt.Errorf("be rcon write: %w", err)
|
||||
}
|
||||
|
||||
// Collect response parts. We loop until we've got every declared part
|
||||
// for this seq, or an idle timeout fires with partial data.
|
||||
parts := map[uint8][]byte{}
|
||||
var totalParts uint8
|
||||
for {
|
||||
buf := make([]byte, 4096)
|
||||
n, err := c.conn.Read(buf)
|
||||
if err != nil {
|
||||
if len(parts) > 0 {
|
||||
// Return whatever we got — better than nothing.
|
||||
return reassembleBEParts(parts, totalParts), nil
|
||||
}
|
||||
return "", fmt.Errorf("be rcon read: %w", err)
|
||||
}
|
||||
pType, payload, ok := parseBEPacket(buf[:n])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if pType == bePacketServer {
|
||||
// Server-pushed event (chat/join/leave). Ack it + keep reading.
|
||||
if len(payload) >= 1 {
|
||||
ack := buildBEPacket(bePacketServer, []byte{payload[0]})
|
||||
_, _ = c.conn.Write(ack)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if pType != bePacketCmd || len(payload) < 1 || payload[0] != seq {
|
||||
continue
|
||||
}
|
||||
// Multi-part response format: [seq, 0x00, total_parts, part_index, ...body...]
|
||||
if len(payload) >= 4 && payload[1] == 0x00 {
|
||||
totalParts = payload[2]
|
||||
idx := payload[3]
|
||||
parts[idx] = payload[4:]
|
||||
if uint8(len(parts)) >= totalParts {
|
||||
return reassembleBEParts(parts, totalParts), nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Single-frame response: [seq, ...body...]
|
||||
return string(payload[1:]), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Close tears down the UDP connection + stops the keep-alive loop.
|
||||
func (c *beClient) Close() error {
|
||||
if !c.closed.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
}
|
||||
if c.stopKA != nil {
|
||||
close(c.stopKA)
|
||||
}
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// keepAlive sends a zero-length command every 30s to keep BE from dropping
|
||||
// the session for inactivity. The protocol explicitly documents this.
|
||||
func (c *beClient) keepAlive() {
|
||||
tick := time.NewTicker(30 * time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-c.stopKA:
|
||||
return
|
||||
case <-tick.C:
|
||||
if c.closed.Load() {
|
||||
return
|
||||
}
|
||||
c.execMu.Lock()
|
||||
c.seqMu.Lock()
|
||||
seq := c.seq
|
||||
c.seq++
|
||||
c.seqMu.Unlock()
|
||||
pkt := buildBEPacket(bePacketCmd, []byte{seq})
|
||||
_ = c.conn.SetWriteDeadline(time.Now().Add(2 * time.Second))
|
||||
_, _ = c.conn.Write(pkt)
|
||||
c.execMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- wire helpers ----
|
||||
|
||||
// buildBEPacket assembles a BE RCON UDP frame: "BE" + CRC32(body) + body,
|
||||
// where body = 0xFF + type + payload.
|
||||
func buildBEPacket(pType uint8, payload []byte) []byte {
|
||||
body := make([]byte, 0, 2+len(payload))
|
||||
body = append(body, bePacketHeader, pType)
|
||||
body = append(body, payload...)
|
||||
crc := crc32.ChecksumIEEE(body)
|
||||
out := make([]byte, 6+len(body))
|
||||
out[0], out[1] = 'B', 'E'
|
||||
binary.LittleEndian.PutUint32(out[2:6], crc)
|
||||
copy(out[6:], body)
|
||||
return out
|
||||
}
|
||||
|
||||
// parseBEPacket validates a "BE..." frame and returns (type, payload, ok).
|
||||
func parseBEPacket(buf []byte) (uint8, []byte, bool) {
|
||||
if len(buf) < 7 || buf[0] != 'B' || buf[1] != 'E' {
|
||||
return 0, nil, false
|
||||
}
|
||||
crc := binary.LittleEndian.Uint32(buf[2:6])
|
||||
if crc32.ChecksumIEEE(buf[6:]) != crc {
|
||||
// Corrupt packet — accept it anyway for lenience; some mods' BE
|
||||
// extensions are known to skip CRC. Emit a debug break here
|
||||
// later if we start seeing real-world issues.
|
||||
_ = crc
|
||||
}
|
||||
if buf[6] != bePacketHeader {
|
||||
return 0, nil, false
|
||||
}
|
||||
return buf[7], buf[8:], true
|
||||
}
|
||||
|
||||
// reassembleBEParts joins multi-frame command responses in order.
|
||||
func reassembleBEParts(parts map[uint8][]byte, total uint8) string {
|
||||
var b strings.Builder
|
||||
for i := uint8(0); i < total; i++ {
|
||||
b.Write(parts[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DockerExecDialer runs per command, opening
|
||||
// a fresh socket inside the container's network namespace each time and
|
||||
// closing it on completion. Fresh process, fresh fd, fresh TCP conn — nothing
|
||||
// is held open between calls.
|
||||
//
|
||||
// Why: ARK Survival Ascended's RCON listener wedges if a polling client holds
|
||||
// a long-lived TCP session — the engine accumulates CLOSE-WAIT sockets and
|
||||
// the listener thread itself dies after enough accumulation. The wedge is
|
||||
// not recoverable without restarting the engine. Per-command exec avoids the
|
||||
// wedge entirely because nothing persists between invocations.
|
||||
//
|
||||
// Trade-off: ~50–200ms of overhead per command for docker exec + rcon-cli
|
||||
// startup. Negligible at ARK SA's polling cadence (30s+); not appropriate
|
||||
// for high-frequency command paths (none exist in panel today).
|
||||
//
|
||||
// Requirements:
|
||||
// - Container has rcon-cli on PATH (acekorneya/asa_server: yes).
|
||||
// - Container env exposes the RCON password to the in-container shell, OR
|
||||
// the agent supplies it via DialOptions.Password (we pass it in).
|
||||
// - Agent process is in the docker group (the panel-agent systemd unit
|
||||
// already declares SupplementaryGroups=docker).
|
||||
type DockerExecDialer struct{}
|
||||
|
||||
func (DockerExecDialer) Name() string { return "docker_exec_rcon" }
|
||||
|
||||
func (DockerExecDialer) Dial(_ context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.ContainerID == "" {
|
||||
return nil, errors.New("docker_exec_rcon dial: ContainerID required")
|
||||
}
|
||||
port := ""
|
||||
if opts.Port != 0 {
|
||||
port = strconv.Itoa(opts.Port)
|
||||
}
|
||||
return &dockerExecClient{
|
||||
containerID: opts.ContainerID,
|
||||
password: opts.Password,
|
||||
port: port,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type dockerExecClient struct {
|
||||
containerID string
|
||||
password string
|
||||
port string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// ansiRe matches CSI escape sequences. rcon-cli emits ANSI color resets
|
||||
// after each response (\033[0m); strip them so the panel's parsers see
|
||||
// clean text.
|
||||
var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
|
||||
|
||||
// Exec runs and returns the cleaned stdout.
|
||||
func (c *dockerExecClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
args := []string{"exec", c.containerID, "rcon-cli", "--host", "localhost"}
|
||||
if c.port != "" {
|
||||
args = append(args, "--port", c.port)
|
||||
}
|
||||
if c.password != "" {
|
||||
args = append(args, "--password", c.password)
|
||||
}
|
||||
args = append(args, cmd)
|
||||
|
||||
dockerCmd := exec.CommandContext(ctx, "docker", args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
dockerCmd.Stdout = &stdout
|
||||
dockerCmd.Stderr = &stderr
|
||||
if err := dockerCmd.Run(); err != nil {
|
||||
errMsg := strings.TrimSpace(stderr.String())
|
||||
if errMsg == "" {
|
||||
errMsg = err.Error()
|
||||
}
|
||||
return "", fmt.Errorf("docker exec rcon-cli: %s", errMsg)
|
||||
}
|
||||
out := ansiRe.ReplaceAllString(stdout.String(), "")
|
||||
return strings.TrimSpace(out), nil
|
||||
}
|
||||
|
||||
// Close is a no-op — there's nothing held open between Exec calls.
|
||||
func (c *dockerExecClient) Close() error { return nil }
|
||||
@@ -0,0 +1,81 @@
|
||||
// Package rcon provides pluggable RCON adapters. Games speak different
|
||||
// dialects — Source RCON (Counter-Strike, Minecraft with Mojang auth),
|
||||
// telnet line protocol (7 Days to Die), HTTP admin APIs (some community
|
||||
// implementations), stdin-pipe for headless servers without a network
|
||||
// surface. The Agent picks an adapter based on the module manifest's
|
||||
// `rcon.adapter` field.
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client is a connected RCON session. Exec runs one command and returns its
|
||||
// raw text response. Clients must be safe for concurrent Exec calls — the
|
||||
// poll loop and a future ad-hoc-command path both need to call through the
|
||||
// same session.
|
||||
type Client interface {
|
||||
Exec(ctx context.Context, cmd string) (string, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// DialOptions configures one RCON connection attempt.
|
||||
type DialOptions struct {
|
||||
Host string
|
||||
Port int
|
||||
Password string
|
||||
ConnectTimeout time.Duration
|
||||
// ReadIdle is how long to wait with no new bytes before treating an Exec
|
||||
// response as complete. Telnet has no packet framing, so this heuristic
|
||||
// is needed; Source RCON overrides it to 0 (ignored).
|
||||
ReadIdle time.Duration
|
||||
|
||||
// ContainerID is the Docker container name or ID for stdio/exec-based
|
||||
// adapters that don't use TCP. Ignored by telnet/source_rcon.
|
||||
ContainerID string
|
||||
// Stdio is the backend that knows how to attach to a container's
|
||||
// stdin/stdout. Provided by the Docker runtime when the adapter is
|
||||
// "stdio". Nil for TCP adapters.
|
||||
Stdio StdioBackend
|
||||
}
|
||||
|
||||
// StdioBackend attaches to a running container's combined stdin/stdout/stderr
|
||||
// stream. Needed by the stdio adapter for games whose only admin console is
|
||||
// stdin (Valheim, Terraria, Barotrauma, Enshrouded, Sons Of The Forest).
|
||||
//
|
||||
// Returned ReadWriteCloser writes go to the container's stdin; reads are the
|
||||
// demux-multiplexed stdout+stderr. Close shuts down the hijacked connection.
|
||||
type StdioBackend interface {
|
||||
AttachStdio(ctx context.Context, containerID string) (io.ReadWriteCloser, error)
|
||||
}
|
||||
|
||||
// Dialer establishes RCON connections of a specific adapter kind.
|
||||
type Dialer interface {
|
||||
Dial(ctx context.Context, opts DialOptions) (Client, error)
|
||||
Name() string
|
||||
}
|
||||
|
||||
// DialerFor returns the dialer for the given adapter id, or an error if the
|
||||
// adapter is unknown. Adapter ids come directly from the module manifest's
|
||||
// `rcon.adapter` field.
|
||||
func DialerFor(adapter string) (Dialer, error) {
|
||||
switch adapter {
|
||||
case "telnet":
|
||||
return &TelnetDialer{}, nil
|
||||
case "source_rcon":
|
||||
return &SourceDialer{}, nil
|
||||
case "stdio":
|
||||
return &StdioDialer{}, nil
|
||||
case "websocket_rcon":
|
||||
return &WebSocketDialer{}, nil
|
||||
case "docker_exec_rcon":
|
||||
return &DockerExecDialer{}, nil
|
||||
case "be_rcon":
|
||||
return &BattlEyeDialer{}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown rcon adapter %q (known: telnet, source_rcon, docker_exec_rcon, stdio, websocket_rcon, be_rcon)", adapter)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SourceDialer speaks Valve's Source RCON protocol — the standard RCON of
|
||||
// Minecraft, CS2, Rust, Valheim-with-plugins, and most Source-engine games.
|
||||
//
|
||||
// Protocol summary (little-endian throughout):
|
||||
//
|
||||
// int32 size = bytes that follow (i.e. frame length − 4)
|
||||
// int32 id = client-chosen, server echoes
|
||||
// int32 type = 3 auth-request, 2 exec, 0 response, 2 auth-response
|
||||
// body ASCIIZ = command string or response body, null-terminated
|
||||
// byte trailing null = required empty string sentinel
|
||||
//
|
||||
// size = 4 (id) + 4 (type) + len(body) + 1 (body NUL) + 1 (sentinel NUL)
|
||||
type SourceDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (SourceDialer) Name() string { return "source_rcon" }
|
||||
|
||||
const (
|
||||
pktAuth int32 = 3
|
||||
pktExecCommand int32 = 2
|
||||
pktResponseValue int32 = 0
|
||||
pktAuthResponse int32 = 2
|
||||
|
||||
minPktSize = 10
|
||||
maxPktSize = 4110
|
||||
)
|
||||
|
||||
func (SourceDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("source rcon dial: host and port are required")
|
||||
}
|
||||
timeout := opts.ConnectTimeout
|
||||
if timeout == 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
d := &net.Dialer{Timeout: timeout}
|
||||
conn, err := d.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", opts.Host, opts.Port))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source rcon dial %s:%d: %w", opts.Host, opts.Port, err)
|
||||
}
|
||||
c := &sourceClient{conn: conn, reader: bufio.NewReader(conn), nextID: 1}
|
||||
if err := c.authenticate(opts.Password); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("source rcon auth: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type sourceClient struct {
|
||||
conn net.Conn
|
||||
reader *bufio.Reader
|
||||
|
||||
mu sync.Mutex
|
||||
nextID int32
|
||||
}
|
||||
|
||||
// authenticate sends SERVERDATA_AUTH and waits for SERVERDATA_AUTH_RESPONSE.
|
||||
// Valve's spec says servers send an empty RESPONSE_VALUE first (id echoed),
|
||||
// then the AUTH_RESPONSE (id = original on success, −1 on failure). Some
|
||||
// implementations skip the probe — we accept either ordering.
|
||||
func (c *sourceClient) authenticate(password string) error {
|
||||
id := c.assignID()
|
||||
if err := c.writePacket(id, pktAuth, password); err != nil {
|
||||
return err
|
||||
}
|
||||
// ASA's RCON handler is slow to answer AUTH during the first few
|
||||
// minutes after "Server has completed startup" — sometimes 3-4s per
|
||||
// reply when the server is still loading tribe data / saving a
|
||||
// snapshot. The previous 2s per-read deadline + 5s overall was
|
||||
// tripping the tracker into its 32s exponential backoff on every
|
||||
// auth, stranding the panel with "RCON not yet connected" for 5-10
|
||||
// min even though the server was otherwise fine. Give AUTH a 12s
|
||||
// outer window and 8s per-read — enough slack for ASA's slowest
|
||||
// responses without letting a truly dead connection hang forever.
|
||||
deadline := time.Now().Add(12 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(8 * time.Second))
|
||||
gotID, gotType, _, err := c.readPacket()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if gotType == pktAuthResponse {
|
||||
// Spec: id == -1 means auth failed (bad password).
|
||||
// Any other id means success. Most servers echo our request id
|
||||
// but some (Conan Exiles, Palworld, a few others) return 0 or
|
||||
// an unrelated id — treat that as success, don't reject.
|
||||
if gotID == -1 {
|
||||
return errors.New("invalid password")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Probe echo (RESPONSE_VALUE with body ""), keep reading for the
|
||||
// AUTH_RESPONSE that follows.
|
||||
}
|
||||
return errors.New("auth timeout")
|
||||
}
|
||||
|
||||
// Exec sends a SERVERDATA_EXECCOMMAND and concatenates RESPONSE_VALUE bodies
|
||||
// until the server goes silent. For responses that straddle multiple packets
|
||||
// the spec recommends a "bogus packet trick" (send a second empty packet and
|
||||
// watch for its echo) but short command output fits in one packet in practice.
|
||||
func (c *sourceClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.conn.SetDeadline(dl)
|
||||
defer c.conn.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
}
|
||||
|
||||
id := c.assignID()
|
||||
if err := c.writePacket(id, pktExecCommand, cmd); err != nil {
|
||||
return "", fmt.Errorf("write: %w", err)
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
// First read waits longer — some servers (Palworld especially) take 1–3s
|
||||
// to build the response for commands like ShowPlayers. Subsequent reads
|
||||
// use a tight idle timeout so we exit promptly once the stream goes quiet.
|
||||
const (
|
||||
firstTimeout = 5 * time.Second
|
||||
idleTimeout = 800 * time.Millisecond
|
||||
)
|
||||
readDeadline := firstTimeout
|
||||
for {
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(readDeadline))
|
||||
readDeadline = idleTimeout
|
||||
gotID, gotType, body, err := c.readPacket()
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() && out.Len() > 0 {
|
||||
return out.String(), nil
|
||||
}
|
||||
if errors.Is(err, io.EOF) && out.Len() > 0 {
|
||||
return out.String(), nil
|
||||
}
|
||||
return out.String(), err
|
||||
}
|
||||
// Source RCON spec says the server MUST echo our request id on
|
||||
// RESPONSE_VALUE. Palworld always returns id=0; Conan Exiles
|
||||
// returns a different arbitrary id. We hold mu.Lock() for the
|
||||
// whole Exec so only one command is ever in flight — ID matching
|
||||
// isn't needed for correctness, so accept any id the server
|
||||
// hands us (except -1, which is reserved for auth failures).
|
||||
if gotID == -1 {
|
||||
continue
|
||||
}
|
||||
// Spec: RESPONSE_VALUE (type 0) carries command output. Conan
|
||||
// Exiles sends command output via AUTH_RESPONSE (type 2) instead —
|
||||
// non-spec but functional. We're past the auth phase by this point
|
||||
// so accepting either type is safe; anything else we skip.
|
||||
if gotType != pktResponseValue && gotType != pktAuthResponse {
|
||||
continue
|
||||
}
|
||||
out.WriteString(body)
|
||||
// Body shorter than ~4000 bytes → likely single-packet response.
|
||||
if len(body) < 3800 {
|
||||
return out.String(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close shuts down the underlying TCP connection.
|
||||
func (c *sourceClient) Close() error { return c.conn.Close() }
|
||||
|
||||
func (c *sourceClient) assignID() int32 {
|
||||
c.nextID++
|
||||
if c.nextID <= 0 {
|
||||
c.nextID = 1
|
||||
}
|
||||
return c.nextID
|
||||
}
|
||||
|
||||
// writePacket wire-encodes a Source RCON packet.
|
||||
func (c *sourceClient) writePacket(id, ptype int32, body string) error {
|
||||
bodyBytes := []byte(body)
|
||||
pktSize := int32(4 + 4 + len(bodyBytes) + 2) // id + type + body + 2×NUL
|
||||
if pktSize > maxPktSize {
|
||||
return fmt.Errorf("source rcon: body too large (%d bytes, max %d)", len(bodyBytes), maxPktSize-10)
|
||||
}
|
||||
buf := make([]byte, 4+pktSize)
|
||||
binary.LittleEndian.PutUint32(buf[0:4], uint32(pktSize))
|
||||
binary.LittleEndian.PutUint32(buf[4:8], uint32(id))
|
||||
binary.LittleEndian.PutUint32(buf[8:12], uint32(ptype))
|
||||
copy(buf[12:], bodyBytes)
|
||||
// trailing two NUL bytes already zero
|
||||
_, err := c.conn.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
// readPacket reads one framed Source RCON packet from the underlying reader.
|
||||
// Returns (id, type, body, err). Body has trailing NULs stripped.
|
||||
func (c *sourceClient) readPacket() (int32, int32, string, error) {
|
||||
var sizeBuf [4]byte
|
||||
if _, err := io.ReadFull(c.reader, sizeBuf[:]); err != nil {
|
||||
return 0, 0, "", err
|
||||
}
|
||||
size := int32(binary.LittleEndian.Uint32(sizeBuf[:]))
|
||||
if size < minPktSize || size > maxPktSize {
|
||||
return 0, 0, "", fmt.Errorf("source rcon: bogus packet size %d", size)
|
||||
}
|
||||
rest := make([]byte, size)
|
||||
if _, err := io.ReadFull(c.reader, rest); err != nil {
|
||||
return 0, 0, "", err
|
||||
}
|
||||
id := int32(binary.LittleEndian.Uint32(rest[0:4]))
|
||||
ptype := int32(binary.LittleEndian.Uint32(rest[4:8]))
|
||||
bodyEnd := size - 2 // trim both trailing NULs
|
||||
if bodyEnd < 8 {
|
||||
bodyEnd = 8
|
||||
}
|
||||
body := string(rest[8:bodyEnd])
|
||||
return id, ptype, body, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StdioDialer "RCON"s a game by attaching to the container's stdin and writing
|
||||
// the admin console command, trusting the game to execute it. It's the correct
|
||||
// choice for games whose admin surface is stdin-only — Valheim, Terraria,
|
||||
// Barotrauma, Enshrouded, Sons Of The Forest — where no TCP RCON exists.
|
||||
//
|
||||
// v1 is write-only: Exec returns empty string. Command output arrives via the
|
||||
// panel's log pipeline (docker logs → agent → bus → SSE) because the game's
|
||||
// replies appear on the same stdout the log stream is tailing. A future
|
||||
// revision could correlate a response window with each write if we ever need
|
||||
// structured replies for state polling.
|
||||
//
|
||||
// Requires the container to have been created with OpenStdin=true; the module
|
||||
// resolver auto-enables that whenever manifest.Rcon.Adapter == "stdio".
|
||||
type StdioDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (StdioDialer) Name() string { return "stdio" }
|
||||
|
||||
// Dial attaches to the container's stdio stream. A background goroutine drains
|
||||
// the multiplexed stdout/stderr so the stream never blocks from our side; the
|
||||
// docker logs pipeline is the canonical consumer of that output.
|
||||
func (StdioDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Stdio == nil {
|
||||
return nil, errors.New("stdio dial: Stdio backend required (agent must provide one)")
|
||||
}
|
||||
if opts.ContainerID == "" {
|
||||
return nil, errors.New("stdio dial: ContainerID required")
|
||||
}
|
||||
session, err := opts.Stdio.AttachStdio(ctx, opts.ContainerID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stdio attach %q: %w", opts.ContainerID, err)
|
||||
}
|
||||
c := &stdioClient{session: session}
|
||||
go func() {
|
||||
// Drain the attached stream so Docker doesn't buffer indefinitely.
|
||||
// We deliberately discard: the log pipeline already surfaces this
|
||||
// output to the panel via a separate docker logs call.
|
||||
_, _ = io.Copy(io.Discard, session)
|
||||
}()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
type stdioClient struct {
|
||||
session io.ReadWriteCloser
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Exec writes cmd + "\n" to the container's stdin. Returns empty string; the
|
||||
// game's response (if any) lands in the log stream and is surfaced in the
|
||||
// Console tab.
|
||||
//
|
||||
// The write is bounded by ctx (and a hard 10s fallback). A raw Write to a
|
||||
// hijacked stdin pipe can block FOREVER if the container's stdin read side
|
||||
// stalls (game paused, buffer full, or the process not draining stdin). Because
|
||||
// the agent dispatches RCON inline on its single recv loop, an unbounded write
|
||||
// here freezes the WHOLE agent — no stop/start/anything — until restart. That's
|
||||
// the Valheim+stdio "stop never lands" hang. We run the write in a goroutine and
|
||||
// abandon it on timeout so the dispatcher stays responsive.
|
||||
func (c *stdioClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.closed {
|
||||
return "", errors.New("stdio: session closed")
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
wctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.session.Write([]byte(cmd + "\n"))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("stdio write: %w", err)
|
||||
}
|
||||
return "", nil
|
||||
case <-wctx.Done():
|
||||
// The write is wedged — leave the goroutine parked (it'll unblock if the
|
||||
// pipe ever drains, or die when the session closes) but DON'T let it hold
|
||||
// up the caller / the agent's recv loop.
|
||||
return "", fmt.Errorf("stdio write timed out after 10s (container stdin not draining): %w", wctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// Close shuts down the hijacked attach connection.
|
||||
func (c *stdioClient) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.closed {
|
||||
return nil
|
||||
}
|
||||
c.closed = true
|
||||
return c.session.Close()
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TelnetDialer speaks 7DTD-style line-oriented telnet: on connect the server
|
||||
// sends a "password:" prompt; after the client writes the password and a
|
||||
// newline, commands are one-line writes and responses are everything read
|
||||
// until the connection goes quiet for a short idle period.
|
||||
type TelnetDialer struct{}
|
||||
|
||||
func (TelnetDialer) Name() string { return "telnet" }
|
||||
|
||||
func (TelnetDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("telnet dial: host and port are required")
|
||||
}
|
||||
addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port)
|
||||
|
||||
timeout := opts.ConnectTimeout
|
||||
if timeout == 0 {
|
||||
timeout = 5 * time.Second
|
||||
}
|
||||
|
||||
d := &net.Dialer{Timeout: timeout}
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("telnet dial %s: %w", addr, err)
|
||||
}
|
||||
|
||||
idle := opts.ReadIdle
|
||||
if idle == 0 {
|
||||
idle = 250 * time.Millisecond
|
||||
}
|
||||
c := &telnetClient{
|
||||
conn: conn,
|
||||
readIdle: idle,
|
||||
}
|
||||
if err := c.authenticate(opts.Password); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("telnet auth: %w", err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// telnetClient is safe for concurrent Exec via mu serialization.
|
||||
type telnetClient struct {
|
||||
conn net.Conn
|
||||
readIdle time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (c *telnetClient) authenticate(password string) error {
|
||||
// Read until the first "password:" prompt arrives, with a 5s cap.
|
||||
if _, err := c.readUntil("password:", 5*time.Second); err != nil {
|
||||
return fmt.Errorf("read password prompt: %w", err)
|
||||
}
|
||||
if _, err := c.conn.Write([]byte(password + "\n")); err != nil {
|
||||
return fmt.Errorf("write password: %w", err)
|
||||
}
|
||||
// Drain whatever welcome/banner text the server sends after auth.
|
||||
_, _ = c.readQuiet(1500*time.Millisecond, 1500*time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exec sends one line-terminated command and returns everything the server
|
||||
// writes back before going quiet for readIdle.
|
||||
func (c *telnetClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
// Wire ctx deadline into the conn's read/write timeouts.
|
||||
if dl, ok := ctx.Deadline(); ok {
|
||||
_ = c.conn.SetDeadline(dl)
|
||||
defer c.conn.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
}
|
||||
|
||||
if _, err := c.conn.Write([]byte(cmd + "\n")); err != nil {
|
||||
return "", fmt.Errorf("write cmd: %w", err)
|
||||
}
|
||||
return c.readQuiet(c.firstByteTimeout(), c.readIdle)
|
||||
}
|
||||
|
||||
func (c *telnetClient) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// firstByteTimeout is how long we wait for the server to START replying to a
|
||||
// command. A busy 7DTD server (heavy load, blood moon, GC pause) can take well
|
||||
// over readIdle to send its first byte, so this must be generous — otherwise
|
||||
// readQuiet times out with zero bytes and returns an empty response even though
|
||||
// the command executed fine on the server.
|
||||
func (c *telnetClient) firstByteTimeout() time.Duration {
|
||||
return 4 * time.Second
|
||||
}
|
||||
|
||||
// readQuiet reads a command response. It waits up to `first` for the first byte
|
||||
// (server may be slow to start replying under load), then once bytes arrive it
|
||||
// uses the short `idle` gap to detect end-of-response. Returns what it read, or
|
||||
// an error on a real (non-timeout) failure.
|
||||
func (c *telnetClient) readQuiet(first, idle time.Duration) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
tmp := make([]byte, 4096)
|
||||
for {
|
||||
// Before any data, allow the longer first-byte window; after data
|
||||
// starts flowing, switch to the short idle gap.
|
||||
wait := idle
|
||||
if buf.Len() == 0 {
|
||||
wait = first
|
||||
}
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(wait))
|
||||
n, err := c.conn.Read(tmp)
|
||||
if n > 0 {
|
||||
buf.Write(tmp[:n])
|
||||
}
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
// Timeout with data → normal end of response.
|
||||
// Timeout with NO data even after the first-byte window → the
|
||||
// server genuinely sent nothing (or is unreachable); return
|
||||
// empty and let the caller decide.
|
||||
return buf.String(), nil
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
return buf.String(), err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readUntil reads up to `overall` time waiting for `substr` (case-insensitive)
|
||||
// to appear in the accumulated buffer. Returns what it saw either way.
|
||||
func (c *telnetClient) readUntil(substr string, overall time.Duration) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
tmp := make([]byte, 4096)
|
||||
deadline := time.Now().Add(overall)
|
||||
want := strings.ToLower(substr)
|
||||
for time.Now().Before(deadline) {
|
||||
_ = c.conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
|
||||
n, err := c.conn.Read(tmp)
|
||||
if n > 0 {
|
||||
buf.Write(tmp[:n])
|
||||
if strings.Contains(strings.ToLower(buf.String()), want) {
|
||||
return buf.String(), nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
continue
|
||||
}
|
||||
return buf.String(), err
|
||||
}
|
||||
}
|
||||
return buf.String(), fmt.Errorf("timed out after %s waiting for %q", overall, substr)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeTelnetServer speaks enough of the 7DTD protocol to exercise dial+auth+exec:
|
||||
// - sends "Please enter password:" on accept
|
||||
// - waits for client password (any line), then sends "Logon successful.\n"
|
||||
// - for each subsequent command line, responds with its registered output
|
||||
type fakeTelnetServer struct {
|
||||
addr string
|
||||
responses map[string]string
|
||||
}
|
||||
|
||||
func startFakeTelnet(t *testing.T, responses map[string]string) *fakeTelnetServer {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
|
||||
fs := &fakeTelnetServer{addr: ln.Addr().String(), responses: responses}
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go fs.handle(conn)
|
||||
}
|
||||
}()
|
||||
return fs
|
||||
}
|
||||
|
||||
func (fs *fakeTelnetServer) handle(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
_, _ = conn.Write([]byte("*** Connected with 7DTD server.\r\nPlease enter password:"))
|
||||
|
||||
br := bufio.NewReader(conn)
|
||||
if _, err := br.ReadString('\n'); err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = conn.Write([]byte("\nLogon successful.\n"))
|
||||
|
||||
for {
|
||||
_ = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
cmd := strings.TrimSpace(line)
|
||||
resp, ok := fs.responses[cmd]
|
||||
if !ok {
|
||||
resp = fmt.Sprintf("(unknown command: %s)\n", cmd)
|
||||
}
|
||||
_, _ = conn.Write([]byte(resp))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelnetDialAndExec(t *testing.T) {
|
||||
srv := startFakeTelnet(t, map[string]string{
|
||||
"lp": "Total of 3 in the game\n\n1. id=1, Alice, pos=(0,0,0)\n2. id=2, Bob, pos=(1,0,0)\n3. id=3, Carol, pos=(2,0,0)\n\n",
|
||||
"say \"hello\"": "*Server: hello\n",
|
||||
})
|
||||
|
||||
host, port := splitAddr(t, srv.addr)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
client, err := TelnetDialer{}.Dial(ctx, DialOptions{
|
||||
Host: host,
|
||||
Port: port,
|
||||
Password: "test",
|
||||
ConnectTimeout: 2 * time.Second,
|
||||
ReadIdle: 150 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Dial: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
out, err := client.Exec(ctx, "lp")
|
||||
if err != nil {
|
||||
t.Fatalf("Exec lp: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "Total of 3 in the game") {
|
||||
t.Errorf("lp output missing expected substring, got: %q", out)
|
||||
}
|
||||
|
||||
out, err = client.Exec(ctx, `say "hello"`)
|
||||
if err != nil {
|
||||
t.Fatalf(`Exec say: %v`, err)
|
||||
}
|
||||
if !strings.Contains(out, "*Server: hello") {
|
||||
t.Errorf("say output missing, got: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialerForUnknown(t *testing.T) {
|
||||
if _, err := DialerFor("source_rcon"); err == nil {
|
||||
t.Error("expected error for unknown adapter")
|
||||
}
|
||||
if d, err := DialerFor("telnet"); err != nil || d.Name() != "telnet" {
|
||||
t.Errorf("DialerFor(telnet): d=%v err=%v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func splitAddr(t *testing.T, addr string) (string, int) {
|
||||
t.Helper()
|
||||
host, portStr, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("split %q: %v", addr, err)
|
||||
}
|
||||
var port int
|
||||
if _, err := fmt.Sscanf(portStr, "%d", &port); err != nil {
|
||||
t.Fatalf("port parse: %v", err)
|
||||
}
|
||||
return host, port
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package rcon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/websocket"
|
||||
)
|
||||
|
||||
// WebSocketDialer speaks Facepunch's WebSocket RCON — the protocol used
|
||||
// by Rust (app 258550) since they removed Source RCON years ago. A small
|
||||
// subset of games have adopted the same pattern (a few Oxide-compatible
|
||||
// community servers, some Unity-based survival games).
|
||||
//
|
||||
// Wire protocol (JSON text frames over a plain websocket):
|
||||
//
|
||||
// client → server: {"Identifier": N, "Message": "command", "Name": "panel"}
|
||||
// server → client: {"Identifier": N, "Message": "result text",
|
||||
// "Type": "Generic", "Stacktrace": ""}
|
||||
//
|
||||
// The password is passed as the URL path, not a header:
|
||||
//
|
||||
// ws://<host>:<port>/<password>
|
||||
//
|
||||
// Connections are long-lived — Rust's RCON doesn't close after each command
|
||||
// the way Conan's does — but we still redial transparently on EOF/reset
|
||||
// via the tracker's existing logic.
|
||||
type WebSocketDialer struct{}
|
||||
|
||||
// Name identifies this adapter in the module manifest.
|
||||
func (WebSocketDialer) Name() string { return "websocket_rcon" }
|
||||
|
||||
// Dial opens a websocket to ws://host:port/password. Returns a client that
|
||||
// serializes Exec calls; Rust sometimes interleaves async log spam into the
|
||||
// same connection as command replies, so we match responses back by id.
|
||||
func (WebSocketDialer) Dial(ctx context.Context, opts DialOptions) (Client, error) {
|
||||
if opts.Host == "" || opts.Port == 0 {
|
||||
return nil, errors.New("websocket rcon dial: host and port are required")
|
||||
}
|
||||
connectTimeout := opts.ConnectTimeout
|
||||
if connectTimeout == 0 {
|
||||
connectTimeout = 5 * time.Second
|
||||
}
|
||||
|
||||
wsURL := fmt.Sprintf("ws://%s:%d/%s", opts.Host, opts.Port, url.PathEscape(opts.Password))
|
||||
origin := fmt.Sprintf("http://%s:%d/", opts.Host, opts.Port)
|
||||
cfg, err := websocket.NewConfig(wsURL, origin)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("websocket rcon: build config: %w", err)
|
||||
}
|
||||
|
||||
// x/net/websocket.Dial doesn't honor a context; run it in a goroutine
|
||||
// so we can bail out when ctx.Done fires. Rust's authentication is
|
||||
// implicit — bad password = server closes the handshake with a 401.
|
||||
type dialResult struct {
|
||||
ws *websocket.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan dialResult, 1)
|
||||
go func() {
|
||||
ws, err := websocket.DialConfig(cfg)
|
||||
ch <- dialResult{ws, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-ch:
|
||||
if r.err != nil {
|
||||
return nil, fmt.Errorf("websocket rcon dial %s:%d: %w", opts.Host, opts.Port, r.err)
|
||||
}
|
||||
return newWebSocketClient(r.ws), nil
|
||||
case <-time.After(connectTimeout):
|
||||
return nil, fmt.Errorf("websocket rcon dial %s:%d: connect timeout", opts.Host, opts.Port)
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
type wsRequest struct {
|
||||
Identifier int32 `json:"Identifier"`
|
||||
Message string `json:"Message"`
|
||||
Name string `json:"Name"`
|
||||
}
|
||||
|
||||
type wsResponse struct {
|
||||
Message string `json:"Message"`
|
||||
Identifier int32 `json:"Identifier"`
|
||||
Type string `json:"Type"`
|
||||
Stacktrace string `json:"Stacktrace"`
|
||||
}
|
||||
|
||||
type wsClient struct {
|
||||
ws *websocket.Conn
|
||||
nextID atomic.Int32
|
||||
execMu sync.Mutex // serializes Exec calls
|
||||
readMu sync.Mutex // guards ws reads; Close may race
|
||||
closed atomic.Bool
|
||||
unsolic chan wsResponse // buffers server-pushed log events we don't consume
|
||||
}
|
||||
|
||||
func newWebSocketClient(ws *websocket.Conn) *wsClient {
|
||||
c := &wsClient{ws: ws, unsolic: make(chan wsResponse, 64)}
|
||||
c.nextID.Store(1)
|
||||
return c
|
||||
}
|
||||
|
||||
// Exec runs one command and returns the matching response body. Rust
|
||||
// echoes our request's Identifier back, so we can reject unrelated log
|
||||
// messages the server pushes on the same connection.
|
||||
func (c *wsClient) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
if c.closed.Load() {
|
||||
return "", errors.New("websocket rcon: client closed")
|
||||
}
|
||||
c.execMu.Lock()
|
||||
defer c.execMu.Unlock()
|
||||
|
||||
id := c.nextID.Add(1)
|
||||
req := wsRequest{Identifier: id, Message: cmd, Name: "panel"}
|
||||
payload, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal: %w", err)
|
||||
}
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
deadline = time.Now().Add(10 * time.Second)
|
||||
}
|
||||
if err := c.ws.SetDeadline(deadline); err != nil {
|
||||
return "", fmt.Errorf("set deadline: %w", err)
|
||||
}
|
||||
defer c.ws.SetDeadline(time.Time{}) //nolint:errcheck
|
||||
|
||||
if err := websocket.Message.Send(c.ws, string(payload)); err != nil {
|
||||
return "", fmt.Errorf("send: %w", err)
|
||||
}
|
||||
|
||||
// Read frames until one matches our Identifier. Rust pipes chat, log,
|
||||
// and player-event messages through the same socket; if their id is
|
||||
// 0 (or doesn't match ours) we drop them into unsolic for someone
|
||||
// else to consume. 200ms allowance for interleaved frames.
|
||||
for {
|
||||
c.readMu.Lock()
|
||||
var raw string
|
||||
rerr := websocket.Message.Receive(c.ws, &raw)
|
||||
c.readMu.Unlock()
|
||||
if rerr != nil {
|
||||
return "", fmt.Errorf("recv: %w", rerr)
|
||||
}
|
||||
var resp wsResponse
|
||||
if err := json.Unmarshal([]byte(raw), &resp); err != nil {
|
||||
// Non-JSON frame — Rust shouldn't send those but be forgiving.
|
||||
continue
|
||||
}
|
||||
if resp.Identifier != id {
|
||||
// Async server push (log line, chat, etc.). Buffer non-blockingly.
|
||||
select {
|
||||
case c.unsolic <- resp:
|
||||
default:
|
||||
}
|
||||
continue
|
||||
}
|
||||
if resp.Stacktrace != "" {
|
||||
return resp.Message, fmt.Errorf("rust rcon error: %s", resp.Stacktrace)
|
||||
}
|
||||
return resp.Message, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Close tears down the underlying websocket connection.
|
||||
func (c *wsClient) Close() error {
|
||||
if !c.closed.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
}
|
||||
return c.ws.Close()
|
||||
}
|
||||
@@ -0,0 +1,992 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/build"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/image"
|
||||
"github.com/docker/docker/api/types/mount"
|
||||
"github.com/docker/docker/api/types/network"
|
||||
"github.com/docker/docker/api/types/volume"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/docker/docker/pkg/stdcopy"
|
||||
"github.com/moby/go-archive"
|
||||
"github.com/docker/go-connections/nat"
|
||||
)
|
||||
|
||||
// DockerRuntime launches instances as Docker containers on the local daemon.
|
||||
// Container IDs (the string returned by Create) are Docker's full container IDs.
|
||||
type DockerRuntime struct {
|
||||
cli *client.Client
|
||||
}
|
||||
|
||||
// NewDocker connects to the local Docker daemon using environment-configured
|
||||
// settings (DOCKER_HOST, DOCKER_CERT_PATH, …) and negotiates an API version.
|
||||
func NewDocker() (*DockerRuntime, error) {
|
||||
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("docker client: %w", err)
|
||||
}
|
||||
return &DockerRuntime{cli: cli}, nil
|
||||
}
|
||||
|
||||
// Close releases daemon resources.
|
||||
func (r *DockerRuntime) Close() error {
|
||||
if r.cli == nil {
|
||||
return nil
|
||||
}
|
||||
return r.cli.Close()
|
||||
}
|
||||
|
||||
// Name identifies the runtime kind.
|
||||
func (r *DockerRuntime) Name() string { return "docker" }
|
||||
|
||||
// Ping verifies we can reach the Docker daemon.
|
||||
func (r *DockerRuntime) Ping(ctx context.Context) error {
|
||||
_, err := r.cli.Ping(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Create pulls the image (if missing locally) and creates the container.
|
||||
// The container is NOT started — call Start afterward.
|
||||
func (r *DockerRuntime) Create(ctx context.Context, spec InstanceSpec) (string, error) {
|
||||
if spec.InstanceID == "" {
|
||||
return "", errors.New("spec.InstanceID is required")
|
||||
}
|
||||
if spec.Image == "" {
|
||||
return "", errors.New("spec.Image is required")
|
||||
}
|
||||
|
||||
if err := r.acquireImage(ctx, spec.Image, spec.BuildContext, spec.LogSink); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
exposed, bindings, err := buildPortBindings(spec.Ports)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Docker rejects PortBindings + ExposedPorts when NetworkMode=host —
|
||||
// the container shares the host's network namespace so publish-port
|
||||
// plumbing is meaningless there.
|
||||
if spec.NetworkMode == "host" {
|
||||
bindings = nil
|
||||
exposed = nil
|
||||
}
|
||||
|
||||
envSlice := make([]string, 0, len(spec.Env))
|
||||
for k, v := range spec.Env {
|
||||
envSlice = append(envSlice, k+"="+v)
|
||||
}
|
||||
|
||||
mounts := make([]mount.Mount, 0, len(spec.Volumes))
|
||||
for _, v := range spec.Volumes {
|
||||
m := mount.Mount{
|
||||
Target: v.ContainerPath,
|
||||
ReadOnly: v.ReadOnly,
|
||||
}
|
||||
switch v.Type {
|
||||
case "volume":
|
||||
if v.VolumeName == "" {
|
||||
return "", fmt.Errorf("volume mount %q: Name required", v.ContainerPath)
|
||||
}
|
||||
if err := r.ensureVolume(ctx, v.VolumeName, spec.InstanceID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
m.Type = mount.TypeVolume
|
||||
m.Source = v.VolumeName
|
||||
default: // "" or "bind"
|
||||
if v.HostPath == "" {
|
||||
return "", fmt.Errorf("bind mount %q: HostPath required", v.ContainerPath)
|
||||
}
|
||||
// Docker CLI auto-creates missing bind sources, but the Go API
|
||||
// does not — it errors with "bind source path does not exist"
|
||||
// and refuses to start the container. Modules declaring subdir
|
||||
// binds like `$DATA_PATH/logs` rely on the source existing.
|
||||
// Two cases:
|
||||
// - Path exists as a file (panel-rendered config like
|
||||
// serverconfig.xml.rendered): leave it alone, it's a
|
||||
// file bind.
|
||||
// - Path missing or already a directory: mkdir -p so docker
|
||||
// bind-mount has something to mount.
|
||||
if st, err := os.Stat(v.HostPath); err == nil && !st.IsDir() {
|
||||
// existing file → file bind, no mkdir
|
||||
} else if err := os.MkdirAll(v.HostPath, 0o755); err != nil {
|
||||
return "", fmt.Errorf("bind mount %q: mkdir source %q: %w", v.ContainerPath, v.HostPath, err)
|
||||
}
|
||||
m.Type = mount.TypeBind
|
||||
m.Source = v.HostPath
|
||||
}
|
||||
mounts = append(mounts, m)
|
||||
}
|
||||
|
||||
restart := spec.RestartPolicy
|
||||
if restart == "" {
|
||||
restart = "unless-stopped"
|
||||
}
|
||||
|
||||
hostCfg := &container.HostConfig{
|
||||
Mounts: mounts,
|
||||
PortBindings: bindings,
|
||||
NetworkMode: container.NetworkMode(spec.NetworkMode),
|
||||
SecurityOpt: spec.SecurityOpts,
|
||||
RestartPolicy: container.RestartPolicy{
|
||||
Name: container.RestartPolicyMode(restart),
|
||||
},
|
||||
Resources: container.Resources{
|
||||
CPUShares: int64(spec.Resources.CPUShares),
|
||||
Memory: int64(spec.Resources.MemoryBytes),
|
||||
NanoCPUs: spec.Resources.NanoCPUs,
|
||||
PidsLimit: pidsPtr(spec.Resources.PidsLimit),
|
||||
},
|
||||
}
|
||||
|
||||
cfg := &container.Config{
|
||||
Image: spec.Image,
|
||||
Env: envSlice,
|
||||
Entrypoint: spec.Entrypoint,
|
||||
Cmd: spec.Command,
|
||||
User: spec.User,
|
||||
WorkingDir: spec.WorkingDir,
|
||||
ExposedPorts: exposed,
|
||||
Tty: false,
|
||||
// OpenStdin + StdinOnce=false = stdin stays open across reconnects.
|
||||
// The stdio RCON adapter attaches to stdin later via ContainerAttach.
|
||||
OpenStdin: spec.OpenStdin,
|
||||
StdinOnce: false,
|
||||
Labels: map[string]string{
|
||||
"panel.instance_id": spec.InstanceID,
|
||||
"panel.role": roleLabel(spec.IsSidecar),
|
||||
},
|
||||
}
|
||||
|
||||
name := "panel-" + spec.InstanceID
|
||||
resp, err := r.cli.ContainerCreate(ctx, cfg, hostCfg, &network.NetworkingConfig{}, nil, name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("container create: %w", err)
|
||||
}
|
||||
return resp.ID, nil
|
||||
}
|
||||
|
||||
// Start starts a previously-created container.
|
||||
func (r *DockerRuntime) Start(ctx context.Context, id string) error {
|
||||
if err := r.cli.ContainerStart(ctx, id, container.StartOptions{}); err != nil {
|
||||
return fmt.Errorf("container start: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the container, waiting up to grace before SIGKILL.
|
||||
func (r *DockerRuntime) Stop(ctx context.Context, id string, grace time.Duration) error {
|
||||
secs := int(grace.Seconds())
|
||||
if secs < 0 {
|
||||
secs = 0
|
||||
}
|
||||
opts := container.StopOptions{Timeout: &secs}
|
||||
if err := r.cli.ContainerStop(ctx, id, opts); err != nil {
|
||||
return fmt.Errorf("container stop: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restart restarts the container. Same StopOptions semantics as Stop —
|
||||
// engine waits up to grace seconds before SIGKILL, then the container is
|
||||
// started fresh. The container's RestartPolicy is irrelevant here;
|
||||
// ContainerRestart always brings it back regardless of policy.
|
||||
func (r *DockerRuntime) Restart(ctx context.Context, id string, grace time.Duration) error {
|
||||
secs := int(grace.Seconds())
|
||||
if secs < 0 {
|
||||
secs = 0
|
||||
}
|
||||
opts := container.StopOptions{Timeout: &secs}
|
||||
if err := r.cli.ContainerRestart(ctx, id, opts); err != nil {
|
||||
return fmt.Errorf("container restart: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove removes the container. Force=true kills if still running.
|
||||
func (r *DockerRuntime) Remove(ctx context.Context, id string) error {
|
||||
err := r.cli.ContainerRemove(ctx, id, container.RemoveOptions{
|
||||
Force: true,
|
||||
RemoveVolumes: false,
|
||||
})
|
||||
if err != nil && !client.IsErrNotFound(err) {
|
||||
return fmt.Errorf("container remove: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StreamLogs follows stdout+stderr and invokes handler for each line.
|
||||
// Blocks until ctx is cancelled, the container exits, or the connection errors.
|
||||
func (r *DockerRuntime) StreamLogs(ctx context.Context, id string, handler LogHandler) error {
|
||||
rc, err := r.cli.ContainerLogs(ctx, id, container.LogsOptions{
|
||||
ShowStdout: true,
|
||||
ShowStderr: true,
|
||||
Follow: true,
|
||||
// Timestamps:true prefixes each line with docker's own RFC3339Nano
|
||||
// time. We parse that prefix below and pass it as the line's `at`
|
||||
// instead of time.Now(). Critical for the readiness gate: the
|
||||
// docker log file persists across container restarts, so a Tail:400
|
||||
// replay can include the PREVIOUS run's "Server has completed
|
||||
// startup" line. Without docker's real timestamp on each line, the
|
||||
// frontend can't tell stale replays from current-run lines and
|
||||
// the pill flips to "running" the instant the buffer is fetched.
|
||||
Timestamps: true,
|
||||
// Start with a little backlog so the console isn't blank if someone
|
||||
// opens the modal mid-run. 400 lines is ~5s of chatter at the busiest
|
||||
// and is trivial to stream.
|
||||
Tail: "400",
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("container logs: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
outPr, outPw := io.Pipe()
|
||||
errPr, errPw := io.Pipe()
|
||||
defer outPw.Close()
|
||||
defer errPw.Close()
|
||||
|
||||
done := make(chan error, 3)
|
||||
|
||||
// Demux multiplexed docker log stream into separate stdout/stderr pipes.
|
||||
go func() {
|
||||
_, err := stdcopy.StdCopy(outPw, errPw, rc)
|
||||
_ = outPw.Close()
|
||||
_ = errPw.Close()
|
||||
done <- err
|
||||
}()
|
||||
|
||||
scan := func(r io.Reader, stream LogStream) {
|
||||
sc := bufio.NewScanner(r)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
// Split on BOTH \n and \r so terminal-style progress updates
|
||||
// (SteamCMD "Update state ... progress: X%" uses \r to overwrite
|
||||
// the same line) show up as they happen instead of buffering
|
||||
// until the next real newline minutes later.
|
||||
sc.Split(scanLogLine)
|
||||
for sc.Scan() {
|
||||
raw := stripANSI(sc.Text())
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
// Strip docker's RFC3339Nano timestamp prefix and use it as
|
||||
// the line's authoritative time. Format is exact:
|
||||
// "2026-04-29T17:46:53.123456789Z <line content>"
|
||||
// On parse failure we fall back to time.Now() so a malformed
|
||||
// line still gets streamed (better wrong timestamp than
|
||||
// dropped log line).
|
||||
line, ts := splitDockerTimestamp(raw)
|
||||
handler(stream, line, ts)
|
||||
}
|
||||
done <- sc.Err()
|
||||
}
|
||||
go scan(outPr, LogStreamStdout)
|
||||
go scan(errPr, LogStreamStderr)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-done:
|
||||
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrClosedPipe) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListVolumesByInstance returns every volume labeled panel.instance_id=id.
|
||||
// Used by the delete-with-purge path to clean up the instance's storage.
|
||||
func (r *DockerRuntime) ListVolumesByInstance(ctx context.Context, instanceID string) ([]string, error) {
|
||||
lst, err := r.cli.VolumeList(ctx, volume.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("volume list: %w", err)
|
||||
}
|
||||
var names []string
|
||||
for _, v := range lst.Volumes {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if v.Labels["panel.instance_id"] == instanceID {
|
||||
names = append(names, v.Name)
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// RemoveVolume removes a Docker volume by name. force=true removes even
|
||||
// if in-use references exist (shouldn't happen for us since we remove
|
||||
// containers first).
|
||||
func (r *DockerRuntime) RemoveVolume(ctx context.Context, name string, force bool) error {
|
||||
if err := r.cli.VolumeRemove(ctx, name, force); err != nil {
|
||||
return fmt.Errorf("volume remove %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListContainersHoldingInstanceVolumes returns container IDs for any
|
||||
// container (running OR exited) whose mount list references one of
|
||||
// the panel volumes labeled for the given instance.
|
||||
//
|
||||
// Used by the delete path to sweep stranded sidecars — exited steamcmd
|
||||
// or backup containers that the agent crashed before removing.
|
||||
// They keep volumes "in use" for `docker volume rm` purposes even when
|
||||
// they're not running, blocking instance delete-with-purge forever.
|
||||
//
|
||||
// We list ALL containers (including exited) and filter by mount name
|
||||
// rather than relying on the container's own panel label, because
|
||||
// updater sidecars don't carry the instance label — they're spawned
|
||||
// by the alpine helper which doesn't know about panel.* labels.
|
||||
func (r *DockerRuntime) ListContainersHoldingInstanceVolumes(ctx context.Context, instanceID string) ([]string, error) {
|
||||
// Build the set of volume names this instance owns.
|
||||
vols, err := r.ListVolumesByInstance(ctx, instanceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(vols) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
wanted := make(map[string]bool, len(vols))
|
||||
for _, v := range vols {
|
||||
wanted[v] = true
|
||||
}
|
||||
// Cheap heuristic: panel-<id>- prefix on the container name catches
|
||||
// most sidecars (panel-X-steamcmd-N, panel-X-fs, panel-X-backup).
|
||||
// We still cross-check via mounts so renamed/old containers don't
|
||||
// slip through.
|
||||
prefix := "panel-" + instanceID + "-"
|
||||
|
||||
cs, err := r.cli.ContainerList(ctx, container.ListOptions{All: true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("container list: %w", err)
|
||||
}
|
||||
out := make([]string, 0, len(cs))
|
||||
for _, c := range cs {
|
||||
// Match by mount.
|
||||
matched := false
|
||||
for _, m := range c.Mounts {
|
||||
if m.Type == mount.TypeVolume && wanted[m.Name] {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
// Or by name prefix.
|
||||
if !matched {
|
||||
for _, n := range c.Names {
|
||||
name := strings.TrimPrefix(n, "/")
|
||||
if strings.HasPrefix(name, prefix) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
out = append(out, c.ID)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// roleLabel maps the sidecar flag to the "panel.role" label value.
|
||||
func roleLabel(isSidecar bool) string {
|
||||
if isSidecar {
|
||||
return "sidecar"
|
||||
}
|
||||
return "instance"
|
||||
}
|
||||
|
||||
// ListMainInstanceNames returns the instance IDs of every main
|
||||
// game-server container (panel.role=instance), excluding the transient
|
||||
// fs / steamcmd / backup / etc. helper sidecars. Used by Rehydrate to
|
||||
// detect orphans: a main container with no matching sidecar metadata,
|
||||
// which would otherwise be silently invisible to the agent.
|
||||
//
|
||||
// Filtering is by the "panel.role" label, set at create time, not by
|
||||
// name-suffix heuristics — so new sidecar kinds can't be misclassified
|
||||
// as instances. Containers predating the label (no panel.role) fall
|
||||
// back to a name check: anything matching a known sidecar suffix is
|
||||
// excluded, everything else under the panel- prefix is treated as a
|
||||
// main instance (the conservative choice — better to surface a stray
|
||||
// helper as a candidate orphan than to hide a real instance).
|
||||
func (r *DockerRuntime) ListMainInstanceNames(ctx context.Context) ([]string, error) {
|
||||
cs, err := r.cli.ContainerList(ctx, container.ListOptions{All: true})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("container list: %w", err)
|
||||
}
|
||||
out := make([]string, 0, len(cs))
|
||||
for _, c := range cs {
|
||||
// Prefer the explicit label.
|
||||
if role, ok := c.Labels["panel.role"]; ok {
|
||||
if role != "instance" {
|
||||
continue
|
||||
}
|
||||
if id := c.Labels["panel.instance_id"]; id != "" {
|
||||
out = append(out, id)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Legacy fallback for containers created before panel.role existed.
|
||||
for _, n := range c.Names {
|
||||
name := strings.TrimPrefix(n, "/")
|
||||
id, ok := strings.CutPrefix(name, "panel-")
|
||||
if !ok || legacyIsSidecarName(id) {
|
||||
continue
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// legacyIsSidecarName classifies pre-"panel.role" containers by their
|
||||
// reserved trailing role. Instance ids themselves can contain hyphens
|
||||
// (e.g. "scorched-earth"), so we match known suffixes rather than
|
||||
// splitting on "-".
|
||||
func legacyIsSidecarName(afterPrefix string) bool {
|
||||
for _, suf := range []string{"-fs", "-backup", "-restore", "-wipe", "-basemods", "-warmseed"} {
|
||||
if strings.HasSuffix(afterPrefix, suf) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// steamcmd sidecars are panel-<id>-steamcmd-<n>.
|
||||
if strings.Contains(afterPrefix, "-steamcmd-") || strings.HasSuffix(afterPrefix, "-steamcmd") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ForceRemoveContainer is `docker rm -f` — kills a running container
|
||||
// and removes it. Used by the orphan-sidecar sweep on delete; the
|
||||
// sidecar's exit-code state doesn't matter since we're nuking it.
|
||||
func (r *DockerRuntime) ForceRemoveContainer(ctx context.Context, containerID string) error {
|
||||
return r.cli.ContainerRemove(ctx, containerID, container.RemoveOptions{
|
||||
Force: true,
|
||||
RemoveVolumes: false, // we'll remove volumes separately
|
||||
})
|
||||
}
|
||||
|
||||
// ensureVolume creates the named volume if it doesn't exist. Labels it so
|
||||
// operators can identify panel-owned volumes (`docker volume ls -f
|
||||
// label=panel.instance_id=ark-test`).
|
||||
func (r *DockerRuntime) ensureVolume(ctx context.Context, name, instanceID string) error {
|
||||
_, err := r.cli.VolumeInspect(ctx, name)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !client.IsErrNotFound(err) {
|
||||
return fmt.Errorf("inspect volume %q: %w", name, err)
|
||||
}
|
||||
_, err = r.cli.VolumeCreate(ctx, volume.CreateOptions{
|
||||
Name: name,
|
||||
Driver: "local",
|
||||
Labels: map[string]string{
|
||||
"panel.instance_id": instanceID,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create volume %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InspectByName looks up a container by its Docker name (leading "/"
|
||||
// stripped by the API) and returns its state. Used by the Agent on cold
|
||||
// start to check whether a sidecar-metadata file still points at a live
|
||||
// container.
|
||||
func (r *DockerRuntime) InspectByName(ctx context.Context, name string) (RuntimeInstanceState, error) {
|
||||
c, err := r.cli.ContainerInspect(ctx, name)
|
||||
if err != nil {
|
||||
return RuntimeInstanceState{}, err
|
||||
}
|
||||
st := RuntimeInstanceState{ContainerID: c.ID}
|
||||
if c.State != nil {
|
||||
st.Status = c.State.Status
|
||||
st.ExitCode = int32(c.State.ExitCode)
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// ContainerMounts returns the mount points of a container by name. Works
|
||||
// on stopped containers. Used to resolve a 7DTD instance's /cluster bind
|
||||
// to its host path for the cluster player-save snapshotter.
|
||||
func (r *DockerRuntime) ContainerMounts(ctx context.Context, name string) ([]MountInfo, error) {
|
||||
c, err := r.cli.ContainerInspect(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]MountInfo, 0, len(c.Mounts))
|
||||
for _, m := range c.Mounts {
|
||||
out = append(out, MountInfo{
|
||||
Type: string(m.Type),
|
||||
Source: m.Source,
|
||||
Destination: m.Destination,
|
||||
Name: m.Name,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ContainerExists reports definitive existence vs. an indeterminate
|
||||
// query failure. A not-found error from the daemon means the container
|
||||
// is gone (exists=false, err=nil); any other error (daemon unreachable,
|
||||
// connection dropped mid-call during agent shutdown, ctx cancelled)
|
||||
// leaves existence UNKNOWN and is returned verbatim so callers don't
|
||||
// mistake "couldn't ask" for "doesn't exist".
|
||||
func (r *DockerRuntime) ContainerExists(ctx context.Context, name string) (bool, error) {
|
||||
_, err := r.cli.ContainerInspect(ctx, name)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if client.IsErrNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// StatsStream opens Docker's ContainerStats (streaming) and returns the
|
||||
// raw JSON reader. Each line is a stats frame — caller parses + throttles.
|
||||
func (r *DockerRuntime) StatsStream(ctx context.Context, id string) (io.ReadCloser, error) {
|
||||
resp, err := r.cli.ContainerStats(ctx, id, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("container stats: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Wait blocks until the container exits and returns its exit code.
|
||||
func (r *DockerRuntime) Wait(ctx context.Context, id string) (int, error) {
|
||||
statusCh, errCh := r.cli.ContainerWait(ctx, id, container.WaitConditionNotRunning)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return -1, ctx.Err()
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
return -1, fmt.Errorf("container wait: %w", err)
|
||||
}
|
||||
return -1, nil
|
||||
case s := <-statusCh:
|
||||
if s.Error != nil {
|
||||
return int(s.StatusCode), fmt.Errorf("wait: %s", s.Error.Message)
|
||||
}
|
||||
return int(s.StatusCode), nil
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Container-path file ops (used by the panel's file manager) ----
|
||||
|
||||
// ExecCapture runs cmd inside the given running container and returns its
|
||||
// captured stdout + stderr + exit code.
|
||||
func (r *DockerRuntime) ExecCapture(ctx context.Context, id string, cmd []string) ([]byte, []byte, int, error) {
|
||||
resp, err := r.cli.ContainerExecCreate(ctx, id, container.ExecOptions{
|
||||
Cmd: cmd,
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, -1, fmt.Errorf("exec create: %w", err)
|
||||
}
|
||||
att, err := r.cli.ContainerExecAttach(ctx, resp.ID, container.ExecStartOptions{})
|
||||
if err != nil {
|
||||
return nil, nil, -1, fmt.Errorf("exec attach: %w", err)
|
||||
}
|
||||
defer att.Close()
|
||||
|
||||
// StdCopy reads the hijacked exec stream, which is NOT bound by ctx — a
|
||||
// container caught mid-restart can leave this Read blocked forever, leaking
|
||||
// the underlying docker connection. Unbounded, that once cascaded into every
|
||||
// runtime op hanging (config file-reads 504'd; a log stream stopped
|
||||
// re-attaching). Run StdCopy off a goroutine and enforce ctx ourselves: on
|
||||
// timeout, Close() the attach to unblock the Read, then drain the goroutine
|
||||
// so the buffers are race-free to read.
|
||||
var stdout, stderr bytes.Buffer
|
||||
copyDone := make(chan error, 1)
|
||||
go func() { _, e := stdcopy.StdCopy(&stdout, &stderr, att.Reader); copyDone <- e }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
att.Close() // unblock the goroutine's Read (defer Close is a safe no-op double-close)
|
||||
<-copyDone
|
||||
return stdout.Bytes(), stderr.Bytes(), -1, fmt.Errorf("exec copy: %w", ctx.Err())
|
||||
case err := <-copyDone:
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return stdout.Bytes(), stderr.Bytes(), -1, fmt.Errorf("exec copy: %w", err)
|
||||
}
|
||||
}
|
||||
insp, err := r.cli.ContainerExecInspect(ctx, resp.ID)
|
||||
if err != nil {
|
||||
return stdout.Bytes(), stderr.Bytes(), -1, fmt.Errorf("exec inspect: %w", err)
|
||||
}
|
||||
return stdout.Bytes(), stderr.Bytes(), insp.ExitCode, nil
|
||||
}
|
||||
|
||||
// CopyFileFromContainer returns the raw bytes of the file at containerPath.
|
||||
// Works on both running and stopped containers. Follows symlinks (Docker's
|
||||
// archive API tars a symlink as a symlink without dereferencing — needed
|
||||
// for cases like the 7DTD module's /game/serverconfig.xml -> /game-saves/…).
|
||||
func (r *DockerRuntime) CopyFileFromContainer(ctx context.Context, id, containerPath string) ([]byte, error) {
|
||||
return r.copyFileFromContainerHop(ctx, id, containerPath, 0)
|
||||
}
|
||||
|
||||
// maxCopyFileSymlinkHops bounds symlink resolution depth so a malicious or
|
||||
// accidental symlink loop can't pin an agent goroutine. 8 is way more than
|
||||
// any legitimate chain we've seen; bump it if a real use-case demands.
|
||||
const maxCopyFileSymlinkHops = 8
|
||||
|
||||
func (r *DockerRuntime) copyFileFromContainerHop(ctx context.Context, id, containerPath string, depth int) ([]byte, error) {
|
||||
if depth >= maxCopyFileSymlinkHops {
|
||||
return nil, fmt.Errorf("symlink chain too deep (>%d hops) at %s", maxCopyFileSymlinkHops, containerPath)
|
||||
}
|
||||
rc, _, err := r.cli.CopyFromContainer(ctx, id, containerPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy from container: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
tr := tar.NewReader(rc)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil, fmt.Errorf("file not found in archive")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tar next: %w", err)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
return io.ReadAll(tr)
|
||||
case tar.TypeDir:
|
||||
return nil, fmt.Errorf("path is a directory")
|
||||
case tar.TypeSymlink, tar.TypeLink:
|
||||
// Resolve the target. Docker hands back the symlink itself —
|
||||
// it never dereferences. Linkname can be absolute or relative;
|
||||
// relative is resolved against the link's containing dir, not
|
||||
// our cwd, mirroring how the kernel resolves it inside the FS.
|
||||
target := hdr.Linkname
|
||||
if !path.IsAbs(target) {
|
||||
target = path.Join(path.Dir(containerPath), target)
|
||||
}
|
||||
return r.copyFileFromContainerHop(ctx, id, target, depth+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AttachStdio attaches to the container's multiplexed stdin/stdout/stderr
|
||||
// stream and returns a single ReadWriteCloser. Writes go to stdin; reads
|
||||
// are the raw multiplexed frame stream (our stdio RCON adapter drains and
|
||||
// discards the reader — responses reach the panel via docker logs instead,
|
||||
// so we don't need to demux here).
|
||||
func (r *DockerRuntime) AttachStdio(ctx context.Context, id string) (io.ReadWriteCloser, error) {
|
||||
resp, err := r.cli.ContainerAttach(ctx, id, container.AttachOptions{
|
||||
Stream: true,
|
||||
Stdin: true,
|
||||
Stdout: true,
|
||||
Stderr: true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("container attach: %w", err)
|
||||
}
|
||||
return &hijackedStream{resp: resp}, nil
|
||||
}
|
||||
|
||||
// hijackedStream adapts Docker's HijackedResponse (a bidirectional net.Conn
|
||||
// plus a buffered reader) to io.ReadWriteCloser so the stdio adapter can
|
||||
// treat it uniformly.
|
||||
type hijackedStream struct {
|
||||
resp types.HijackedResponse
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (h *hijackedStream) Write(p []byte) (int, error) {
|
||||
return h.resp.Conn.Write(p)
|
||||
}
|
||||
func (h *hijackedStream) Read(p []byte) (int, error) {
|
||||
return h.resp.Reader.Read(p)
|
||||
}
|
||||
func (h *hijackedStream) Close() error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.resp.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// CopyFileToContainer writes content to containerPath inside the container.
|
||||
// Parent dir must exist (Docker's archive API doesn't create them).
|
||||
func (r *DockerRuntime) CopyFileToContainer(ctx context.Context, id, containerPath string, content []byte) error {
|
||||
dir := path.Dir(containerPath)
|
||||
base := path.Base(containerPath)
|
||||
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Name: base,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("tar header: %w", err)
|
||||
}
|
||||
if _, err := tw.Write(content); err != nil {
|
||||
return fmt.Errorf("tar body: %w", err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return fmt.Errorf("tar close: %w", err)
|
||||
}
|
||||
|
||||
return r.cli.CopyToContainer(ctx, id, dir, &buf, container.CopyToContainerOptions{AllowOverwriteDirWithFile: false})
|
||||
}
|
||||
|
||||
// CopyTarToContainer writes a raw tar stream onto dstDir. Caller is
|
||||
// responsible for the tar formatting (one Header+body per entry, then
|
||||
// Close()). Docker extracts in place. Used by the file manager's
|
||||
// extract endpoint so we can ship a whole archive's contents in one
|
||||
// API call instead of N per-file CopyFileToContainer round-trips.
|
||||
func (r *DockerRuntime) CopyTarToContainer(ctx context.Context, id, dstDir string, tarStream io.Reader) error {
|
||||
return r.cli.CopyToContainer(ctx, id, dstDir, tarStream, container.CopyToContainerOptions{AllowOverwriteDirWithFile: true})
|
||||
}
|
||||
|
||||
// CopyDirFromContainer returns a tar stream rooted at containerPath.
|
||||
// Caller must Close() the reader. Used by the file manager's compress
|
||||
// endpoint to walk a directory's contents without per-file API calls.
|
||||
func (r *DockerRuntime) CopyDirFromContainer(ctx context.Context, id, containerPath string) (io.ReadCloser, error) {
|
||||
rc, _, err := r.cli.CopyFromContainer(ctx, id, containerPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy from container: %w", err)
|
||||
}
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// acquireImage ensures the requested image is present locally. Fast path:
|
||||
// already on disk → return. Otherwise:
|
||||
//
|
||||
// - `panel-*` images are NEVER published to a registry. We auto-build
|
||||
// from buildContext/Dockerfile when buildContext is set. This is the
|
||||
// "drop in a new agent and any module just works" behavior — first
|
||||
// instance create on a fresh agent triggers the local build of the
|
||||
// module's runtime image. Subsequent creates hit the inspect-success
|
||||
// fast path.
|
||||
//
|
||||
// - Non-panel images (alpine, acekorneya/asa-linux-server, etc.) are
|
||||
// pulled from the registry as usual.
|
||||
//
|
||||
// If the image is `panel-*` and buildContext is empty (no Dockerfile in
|
||||
// the module dir), we surface the same "build it on the agent host"
|
||||
// message as before — the operator's module is broken, not the agent.
|
||||
func (r *DockerRuntime) acquireImage(ctx context.Context, ref, buildContext string, sink func(string)) error {
|
||||
emit := func(line string) {
|
||||
if sink != nil {
|
||||
sink(line)
|
||||
}
|
||||
}
|
||||
if _, err := r.cli.ImageInspect(ctx, ref); err == nil {
|
||||
return nil
|
||||
} else if !client.IsErrNotFound(err) {
|
||||
return fmt.Errorf("image inspect: %w", err)
|
||||
}
|
||||
|
||||
isPanel := strings.HasPrefix(ref, "panel-") && !strings.Contains(ref, "/")
|
||||
if isPanel {
|
||||
if buildContext == "" {
|
||||
return fmt.Errorf("module image %q not built on this agent and no build context provided", ref)
|
||||
}
|
||||
dockerfile := filepath.Join(buildContext, "Dockerfile")
|
||||
if _, err := os.Stat(dockerfile); err != nil {
|
||||
return fmt.Errorf("module image %q missing on this agent and no Dockerfile at %s", ref, dockerfile)
|
||||
}
|
||||
emit(fmt.Sprintf("[image] %s missing on this agent — building from %s/Dockerfile (first-run; takes 30s–3min)", ref, buildContext))
|
||||
return r.buildLocalImage(ctx, ref, buildContext, emit)
|
||||
}
|
||||
|
||||
// Public image — pull from registry.
|
||||
emit(fmt.Sprintf("[image] pulling %s from registry…", ref))
|
||||
rc, err := r.cli.ImagePull(ctx, ref, image.PullOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("image pull: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
streamDockerJSON(rc, emit)
|
||||
emit(fmt.Sprintf("[image] %s ready", ref))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildLocalImage runs `docker build -t <ref> <ctxDir>` via the Docker
|
||||
// daemon's API. Build output is parsed (NDJSON) and forwarded to `emit`
|
||||
// so the controller's Console tab gets live progress while the operator
|
||||
// waits for first-time builds.
|
||||
func (r *DockerRuntime) buildLocalImage(ctx context.Context, ref, ctxDir string, emit func(string)) error {
|
||||
tarball, err := archive.TarWithOptions(ctxDir, &archive.TarOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("tar build context: %w", err)
|
||||
}
|
||||
defer tarball.Close()
|
||||
resp, err := r.cli.ImageBuild(ctx, tarball, build.ImageBuildOptions{
|
||||
Tags: []string{ref},
|
||||
Dockerfile: "Dockerfile",
|
||||
Remove: true,
|
||||
ForceRemove: true,
|
||||
PullParent: false,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("image build start: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if buildErr := streamDockerJSON(resp.Body, emit); buildErr != nil {
|
||||
return fmt.Errorf("image build: %w", buildErr)
|
||||
}
|
||||
if _, err := r.cli.ImageInspect(ctx, ref); err != nil {
|
||||
return fmt.Errorf("image build %q produced no tag: %w", ref, err)
|
||||
}
|
||||
emit(fmt.Sprintf("[image] %s built and tagged", ref))
|
||||
return nil
|
||||
}
|
||||
|
||||
// streamDockerJSON parses Docker's streaming JSON output (one object per
|
||||
// line, fields like {"stream":"...","status":"...","error":"...","errorDetail":{...}})
|
||||
// and forwards meaningful lines to emit. Returns a non-nil error iff the
|
||||
// daemon reported an error in the stream.
|
||||
func streamDockerJSON(rc io.Reader, emit func(string)) error {
|
||||
dec := json.NewDecoder(rc)
|
||||
for {
|
||||
var msg struct {
|
||||
Stream string `json:"stream"`
|
||||
Status string `json:"status"`
|
||||
Progress string `json:"progress"`
|
||||
ID string `json:"id"`
|
||||
Error string `json:"error"`
|
||||
ErrorDetail json.RawMessage `json:"errorDetail"`
|
||||
}
|
||||
if err := dec.Decode(&msg); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
// Non-JSON tail / partial trailer — stop parsing but don't
|
||||
// fail the build; the post-build ImageInspect will catch
|
||||
// any tag-missing case.
|
||||
return nil
|
||||
}
|
||||
if msg.Error != "" {
|
||||
return errors.New(msg.Error)
|
||||
}
|
||||
// Build → "stream" lines (Step 1/12 : FROM debian:slim, etc).
|
||||
if s := strings.TrimRight(msg.Stream, "\r\n"); s != "" {
|
||||
emit(s)
|
||||
}
|
||||
// Pull → "status" lines, sometimes paired with progress for
|
||||
// each layer. Skip the high-frequency "Downloading" updates
|
||||
// (would spam the console at 10–30 Hz) — keep "Pulling /
|
||||
// Pulled" markers and final messages.
|
||||
if msg.Status != "" {
|
||||
st := msg.Status
|
||||
if strings.HasPrefix(st, "Downloading") || strings.HasPrefix(st, "Extracting") || strings.HasPrefix(st, "Verifying") {
|
||||
continue
|
||||
}
|
||||
if msg.ID != "" {
|
||||
emit(fmt.Sprintf("%s: %s", msg.ID, st))
|
||||
} else {
|
||||
emit(st)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// buildPortBindings translates PortSpec to Docker's nat.PortSet / PortMap.
|
||||
func buildPortBindings(ports []PortSpec) (nat.PortSet, nat.PortMap, error) {
|
||||
exposed := nat.PortSet{}
|
||||
bindings := nat.PortMap{}
|
||||
for _, p := range ports {
|
||||
proto := strings.ToLower(p.Proto)
|
||||
if proto != "tcp" && proto != "udp" {
|
||||
return nil, nil, fmt.Errorf("port %d/%s: proto must be tcp or udp", p.ContainerPort, p.Proto)
|
||||
}
|
||||
natPort, err := nat.NewPort(proto, strconv.Itoa(int(p.ContainerPort)))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("invalid port %d/%s: %w", p.ContainerPort, proto, err)
|
||||
}
|
||||
exposed[natPort] = struct{}{}
|
||||
hostPort := p.HostPort
|
||||
if hostPort == 0 {
|
||||
hostPort = p.ContainerPort
|
||||
}
|
||||
bindings[natPort] = []nat.PortBinding{{
|
||||
HostIP: p.HostIP,
|
||||
HostPort: strconv.Itoa(int(hostPort)),
|
||||
}}
|
||||
}
|
||||
return exposed, bindings, nil
|
||||
}
|
||||
|
||||
func pidsPtr(v int64) *int64 {
|
||||
if v <= 0 {
|
||||
return nil
|
||||
}
|
||||
return &v
|
||||
}
|
||||
|
||||
// splitDockerTimestamp pulls docker's RFC3339Nano timestamp prefix off a
|
||||
// log line and returns the remainder + parsed time. Docker emits each
|
||||
// line as "<rfc3339nano-z> <line>". A space immediately follows the Z;
|
||||
// anything else (including missing/garbled prefix) falls back to
|
||||
// time.Now() so we never drop the line.
|
||||
func splitDockerTimestamp(s string) (string, time.Time) {
|
||||
// Minimum valid prefix: "2026-01-01T00:00:00Z " — 21 chars. With nano
|
||||
// precision it's longer; the space after Z is what we look for.
|
||||
sp := strings.IndexByte(s, ' ')
|
||||
if sp < 20 || sp > 35 {
|
||||
return s, time.Now()
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339Nano, s[:sp])
|
||||
if err != nil {
|
||||
return s, time.Now()
|
||||
}
|
||||
return s[sp+1:], t
|
||||
}
|
||||
|
||||
// scanLogLine is a bufio.SplitFunc that treats \n, \r, and \r\n as line
|
||||
// terminators. The stdlib default (bufio.ScanLines) only splits on \n and
|
||||
// will buffer everything between \r-overwrites until it sees a real newline
|
||||
// — that's how SteamCMD's "Update state downloading progress: X%" bars
|
||||
// used to sit silent for minutes before dumping in one chunk.
|
||||
func scanLogLine(data []byte, atEOF bool) (advance int, token []byte, err error) {
|
||||
if atEOF && len(data) == 0 {
|
||||
return 0, nil, nil
|
||||
}
|
||||
if i := bytes.IndexAny(data, "\r\n"); i >= 0 {
|
||||
// Swallow a trailing \n after \r so "\r\n" yields one line not two.
|
||||
advance = i + 1
|
||||
if data[i] == '\r' && i+1 < len(data) && data[i+1] == '\n' {
|
||||
advance++
|
||||
}
|
||||
return advance, bytes.TrimRight(data[:i], "\r\n"), nil
|
||||
}
|
||||
if atEOF {
|
||||
return len(data), data, nil
|
||||
}
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
// ansiRE matches ANSI CSI sequences (colour, cursor movement, erase-line, …).
|
||||
// vinanrra's LGSM wrapper for 7DTD emits these liberally; they're noise in a
|
||||
// browser console and garble line-collapsing logic on the UI side.
|
||||
var ansiRE = regexp.MustCompile(`\x1b\[[0-9;?]*[ -/]*[@-~]`)
|
||||
|
||||
func stripANSI(s string) string {
|
||||
if !strings.ContainsRune(s, 0x1b) {
|
||||
return s
|
||||
}
|
||||
return ansiRE.ReplaceAllString(s, "")
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
// Package runtime defines the Target-side instance execution abstraction.
|
||||
// A Runtime is the thing that actually launches and tears down game server
|
||||
// processes — either inside a container (DockerRuntime) or directly on the
|
||||
// host (HostRuntime — future).
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// InstanceSpec is the resolved, runtime-ready launch recipe for one instance.
|
||||
// It is independent of the manifest format: a resolver in agent/internal/module
|
||||
// flattens (Manifest, InstanceCreate RPC) into this struct.
|
||||
type InstanceSpec struct {
|
||||
InstanceID string // panel-side identifier, used as container name
|
||||
Image string // e.g. vinanrra/7dtd-server:latest
|
||||
Entrypoint []string // override, empty = image default
|
||||
Command []string // override, empty = image default
|
||||
Env map[string]string // environment variables
|
||||
User string // uid:gid or name, empty = image default
|
||||
WorkingDir string
|
||||
Volumes []VolumeSpec
|
||||
Ports []PortSpec
|
||||
Resources ResourceLimits
|
||||
// RestartPolicy: "no", "on-failure", "unless-stopped", "always"
|
||||
RestartPolicy string
|
||||
// OpenStdin keeps the container's stdin open so we can later attach and
|
||||
// write admin-console commands (stdio RCON adapter). Auto-enabled by
|
||||
// the module resolver when rcon.adapter == "stdio".
|
||||
OpenStdin bool
|
||||
// NetworkMode is the Docker network mode. Empty → bridge (default).
|
||||
// "host" shares the host's network namespace; required for modules
|
||||
// whose TURN/STUN clients can't survive Docker's UDP NAT rebinds
|
||||
// (e.g. Windrose). In host mode, Ports become informational only.
|
||||
NetworkMode string
|
||||
// BuildContext is an absolute filesystem path containing a Dockerfile
|
||||
// that produces this Image. When set, the runtime auto-builds the
|
||||
// image on first use if it's missing locally — the AMP-class "drop in
|
||||
// an agent and any module just works" behavior. Required for `panel-*`
|
||||
// images, since they are never pushed to a registry. Empty for stock
|
||||
// public images (alpine, the acekorneya ASA image, etc.) which the
|
||||
// runtime pulls normally.
|
||||
BuildContext string
|
||||
// LogSink optionally receives runtime-side progress lines (image
|
||||
// pull / image build / etc.) so the agent can forward them to the
|
||||
// controller's log stream. Without this, first-time builds happen
|
||||
// silently and the operator sees "installing" with no visible
|
||||
// progress for ~30s–3min while the Dockerfile runs.
|
||||
LogSink func(line string)
|
||||
// SecurityOpts is passed through to Docker's HostConfig.SecurityOpt
|
||||
// (one entry per --security-opt CLI flag). The SteamCMD sidecar sets
|
||||
// "seccomp=unconfined" so that the 32-bit Steam runtime's socket
|
||||
// syscalls aren't ENOSYS'd by Docker's stricter default seccomp
|
||||
// profile in newer Docker releases (observed on Docker 29.4 + Ubuntu
|
||||
// 24.04 kernel 6.8: "CreateBoundSocket: failed to create socket,
|
||||
// error [no name available] (38)"). Same image works fine on
|
||||
// Docker 29.1 with the default profile.
|
||||
SecurityOpts []string
|
||||
// IsSidecar marks this as a transient helper container (fs browse,
|
||||
// steamcmd, backup, restore, wipe, warmseed, basemods) rather than a
|
||||
// main game-server instance. The runtime stamps a "panel.role" label
|
||||
// accordingly so orphan-detection (Rehydrate) can tell main instances
|
||||
// apart from helpers without brittle name-suffix matching.
|
||||
IsSidecar bool
|
||||
}
|
||||
|
||||
// VolumeSpec is a mount declaration, either a bind mount (host path →
|
||||
// container path) or a Docker named volume. The resolver is responsible
|
||||
// for substituting $DATA_PATH / $INSTANCE_ID before we see it here.
|
||||
type VolumeSpec struct {
|
||||
Type string // "bind" (default) or "volume"
|
||||
HostPath string // populated when Type == "bind"
|
||||
VolumeName string // populated when Type == "volume"
|
||||
ContainerPath string
|
||||
ReadOnly bool
|
||||
}
|
||||
|
||||
// PortSpec is a port mapping.
|
||||
type PortSpec struct {
|
||||
HostPort uint16
|
||||
ContainerPort uint16
|
||||
Proto string // "tcp" or "udp"
|
||||
// HostIP — bind address on the Target. Empty = 0.0.0.0.
|
||||
HostIP string
|
||||
}
|
||||
|
||||
// ResourceLimits caps CPU / memory for the instance.
|
||||
type ResourceLimits struct {
|
||||
CPUShares uint32
|
||||
MemoryBytes uint64
|
||||
PidsLimit int64 // 0 = unlimited
|
||||
NanoCPUs int64 // Docker "--cpus" as nano-CPUs (1.5 cpus = 1.5e9)
|
||||
StopGrace time.Duration
|
||||
}
|
||||
|
||||
// RuntimeInstanceState is the runtime's view of an instance at a point in
|
||||
// time, used by the Agent to rehydrate after restart or detect drift.
|
||||
type RuntimeInstanceState struct {
|
||||
ContainerID string
|
||||
Status string // "running" | "exited" | "created" | "paused" | "restarting" | "removing" | "dead"
|
||||
ExitCode int32
|
||||
}
|
||||
|
||||
// LogStream identifies which stream a log line came from.
|
||||
type LogStream string
|
||||
|
||||
const (
|
||||
LogStreamStdout LogStream = "stdout"
|
||||
LogStreamStderr LogStream = "stderr"
|
||||
)
|
||||
|
||||
// LogHandler receives one log line at a time. Lines are already stripped of
|
||||
// trailing newline. Handler must not block; if it does, log pressure will
|
||||
// build up in the Runtime.
|
||||
type LogHandler func(stream LogStream, line string, at time.Time)
|
||||
|
||||
// Runtime is the interface that both Docker and Host runtimes implement.
|
||||
type Runtime interface {
|
||||
// Create pulls the image (if needed) and creates the container.
|
||||
// Returns a runtime-specific identifier used by Start/Stop/Remove.
|
||||
Create(ctx context.Context, spec InstanceSpec) (id string, err error)
|
||||
|
||||
// Start starts a previously-created instance.
|
||||
Start(ctx context.Context, id string) error
|
||||
|
||||
// Stop stops a running instance, waiting up to grace before SIGKILL.
|
||||
Stop(ctx context.Context, id string, grace time.Duration) error
|
||||
|
||||
// Restart restarts a running instance, waiting up to grace before
|
||||
// SIGKILL. Container ends up running again — equivalent to
|
||||
// `docker container restart`. Used by guardrails (e.g. arkHangGuard)
|
||||
// that want a bounce without surfacing as an operator stop.
|
||||
Restart(ctx context.Context, id string, grace time.Duration) error
|
||||
|
||||
// Remove deletes the instance (after Stop). Safe to call on stopped.
|
||||
Remove(ctx context.Context, id string) error
|
||||
|
||||
// StreamLogs follows stdout/stderr and calls handler for each line.
|
||||
// Returns when ctx is cancelled or the instance exits.
|
||||
StreamLogs(ctx context.Context, id string, handler LogHandler) error
|
||||
|
||||
// Wait blocks until the instance exits and returns its exit code.
|
||||
Wait(ctx context.Context, id string) (exitCode int, err error)
|
||||
|
||||
// StatsStream returns a reader of Docker's stats stream (JSON frames,
|
||||
// one per second). Closer cancels the stream. Used by the per-instance
|
||||
// stats poller to compute CPU%/mem/net for UI + alerting.
|
||||
StatsStream(ctx context.Context, id string) (io.ReadCloser, error)
|
||||
|
||||
// InspectByName looks up an instance by its runtime-level name
|
||||
// (for Docker: "panel-<instance_id>") and returns its current state.
|
||||
// Returns an error whose predicate is not-found-able if the instance
|
||||
// has been removed from the runtime — used during agent rehydrate
|
||||
// to detect stale sidecar metadata.
|
||||
InspectByName(ctx context.Context, name string) (RuntimeInstanceState, error)
|
||||
|
||||
// ContainerExists reports whether a container with the given
|
||||
// runtime-level name currently exists. Unlike InspectByName, it
|
||||
// distinguishes "definitely gone" from "couldn't tell":
|
||||
// exists=false, err=nil → the runtime confirmed no such container
|
||||
// exists=false, err!=nil → the query itself failed (daemon down,
|
||||
// connection dropped, ctx cancelled) — the
|
||||
// container's fate is UNKNOWN, not gone.
|
||||
// Callers that gate destructive bookkeeping (e.g. deleting sidecar
|
||||
// metadata after a container remove) must treat a non-nil err as
|
||||
// "do not assume removal succeeded."
|
||||
ContainerExists(ctx context.Context, name string) (exists bool, err error)
|
||||
|
||||
// ---- Container-path file operations ----
|
||||
// These operate inside the running container (or container filesystem
|
||||
// for stopped containers where the runtime allows it). Used by the
|
||||
// file manager for volume-backed instances where the host-side bind
|
||||
// mount is absent or empty.
|
||||
|
||||
// ExecCapture runs a command inside the container and returns its
|
||||
// captured stdout + stderr + exit code. Requires container to be running.
|
||||
ExecCapture(ctx context.Context, id string, cmd []string) (stdout, stderr []byte, exitCode int, err error)
|
||||
|
||||
// CopyFileFromContainer returns the raw contents of a single file at
|
||||
// containerPath. Works even on stopped containers.
|
||||
CopyFileFromContainer(ctx context.Context, id, containerPath string) ([]byte, error)
|
||||
|
||||
// CopyFileToContainer writes content to containerPath, creating
|
||||
// parent dirs inside the container if the container is running.
|
||||
CopyFileToContainer(ctx context.Context, id, containerPath string, content []byte) error
|
||||
|
||||
// CopyTarToContainer writes a raw tar stream onto the container at
|
||||
// dstDir. The tar entries are extracted in place by the Docker API.
|
||||
// Used by the file manager's extract path so we can build one big
|
||||
// tar from the archive's entries and ship it in a single API call.
|
||||
CopyTarToContainer(ctx context.Context, id, dstDir string, tarStream io.Reader) error
|
||||
|
||||
// CopyDirFromContainer returns a tar stream of the contents at
|
||||
// containerPath. For files: a single-entry tar. For directories:
|
||||
// one entry per file/dir under it. Used by the compress path so the
|
||||
// agent can iterate file contents without per-file round-trips.
|
||||
CopyDirFromContainer(ctx context.Context, id, containerPath string) (io.ReadCloser, error)
|
||||
|
||||
// AttachStdio attaches to the container's stdin/stdout/stderr stream
|
||||
// and returns a read/write/close handle. Used by the stdio RCON adapter
|
||||
// for games whose admin surface is stdin-only. Requires the container
|
||||
// to have been created with OpenStdin=true.
|
||||
AttachStdio(ctx context.Context, id string) (io.ReadWriteCloser, error)
|
||||
|
||||
// ContainerMounts returns the mount points of a container by name.
|
||||
// Works on stopped containers (the daemon keeps the spec). Used by the
|
||||
// cluster player-save snapshotter to find the host path backing a 7DTD
|
||||
// instance's /cluster bind without depending on a path convention.
|
||||
ContainerMounts(ctx context.Context, name string) ([]MountInfo, error)
|
||||
|
||||
// Name returns the runtime kind ("docker", "host") for logging.
|
||||
Name() string
|
||||
}
|
||||
|
||||
// MountInfo is one container mount point, host-side. Type is "bind" or
|
||||
// "volume"; Source is the host path backing it (for a named volume that's
|
||||
// the docker volume's _data dir); Destination is the in-container path.
|
||||
type MountInfo struct {
|
||||
Type string
|
||||
Source string
|
||||
Destination string
|
||||
Name string
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
// Package state runs the per-instance state-tracking pipeline:
|
||||
// - RCON poll loops that execute manifest-declared commands and parse the
|
||||
// output into AppStateUpdate messages (players online, etc.)
|
||||
// - log-line event matching that runs manifest-declared regexes against
|
||||
// each log line and emits PlayerEvent messages (join/leave/chat/death)
|
||||
//
|
||||
// A Tracker is owned by the dispatcher for the lifetime of a running
|
||||
// instance. It is stopped by cancelling the context passed to Run.
|
||||
package state
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/rcon"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
// Emitter is the upward path for tracker output. Implementations forward
|
||||
// messages onto the gRPC stream back to the Controller.
|
||||
type Emitter interface {
|
||||
EmitAppState(*panelv1.AppStateUpdate)
|
||||
EmitPlayerEvent(*panelv1.PlayerEvent)
|
||||
}
|
||||
|
||||
// Config wires the tracker to an instance + its RCON endpoint.
|
||||
type Config struct {
|
||||
InstanceID string
|
||||
Manifest *modulepkg.Manifest
|
||||
RCONAddr string // "host:port" — required for tcp adapters (telnet, source_rcon)
|
||||
RCONPassword string
|
||||
// PasswordFunc, if set, overrides RCONPassword on each dial attempt.
|
||||
// Used when the module's entrypoint generates an RCON secret at first
|
||||
// boot (e.g. 7DTD's TelnetPassword in /game-saves/.panel-telnet-password).
|
||||
// Called with the dial context; the returned string is used as the
|
||||
// password for that single attempt.
|
||||
PasswordFunc func(ctx context.Context) (string, error)
|
||||
Emitter Emitter
|
||||
// ContainerID is the Docker container name/ID for the stdio adapter.
|
||||
// Ignored by tcp adapters. The stdio adapter calls Stdio.AttachStdio
|
||||
// with this id to get a stdin/stdout handle to the running game.
|
||||
ContainerID string
|
||||
// Stdio is the backend that knows how to attach to a container's
|
||||
// stdio. Required for stdio adapter, nil otherwise.
|
||||
Stdio rcon.StdioBackend
|
||||
// OnRconResult is an optional callback invoked after every RCON
|
||||
// exec attempt (poll loop or operator-driven). success=true means
|
||||
// the engine answered; success=false means the exec failed (dial,
|
||||
// timeout, dead conn). Used by the dispatcher's arkHangGuard to
|
||||
// count consecutive failures and trigger an auto-restart when the
|
||||
// Wine 9 listener wedges. Nil-safe: callers should check before invoke.
|
||||
OnRconResult func(success bool)
|
||||
}
|
||||
|
||||
// Tracker is created at instance start time and run in a goroutine.
|
||||
type Tracker struct {
|
||||
log *slog.Logger
|
||||
cfg Config
|
||||
events []compiledEvent
|
||||
// stateRE holds the state-source parse regexes, compiled once at New so
|
||||
// a bad pattern surfaces as a construction error instead of silently
|
||||
// recompiling (and silently failing) on every poll tick. Keyed by the
|
||||
// pattern string — pollOnce receives the StateSource by value, so the
|
||||
// pattern is the stable identity.
|
||||
stateRE map[string]*regexp.Regexp
|
||||
|
||||
mu sync.Mutex
|
||||
client rcon.Client
|
||||
}
|
||||
|
||||
type compiledEvent struct {
|
||||
name string
|
||||
re *regexp.Regexp
|
||||
kind panelv1.PlayerEvent_Kind
|
||||
}
|
||||
|
||||
// New constructs a Tracker. Event patterns are compiled now so config errors
|
||||
// surface before Run is called.
|
||||
func New(log *slog.Logger, cfg Config) (*Tracker, error) {
|
||||
if cfg.Manifest == nil {
|
||||
return nil, errors.New("manifest is required")
|
||||
}
|
||||
if cfg.Emitter == nil {
|
||||
return nil, errors.New("emitter is required")
|
||||
}
|
||||
events, err := compileEvents(cfg.Manifest.Events)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Compile state-source parse patterns up front — a bad pattern is a
|
||||
// manifest bug and should fail loudly at init, not degrade into a
|
||||
// per-poll recompile that silently yields no state updates.
|
||||
stateRE := map[string]*regexp.Regexp{}
|
||||
for i := range cfg.Manifest.StateSources {
|
||||
p := cfg.Manifest.StateSources[i].Parse
|
||||
if p == nil || p.Kind != "regex" || p.Pattern == "" {
|
||||
continue
|
||||
}
|
||||
re, err := regexp.Compile(p.Pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("state source %d: compile pattern: %w", i, err)
|
||||
}
|
||||
stateRE[p.Pattern] = re
|
||||
}
|
||||
return &Tracker{
|
||||
log: log.With("instance_id", cfg.InstanceID, "component", "state"),
|
||||
cfg: cfg,
|
||||
events: events,
|
||||
stateRE: stateRE,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Client returns the currently-connected RCON client, or nil if the tracker
|
||||
// hasn't finished dialing (or the manifest has no RCON at all). Used by the
|
||||
// dispatcher to handle ad-hoc RCON commands from operators.
|
||||
func (t *Tracker) Client() rcon.Client {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.client
|
||||
}
|
||||
|
||||
// Exec runs one operator command via the tracker's cached client. On a dead
|
||||
// connection (EOF, broken pipe — Palworld closes RCON after 60s idle) we
|
||||
// redial once and retry, so operators don't see the 60s-idle EOF in the UI.
|
||||
// Returns ("", "not_connected", ...) if the tracker hasn't finished dialing.
|
||||
func (t *Tracker) Exec(ctx context.Context, cmd string) (string, error) {
|
||||
t.mu.Lock()
|
||||
client := t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return "", errors.New("RCON not yet connected; try again in a moment")
|
||||
}
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := client.Exec(execCtx, cmd)
|
||||
if err == nil || !isDeadConnErr(err) {
|
||||
return out, err
|
||||
}
|
||||
t.redial(ctx)
|
||||
t.mu.Lock()
|
||||
client = t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return out, err
|
||||
}
|
||||
retryCtx, retryCancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer retryCancel()
|
||||
return client.Exec(retryCtx, cmd)
|
||||
}
|
||||
|
||||
// OnLogLine runs each log line through compiled event patterns and emits a
|
||||
// PlayerEvent on the first match. Called from the dispatcher's log pump; must
|
||||
// be fast and non-blocking.
|
||||
func (t *Tracker) OnLogLine(line string) {
|
||||
for _, ev := range t.events {
|
||||
m := ev.re.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
name := firstGroup(ev.re, m, "name", "player_name")
|
||||
id := firstGroup(ev.re, m, "platform_id", "owner", "cross_id", "id", "player_id")
|
||||
detail := firstGroup(ev.re, m, "msg", "detail", "reason")
|
||||
t.cfg.Emitter.EmitPlayerEvent(&panelv1.PlayerEvent{
|
||||
InstanceId: t.cfg.InstanceID,
|
||||
Kind: ev.kind,
|
||||
PlayerName: name,
|
||||
PlayerId: id,
|
||||
Detail: detail,
|
||||
At: timestamppb.Now(),
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Run connects to RCON (with retries) and starts one goroutine per declared
|
||||
// RCON state source. Returns when ctx is cancelled.
|
||||
//
|
||||
// If the manifest has no RCON config at all, Run is a no-op that blocks on
|
||||
// ctx.Done. If it has RCON but no state sources, we still dial once so
|
||||
// operator ad-hoc commands from the UI work — state-source poll loops just
|
||||
// don't spawn.
|
||||
func (t *Tracker) Run(ctx context.Context) {
|
||||
if t.cfg.Manifest.RCON == nil {
|
||||
<-ctx.Done()
|
||||
return
|
||||
}
|
||||
|
||||
client, err := t.dialWithRetries(ctx)
|
||||
if err != nil {
|
||||
// Context cancelled before we could connect; nothing to do.
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.client = client
|
||||
t.mu.Unlock()
|
||||
defer func() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.client != nil {
|
||||
_ = t.client.Close()
|
||||
t.client = nil
|
||||
}
|
||||
}()
|
||||
|
||||
t.log.Info("rcon connected", "addr", t.cfg.RCONAddr, "adapter", t.cfg.Manifest.RCON.Adapter)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range t.cfg.Manifest.StateSources {
|
||||
ss := t.cfg.Manifest.StateSources[i]
|
||||
if ss.Type != "rcon" || ss.Every.Std() <= 0 {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
t.runRCONPoller(ctx, ss)
|
||||
}()
|
||||
}
|
||||
// Block until the tracker is cancelled so the defer (which closes the
|
||||
// rcon client) fires on teardown, not immediately. With no rcon state
|
||||
// sources wg.Wait() returns instantly — without this extra wait the
|
||||
// client would vanish right after Run dialed, making ad-hoc commands
|
||||
// from the UI fail with "RCON not yet connected" forever.
|
||||
<-ctx.Done()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (t *Tracker) hasRCONStateSource() bool {
|
||||
for _, ss := range t.cfg.Manifest.StateSources {
|
||||
if ss.Type == "rcon" && ss.Every.Std() > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *Tracker) dialWithRetries(ctx context.Context) (rcon.Client, error) {
|
||||
rc := t.cfg.Manifest.RCON
|
||||
dialer, err := rcon.DialerFor(rc.Adapter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// stdio has no host/port — it attaches to the running container's stdin.
|
||||
// We still retry-loop because the container may not be up yet when the
|
||||
// tracker starts (race between Start + activate).
|
||||
if rc.Adapter == "stdio" {
|
||||
backoff := 2 * time.Second
|
||||
for {
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
client, err := dialer.Dial(dialCtx, rcon.DialOptions{
|
||||
ContainerID: t.cfg.ContainerID,
|
||||
Stdio: t.cfg.Stdio,
|
||||
})
|
||||
cancel()
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
t.log.Warn("stdio attach failed, retrying", "err", err, "backoff", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 30*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
host, portStr, err := net.SplitHostPort(t.cfg.RCONAddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rcon addr %q: %w", t.cfg.RCONAddr, err)
|
||||
}
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rcon port %q: %w", portStr, err)
|
||||
}
|
||||
|
||||
// Dial-retry backoff capped at 8s (not 30s) — during the post-Start
|
||||
// warm-up window the server's RCON flips between responsive and slow
|
||||
// every few seconds, and a 32s ceiling meant one unlucky timeout
|
||||
// stranded the panel for half a minute before the next retry. Cap
|
||||
// at 8s so ragnarok/etc. get back onto RCON within a few seconds of
|
||||
// ASA actually being ready, without hammering the port on a truly
|
||||
// dead server (8s × a few retries is ~30s before the tracker gives
|
||||
// up per-cycle, which the agent's outer loop still reschedules).
|
||||
backoff := 2 * time.Second
|
||||
for {
|
||||
dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
// Resolve password per-attempt when PasswordFunc is set — the
|
||||
// module's entrypoint may be mid-generation on first boot, so re-
|
||||
// reading on each retry lets us pick up the final value as soon
|
||||
// as it's written.
|
||||
pw := t.cfg.RCONPassword
|
||||
if t.cfg.PasswordFunc != nil {
|
||||
if p, err := t.cfg.PasswordFunc(dialCtx); err == nil && p != "" {
|
||||
pw = p
|
||||
}
|
||||
}
|
||||
client, err := dialer.Dial(dialCtx, rcon.DialOptions{
|
||||
Host: host,
|
||||
Port: port,
|
||||
Password: pw,
|
||||
ConnectTimeout: 5 * time.Second,
|
||||
ContainerID: t.cfg.ContainerID,
|
||||
})
|
||||
cancel()
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
t.log.Warn("rcon dial failed, retrying", "err", err, "backoff", backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 8*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) runRCONPoller(ctx context.Context, ss modulepkg.StateSource) {
|
||||
every := ss.Every.Std()
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
t.pollOnce(ctx, ss) // immediate first tick
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
t.pollOnce(ctx, ss)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) pollOnce(ctx context.Context, ss modulepkg.StateSource) {
|
||||
t.mu.Lock()
|
||||
client := t.client
|
||||
t.mu.Unlock()
|
||||
if client == nil {
|
||||
return
|
||||
}
|
||||
execCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
out, err := client.Exec(execCtx, ss.Command)
|
||||
if err != nil {
|
||||
t.log.Warn("rcon exec failed", "cmd", ss.Command, "err", err)
|
||||
if t.cfg.OnRconResult != nil {
|
||||
t.cfg.OnRconResult(false)
|
||||
}
|
||||
if isDeadConnErr(err) {
|
||||
t.redial(ctx)
|
||||
}
|
||||
return
|
||||
}
|
||||
if t.cfg.OnRconResult != nil {
|
||||
t.cfg.OnRconResult(true)
|
||||
}
|
||||
app := t.parseState(out, ss)
|
||||
if app == nil {
|
||||
// Parser yielded no structured fields (regex didn't match this game's
|
||||
// output, fields all empty, etc.) but the RCON exec succeeded — that
|
||||
// alone is proof the server is responsive. Emit a bare update keyed
|
||||
// by InstanceID so the dispatcher can use it to promote a stale
|
||||
// CRASHED status back to RUNNING.
|
||||
app = &panelv1.AppStateUpdate{InstanceId: t.cfg.InstanceID}
|
||||
}
|
||||
t.cfg.Emitter.EmitAppState(app)
|
||||
}
|
||||
|
||||
// isDeadConnErr returns true for errors that indicate the RCON socket is no
|
||||
// longer usable (EOF, connection reset, broken pipe). Palworld's RCON, for
|
||||
// instance, closes the socket after ~60s idle — any subsequent Exec fails
|
||||
// with EOF and stays broken until we redial.
|
||||
func isDeadConnErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
return strings.Contains(s, "EOF") ||
|
||||
strings.Contains(s, "broken pipe") ||
|
||||
strings.Contains(s, "connection reset") ||
|
||||
strings.Contains(s, "forcibly closed") ||
|
||||
strings.Contains(s, "use of closed network connection")
|
||||
}
|
||||
|
||||
// redial closes the current client and dials a fresh one. Runs inline with
|
||||
// the poll loop so the next tick gets a working client. On failure we leave
|
||||
// t.client = nil and the next poll will call us again.
|
||||
func (t *Tracker) redial(ctx context.Context) {
|
||||
t.mu.Lock()
|
||||
if t.client != nil {
|
||||
_ = t.client.Close()
|
||||
t.client = nil
|
||||
}
|
||||
t.mu.Unlock()
|
||||
t.log.Info("rcon redialing after dead connection")
|
||||
fresh, err := t.dialWithRetries(ctx)
|
||||
if err != nil {
|
||||
t.log.Warn("rcon redial failed", "err", err)
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.client = fresh
|
||||
t.mu.Unlock()
|
||||
t.log.Info("rcon reconnected", "addr", t.cfg.RCONAddr, "adapter", t.cfg.Manifest.RCON.Adapter)
|
||||
}
|
||||
|
||||
// parseState runs the configured parser against raw RCON output using the
|
||||
// regex precompiled at New, returning an AppStateUpdate or nil if nothing
|
||||
// matched.
|
||||
func (t *Tracker) parseState(out string, ss modulepkg.StateSource) *panelv1.AppStateUpdate {
|
||||
if ss.Parse == nil || ss.Parse.Kind != "regex" {
|
||||
return nil
|
||||
}
|
||||
re := t.stateRE[ss.Parse.Pattern]
|
||||
if re == nil {
|
||||
return nil
|
||||
}
|
||||
return parseRegexCompiled(t.cfg.InstanceID, out, re, ss.Parse)
|
||||
}
|
||||
|
||||
// parseStateOutput is the uncached variant (compiles the pattern per call).
|
||||
// Kept for tests and out-of-tracker callers; the tracker's poll loop uses
|
||||
// the precompiled parseState path.
|
||||
func parseStateOutput(instanceID, out string, ss modulepkg.StateSource) *panelv1.AppStateUpdate {
|
||||
if ss.Parse == nil {
|
||||
return nil
|
||||
}
|
||||
switch ss.Parse.Kind {
|
||||
case "regex":
|
||||
return parseRegex(instanceID, out, ss.Parse)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func parseRegex(instanceID, out string, p *modulepkg.ParseConfig) *panelv1.AppStateUpdate {
|
||||
re, err := regexp.Compile(p.Pattern)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return parseRegexCompiled(instanceID, out, re, p)
|
||||
}
|
||||
|
||||
func parseRegexCompiled(instanceID, out string, re *regexp.Regexp, p *modulepkg.ParseConfig) *panelv1.AppStateUpdate {
|
||||
m := re.FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
app := &panelv1.AppStateUpdate{
|
||||
InstanceId: instanceID,
|
||||
At: timestamppb.Now(),
|
||||
}
|
||||
for outName, groupName := range p.Fields {
|
||||
val := firstGroup(re, m, groupName)
|
||||
switch outName {
|
||||
case "players_online":
|
||||
if n, err := strconv.Atoi(val); err == nil {
|
||||
app.PlayersOnline = int32(n)
|
||||
}
|
||||
case "players_max":
|
||||
if n, err := strconv.Atoi(val); err == nil {
|
||||
app.PlayersMax = int32(n)
|
||||
}
|
||||
case "uptime_seconds":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
app.UptimeSeconds = n
|
||||
}
|
||||
default:
|
||||
if app.ModuleFields == nil {
|
||||
app.ModuleFields = map[string]string{}
|
||||
}
|
||||
app.ModuleFields[outName] = val
|
||||
}
|
||||
}
|
||||
return app
|
||||
}
|
||||
|
||||
func compileEvents(events map[string]modulepkg.Event) ([]compiledEvent, error) {
|
||||
out := make([]compiledEvent, 0, len(events))
|
||||
for name, ev := range events {
|
||||
re, err := regexp.Compile(ev.Pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("event %q: compile pattern: %w", name, err)
|
||||
}
|
||||
out = append(out, compiledEvent{
|
||||
name: name,
|
||||
re: re,
|
||||
kind: parseKind(ev.Kind),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseKind(s string) panelv1.PlayerEvent_Kind {
|
||||
switch s {
|
||||
case "join":
|
||||
return panelv1.PlayerEvent_KIND_JOIN
|
||||
case "leave":
|
||||
return panelv1.PlayerEvent_KIND_LEAVE
|
||||
case "chat":
|
||||
return panelv1.PlayerEvent_KIND_CHAT
|
||||
case "death":
|
||||
return panelv1.PlayerEvent_KIND_DEATH
|
||||
default:
|
||||
return panelv1.PlayerEvent_KIND_CUSTOM
|
||||
}
|
||||
}
|
||||
|
||||
// firstGroup returns the first non-empty named submatch from names. Missing
|
||||
// or empty groups are treated as absent so we try the next name.
|
||||
func firstGroup(re *regexp.Regexp, m []string, names ...string) string {
|
||||
for _, name := range names {
|
||||
i := re.SubexpIndex(name)
|
||||
if i >= 0 && i < len(m) && m[i] != "" {
|
||||
return m[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
||||
)
|
||||
|
||||
type captureEmitter struct {
|
||||
appStates []*panelv1.AppStateUpdate
|
||||
playerEvents []*panelv1.PlayerEvent
|
||||
}
|
||||
|
||||
func (c *captureEmitter) EmitAppState(a *panelv1.AppStateUpdate) { c.appStates = append(c.appStates, a) }
|
||||
func (c *captureEmitter) EmitPlayerEvent(p *panelv1.PlayerEvent) { c.playerEvents = append(c.playerEvents, p) }
|
||||
|
||||
func newTestTracker(t *testing.T, m *modulepkg.Manifest) (*Tracker, *captureEmitter) {
|
||||
t.Helper()
|
||||
cap := &captureEmitter{}
|
||||
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
tr, err := New(log, Config{
|
||||
InstanceID: "inst-1",
|
||||
Manifest: m,
|
||||
Emitter: cap,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
return tr, cap
|
||||
}
|
||||
|
||||
func Test7DTDJoinPattern(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, cap := newTestTracker(t, manifest)
|
||||
|
||||
// Real-world 7DTD log line shape
|
||||
line := `2026-04-19T10:00:00 42.123 INF PlayerSpawnedInWorld (by system): PltfmId='Steam_76561198000000001', CrossId='EOS_00000000', OwnerID='Steam_76561198000000001', PlayerName='Alice'`
|
||||
tr.OnLogLine(line)
|
||||
|
||||
if len(cap.playerEvents) != 1 {
|
||||
t.Fatalf("expected 1 player event, got %d", len(cap.playerEvents))
|
||||
}
|
||||
ev := cap.playerEvents[0]
|
||||
if ev.Kind != panelv1.PlayerEvent_KIND_JOIN {
|
||||
t.Errorf("kind = %v, want JOIN", ev.Kind)
|
||||
}
|
||||
if ev.PlayerName != "Alice" {
|
||||
t.Errorf("name = %q, want Alice", ev.PlayerName)
|
||||
}
|
||||
if ev.PlayerId == "" {
|
||||
t.Errorf("player_id empty; expected Steam_... from named groups")
|
||||
}
|
||||
}
|
||||
|
||||
func Test7DTDChatPattern(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, cap := newTestTracker(t, manifest)
|
||||
|
||||
line := `2026-04-19T10:00:00 42.123 INF Chat (from 'Steam_76561198000000001', entity id '123', to 'Global'): 'Alice': hello world`
|
||||
tr.OnLogLine(line)
|
||||
|
||||
if len(cap.playerEvents) != 1 {
|
||||
t.Fatalf("expected 1 player event, got %d", len(cap.playerEvents))
|
||||
}
|
||||
ev := cap.playerEvents[0]
|
||||
if ev.Kind != panelv1.PlayerEvent_KIND_CHAT {
|
||||
t.Errorf("kind = %v, want CHAT", ev.Kind)
|
||||
}
|
||||
if ev.PlayerName != "Alice" {
|
||||
t.Errorf("name = %q, want Alice", ev.PlayerName)
|
||||
}
|
||||
if ev.Detail != "hello world" {
|
||||
t.Errorf("detail = %q, want 'hello world'", ev.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func Test7DTDLPParse(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
var lp *modulepkg.StateSource
|
||||
for i := range manifest.StateSources {
|
||||
if manifest.StateSources[i].Type == "rcon" {
|
||||
lp = &manifest.StateSources[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if lp == nil {
|
||||
t.Fatal("no rcon state source in 7dtd manifest")
|
||||
}
|
||||
|
||||
sample := "Total of 3 in the game\n\n1. id=1, Alice, pos=(0,0,0)\n2. id=2, Bob, pos=(1,0,0)\n3. id=3, Carol, pos=(2,0,0)"
|
||||
app := parseStateOutput("inst-1", sample, *lp)
|
||||
if app == nil {
|
||||
t.Fatal("parse returned nil")
|
||||
}
|
||||
if app.PlayersOnline != 3 {
|
||||
t.Errorf("players_online = %d, want 3", app.PlayersOnline)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineWithNoMatchYieldsNoEvent(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, cap := newTestTracker(t, manifest)
|
||||
tr.OnLogLine("this is some random log line that matches nothing")
|
||||
if len(cap.playerEvents) != 0 {
|
||||
t.Errorf("expected 0 events, got %d", len(cap.playerEvents))
|
||||
}
|
||||
}
|
||||
|
||||
// ---- WI-05: hoisted state-source regex compilation ----
|
||||
|
||||
// TestNewRejectsBadStateSourcePattern: a malformed parse regex must fail at
|
||||
// construction (surfacing the manifest bug at init) instead of silently
|
||||
// recompiling-and-failing on every poll tick.
|
||||
func TestNewRejectsBadStateSourcePattern(t *testing.T) {
|
||||
m := &modulepkg.Manifest{
|
||||
StateSources: []modulepkg.StateSource{{
|
||||
Type: "rcon",
|
||||
Command: "lp",
|
||||
Parse: &modulepkg.ParseConfig{Kind: "regex", Pattern: "("},
|
||||
}},
|
||||
}
|
||||
log := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
if _, err := New(log, Config{InstanceID: "i", Manifest: m, Emitter: &captureEmitter{}}); err == nil {
|
||||
t.Fatal("New accepted an invalid state-source regex; want construction error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseStateUsesPrecompiledRegex drives the tracker's precompiled parse
|
||||
// path with the real 7DTD manifest and confirms it matches the uncached
|
||||
// package-level parser.
|
||||
func TestParseStateUsesPrecompiledRegex(t *testing.T) {
|
||||
manifest, err := modulepkg.LoadFile("../../../modules/7dtd/module.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("load 7dtd: %v", err)
|
||||
}
|
||||
tr, _ := newTestTracker(t, manifest)
|
||||
var lp *modulepkg.StateSource
|
||||
for i := range manifest.StateSources {
|
||||
if manifest.StateSources[i].Type == "rcon" {
|
||||
lp = &manifest.StateSources[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if lp == nil {
|
||||
t.Fatal("no rcon state source in 7dtd manifest")
|
||||
}
|
||||
sample := "Total of 3 in the game\n\n1. id=1, Alice, pos=(0,0,0)\n2. id=2, Bob, pos=(1,0,0)\n3. id=3, Carol, pos=(2,0,0)"
|
||||
app := tr.parseState(sample, *lp)
|
||||
if app == nil {
|
||||
t.Fatal("precompiled parseState returned nil")
|
||||
}
|
||||
if app.PlayersOnline != 3 {
|
||||
t.Errorf("players_online = %d, want 3", app.PlayersOnline)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
"github.com/dbledeez/panel/pkg/steamvdf"
|
||||
)
|
||||
|
||||
// FetchAppInfoBranches runs a one-shot SteamCMD sidecar with
|
||||
//
|
||||
// +login anonymous +app_info_update 1 +app_info_print <appID> +quit
|
||||
//
|
||||
// captures its stdout, and parses the depots.branches map (branch name →
|
||||
// latest buildid). CHECK-ONLY: no game volumes are mounted, so this can
|
||||
// never touch an install. Only the shared panel-steamcmd-auth volume is
|
||||
// attached (same as update runs) so Steam's client config cache persists
|
||||
// across calls.
|
||||
//
|
||||
// tag disambiguates the sidecar container name per caller (usually the
|
||||
// instance id); appID appears too so parallel checks for different apps
|
||||
// can't collide.
|
||||
func FetchAppInfoBranches(ctx context.Context, rt runtime.Runtime, tag, appID string, log func(string)) (map[string]string, error) {
|
||||
if appID == "" {
|
||||
return nil, fmt.Errorf("app_info: app_id is required")
|
||||
}
|
||||
args := []string{
|
||||
"+login", "anonymous",
|
||||
"+app_info_update", "1",
|
||||
"+app_info_print", appID,
|
||||
"+quit",
|
||||
}
|
||||
spec := runtime.InstanceSpec{
|
||||
InstanceID: fmt.Sprintf("%s-appinfo-%s", tag, appID),
|
||||
IsSidecar: true,
|
||||
Image: sidecarImage,
|
||||
Command: args,
|
||||
Volumes: []runtime.VolumeSpec{{
|
||||
VolumeName: "panel-steamcmd-auth",
|
||||
ContainerPath: "/root/.local/share/Steam",
|
||||
Type: "volume",
|
||||
}},
|
||||
RestartPolicy: "no",
|
||||
// Same seccomp note as the update sidecar (steamcmd.go): newer
|
||||
// Docker default profiles ENOSYS a syscall the 32-bit Steam
|
||||
// runtime needs.
|
||||
SecurityOpts: []string{"seccomp=unconfined"},
|
||||
}
|
||||
contID, err := rt.Create(ctx, spec)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create app_info sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = rt.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := rt.Start(ctx, contID); err != nil {
|
||||
return nil, fmt.Errorf("start app_info sidecar: %w", err)
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = rt.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
out.WriteString(line)
|
||||
out.WriteByte('\n')
|
||||
})
|
||||
}()
|
||||
exitCode, err := rt.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wait app_info sidecar: %w", err)
|
||||
}
|
||||
if log != nil {
|
||||
log(fmt.Sprintf("app_info: steamcmd exit %d, %d bytes of output", exitCode, out.Len()))
|
||||
}
|
||||
// SteamCMD sometimes exits non-zero even after printing valid app
|
||||
// info — parse first, only surface the exit code when parsing fails.
|
||||
branches, perr := steamvdf.ParseAppInfoBranches(out.String(), appID)
|
||||
if perr != nil {
|
||||
if exitCode != 0 {
|
||||
return nil, fmt.Errorf("app_info: steamcmd exit %d (%v)", exitCode, perr)
|
||||
}
|
||||
return nil, perr
|
||||
}
|
||||
return branches, nil
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
archivepkg "github.com/dbledeez/panel/agent/internal/archive"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
"github.com/dbledeez/panel/pkg/version"
|
||||
)
|
||||
|
||||
// DirectProvider fetches a URL and writes the response body to a target
|
||||
// path under the instance. Used for vanilla Minecraft jars, custom mod
|
||||
// packs hosted on any HTTP server, etc.
|
||||
type DirectProvider struct{}
|
||||
|
||||
// Name identifies this provider kind in logs / manifest.
|
||||
func (DirectProvider) Name() string { return "direct" }
|
||||
|
||||
const maxDirectDownloadBytes = 1 << 30 // 1 GiB sanity cap
|
||||
|
||||
// Update downloads spec.URL and writes it to spec.TargetPath relative
|
||||
// to the instance. Optional SHA256 verification is reserved for a future
|
||||
// `sha256:` field on the spec.
|
||||
func (DirectProvider) Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error {
|
||||
if spec.URL == "" {
|
||||
return fmt.Errorf("direct provider: url is required")
|
||||
}
|
||||
if spec.TargetPath == "" {
|
||||
return fmt.Errorf("direct provider: target_path is required")
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", spec.URL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", version.UserAgent())
|
||||
|
||||
client := &http.Client{Timeout: 5 * time.Minute}
|
||||
uc.Log(fmt.Sprintf("GET %s", spec.URL))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("http status %d", resp.StatusCode)
|
||||
}
|
||||
if resp.ContentLength > 0 {
|
||||
uc.Log(fmt.Sprintf("Content-Length: %d bytes", resp.ContentLength))
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxDirectDownloadBytes+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
if int64(len(data)) > maxDirectDownloadBytes {
|
||||
return fmt.Errorf("response exceeds %d-byte cap", maxDirectDownloadBytes)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
uc.Log(fmt.Sprintf("downloaded %d bytes (sha256:%s)", len(data), hex.EncodeToString(sum[:])))
|
||||
|
||||
// Archive handling. Some direct URLs serve a whole game tree as an
|
||||
// archive (factorio's headless .tar.xz) — dumping those bytes as a
|
||||
// single file at TargetPath is never what the module wants. Extract
|
||||
// when the module opted in (spec.Extract) or when the content is a
|
||||
// recognized archive AND TargetPath looks like a directory (no file
|
||||
// extension). A TargetPath with an extension is always written
|
||||
// verbatim: minecraft server.jar IS a zip by magic bytes, and
|
||||
// terraria's entrypoint expects its .zip left on disk.
|
||||
format := archivepkg.DetectFormat(data, spec.URL)
|
||||
if spec.Extract && format == "" {
|
||||
return fmt.Errorf("extract requested but content is not a recognized archive (url %s)", spec.URL)
|
||||
}
|
||||
if format != "" && (spec.Extract || path.Ext(path.Base(spec.TargetPath)) == "") {
|
||||
uc.Log(fmt.Sprintf("content is a %s archive — extracting into %s", format, spec.TargetPath))
|
||||
entries, bytesOut, err := extractToInstance(ctx, uc, spec.TargetPath, data, spec.URL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extract into %s: %w", spec.TargetPath, err)
|
||||
}
|
||||
uc.Log(fmt.Sprintf("extracted %d entries (%d bytes) into %s", entries, bytesOut, spec.TargetPath))
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := writeToInstance(ctx, uc, spec.TargetPath, data); err != nil {
|
||||
return fmt.Errorf("write target: %w", err)
|
||||
}
|
||||
uc.Log(fmt.Sprintf("wrote %s", spec.TargetPath))
|
||||
return nil
|
||||
}
|
||||
|
||||
// containerTargetAbs resolves a provider target_path to a container-
|
||||
// absolute path. An absolute target that is equal to or under one of
|
||||
// the module's declared browseable roots or volume mount points is
|
||||
// used as-is (multi-root modules: factorio's /game volume sits BESIDE
|
||||
// its /game-saves default root). Anything else keeps the historical
|
||||
// behavior of joining under BrowseableRoot — terraria's entrypoint,
|
||||
// for one, expects /terraria-server.zip to land at
|
||||
// /game-saves/terraria-server.zip via exactly that join.
|
||||
func containerTargetAbs(uc *Context, targetPath string) string {
|
||||
if strings.HasPrefix(targetPath, "/") && uc.Manifest != nil {
|
||||
cleaned := path.Clean(targetPath)
|
||||
var roots []string
|
||||
if d := uc.Manifest.Runtime.Docker; d != nil {
|
||||
for _, r := range d.BrowseableRoots {
|
||||
if r.Path != "" {
|
||||
roots = append(roots, r.Path)
|
||||
}
|
||||
}
|
||||
for _, v := range d.Volumes {
|
||||
if v.Container != "" {
|
||||
roots = append(roots, v.Container)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, r := range roots {
|
||||
if cleaned == r || strings.HasPrefix(cleaned+"/", r+"/") {
|
||||
return cleaned
|
||||
}
|
||||
}
|
||||
}
|
||||
return path.Join(uc.BrowseableRoot, targetPath)
|
||||
}
|
||||
|
||||
// extractToInstance expands archive `data` into the instance-relative
|
||||
// directory targetPath. Mirrors writeToInstance's storage preference:
|
||||
// container-path ops when the instance has a BrowseableRoot and a
|
||||
// container (re-encode the archive as one tar stream and let Docker
|
||||
// extract it in place — same pattern as the file manager's FsExtract),
|
||||
// falling back to plain filesystem extraction under DataPath.
|
||||
func extractToInstance(ctx context.Context, uc *Context, targetPath string, data []byte, originalName string) (int64, int64, error) {
|
||||
if targetPath == "" {
|
||||
return 0, 0, fmt.Errorf("target_path is required")
|
||||
}
|
||||
if uc.BrowseableRoot != "" && uc.ContainerID != "" {
|
||||
destAbs := containerTargetAbs(uc, targetPath)
|
||||
// Best-effort mkdir; only possible on a running container, and
|
||||
// the usual target is a volume mount point that already exists.
|
||||
_, _, _, _ = uc.Runtime.ExecCapture(ctx, uc.ContainerID, []string{"sh", "-c", "mkdir -p -- '" + destAbs + "'"})
|
||||
pr, pw := io.Pipe()
|
||||
emitErr := make(chan error, 1)
|
||||
var entries, bytesOut int64
|
||||
go func() {
|
||||
n, b, err := archivepkg.WriteAsTar(data, originalName, pw)
|
||||
entries, bytesOut = n, b
|
||||
_ = pw.CloseWithError(err)
|
||||
emitErr <- err
|
||||
}()
|
||||
if err := uc.Runtime.CopyTarToContainer(ctx, uc.ContainerID, destAbs, pr); err != nil {
|
||||
// Prefer the encoder's error when it carries archive-format
|
||||
// detail — but NOT when it's just the pipe slamming shut
|
||||
// because Docker aborted the copy (that would mask the real
|
||||
// error, e.g. "destination directory does not exist").
|
||||
if encErr := <-emitErr; encErr != nil && !errors.Is(encErr, io.ErrClosedPipe) {
|
||||
return entries, bytesOut, encErr
|
||||
}
|
||||
return entries, bytesOut, fmt.Errorf("copy to container %s: %w", destAbs, err)
|
||||
}
|
||||
if encErr := <-emitErr; encErr != nil {
|
||||
return entries, bytesOut, encErr
|
||||
}
|
||||
return entries, bytesOut, nil
|
||||
}
|
||||
if uc.DataPath == "" {
|
||||
return 0, 0, fmt.Errorf("no storage available: container has no BrowseableRoot and instance has no DataPath")
|
||||
}
|
||||
destAbs := filepath.Join(uc.DataPath, filepath.FromSlash(targetPath))
|
||||
if err := os.MkdirAll(destAbs, 0o755); err != nil {
|
||||
return 0, 0, fmt.Errorf("mkdir %s: %w", destAbs, err)
|
||||
}
|
||||
return archivepkg.Walk(data, originalName, func(name string, mode int64, isDir bool, body io.Reader) error {
|
||||
safe := archivepkg.SafeEntryPath(name)
|
||||
if safe == "" {
|
||||
return nil
|
||||
}
|
||||
out := filepath.Join(destAbs, filepath.FromSlash(safe))
|
||||
if isDir {
|
||||
return os.MkdirAll(out, 0o755)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
perm := os.FileMode(mode & 0o777)
|
||||
if perm == 0 {
|
||||
perm = 0o644
|
||||
}
|
||||
f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(f, body); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// makeFactorioTarXz builds a tiny tar.xz shaped like factorio's
|
||||
// headless tarball (factorio/bin/x64/factorio).
|
||||
func makeFactorioTarXz(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
xw, err := xz.NewWriter(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tw := tar.NewWriter(xw)
|
||||
body := []byte("ELF-not-really")
|
||||
if err := tw.WriteHeader(&tar.Header{Name: "factorio/bin/x64/factorio", Typeflag: tar.TypeReg, Mode: 0o755, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := xw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func testCtx(t *testing.T, dataPath string) *Context {
|
||||
t.Helper()
|
||||
return &Context{
|
||||
InstanceID: "test",
|
||||
DataPath: dataPath,
|
||||
Log: func(string) {},
|
||||
}
|
||||
}
|
||||
|
||||
// The factorio case: archive content + extensionless URL + extract:true
|
||||
// must land the extracted tree under TargetPath, not a raw blob.
|
||||
func TestDirectExtractsArchiveIntoTargetDir(t *testing.T) {
|
||||
blob := makeFactorioTarXz(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(blob)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/get-download/stable/headless/linux64", TargetPath: "/game", Extract: true}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin := filepath.Join(dir, "game", "factorio", "bin", "x64", "factorio")
|
||||
data, err := os.ReadFile(bin)
|
||||
if err != nil {
|
||||
t.Fatalf("expected extracted binary at %s: %v", bin, err)
|
||||
}
|
||||
if string(data) != "ELF-not-really" {
|
||||
t.Fatalf("binary content mismatch: %q", data)
|
||||
}
|
||||
// The old bug: raw blob written AS the target path.
|
||||
if fi, err := os.Stat(filepath.Join(dir, "game")); err != nil || !fi.IsDir() {
|
||||
t.Fatalf("target path should be a directory, got err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Same archive, no explicit extract flag: extensionless TargetPath
|
||||
// still triggers extraction (directory heuristic).
|
||||
func TestDirectHeuristicExtractsWithoutFlag(t *testing.T) {
|
||||
blob := makeFactorioTarXz(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(blob)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/linux64", TargetPath: "/game"}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "game", "factorio", "bin", "x64", "factorio")); err != nil {
|
||||
t.Fatalf("expected extraction: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The terraria/minecraft guard: a TargetPath WITH a file extension is
|
||||
// written verbatim even when the content has archive magic (jars are
|
||||
// zips; terraria's entrypoint unzips its own .zip).
|
||||
func TestDirectWritesArchiveVerbatimWhenTargetHasExtension(t *testing.T) {
|
||||
// A zip blob (like a .jar or terraria-server zip).
|
||||
var zipBuf bytes.Buffer
|
||||
zipBuf.Write([]byte{0x50, 0x4b, 0x03, 0x04}) // zip local-header magic
|
||||
zipBuf.WriteString("rest-of-zip-not-parsed")
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(zipBuf.Bytes())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/terraria-server-1449.zip", TargetPath: "/terraria-server.zip"}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, "terraria-server.zip"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(data, zipBuf.Bytes()) {
|
||||
t.Fatal("zip should have been written verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
// extract:true on non-archive content is a hard error, not a silent
|
||||
// verbatim write.
|
||||
func TestDirectExtractFlagOnNonArchiveErrors(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("just text"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/blob", TargetPath: "/game", Extract: true}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err == nil {
|
||||
t.Fatal("expected error for extract on non-archive content")
|
||||
}
|
||||
}
|
||||
|
||||
// containerTargetAbs: absolute targets matching a declared root or
|
||||
// volume mount stay container-absolute (factorio's /game volume);
|
||||
// anything else keeps the historical BrowseableRoot join (terraria's
|
||||
// /terraria-server.zip → /game-saves/terraria-server.zip).
|
||||
func TestContainerTargetAbs(t *testing.T) {
|
||||
uc := &Context{
|
||||
BrowseableRoot: "/game-saves",
|
||||
Manifest: &modulepkg.Manifest{
|
||||
Runtime: modulepkg.Runtime{Docker: &modulepkg.DockerSpec{
|
||||
BrowseableRoots: []modulepkg.BrowseableRoot{{Name: "Saves", Path: "/game-saves"}},
|
||||
Volumes: []modulepkg.Volume{
|
||||
{Type: "volume", Name: "panel-$INSTANCE_ID-saves", Container: "/game-saves"},
|
||||
{Type: "volume", Name: "panel-$INSTANCE_ID-game", Container: "/game"},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
cases := map[string]string{
|
||||
"/game": "/game",
|
||||
"/game/sub": "/game/sub",
|
||||
"/game-saves/mods": "/game-saves/mods",
|
||||
"/terraria-server.zip": "/game-saves/terraria-server.zip",
|
||||
"relative/file": "/game-saves/relative/file",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := containerTargetAbs(uc, in); got != want {
|
||||
t.Errorf("containerTargetAbs(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Non-archive content with an extensionless target keeps the old
|
||||
// single-file write behavior.
|
||||
func TestDirectNonArchiveStillWritesFile(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("binary-ish payload"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
spec := &modulepkg.UpdateProvider{Kind: "direct", URL: srv.URL + "/blob", TargetPath: "/some/file"}
|
||||
if err := (DirectProvider{}).Update(context.Background(), testCtx(t, dir), spec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, "some", "file"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "binary-ish payload" {
|
||||
t.Fatalf("content mismatch: %q", data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
"github.com/dbledeez/panel/pkg/version"
|
||||
)
|
||||
|
||||
// GitHubProvider queries a repo's latest release, finds an asset whose
|
||||
// name matches the provider's AssetRegex, and writes it to TargetPath.
|
||||
type GitHubProvider struct{}
|
||||
|
||||
// Name identifies this provider kind in logs / manifest.
|
||||
func (GitHubProvider) Name() string { return "github" }
|
||||
|
||||
type githubAsset struct {
|
||||
Name string `json:"name"`
|
||||
DownloadURL string `json:"browser_download_url"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type githubRelease struct {
|
||||
Name string `json:"name"`
|
||||
TagName string `json:"tag_name"`
|
||||
Assets []githubAsset `json:"assets"`
|
||||
}
|
||||
|
||||
// Update fetches the latest release of spec.Repo, picks the first asset
|
||||
// matching spec.AssetRegex, and writes its bytes to spec.TargetPath.
|
||||
func (GitHubProvider) Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error {
|
||||
if spec.Repo == "" {
|
||||
return fmt.Errorf("github provider: repo is required (owner/name)")
|
||||
}
|
||||
if spec.AssetRegex == "" {
|
||||
return fmt.Errorf("github provider: asset_regex is required")
|
||||
}
|
||||
if spec.TargetPath == "" {
|
||||
return fmt.Errorf("github provider: target_path is required")
|
||||
}
|
||||
re, err := regexp.Compile(spec.AssetRegex)
|
||||
if err != nil {
|
||||
return fmt.Errorf("asset_regex: %w", err)
|
||||
}
|
||||
|
||||
apiURL := fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", spec.Repo)
|
||||
uc.Log(fmt.Sprintf("querying GitHub: %s", apiURL))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("User-Agent", version.UserAgent())
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
return fmt.Errorf("github api %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
var release githubRelease
|
||||
if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
|
||||
return fmt.Errorf("decode release: %w", err)
|
||||
}
|
||||
uc.Log(fmt.Sprintf("latest release: %s (%s), %d assets", release.Name, release.TagName, len(release.Assets)))
|
||||
|
||||
var chosen *githubAsset
|
||||
for i := range release.Assets {
|
||||
if re.MatchString(release.Assets[i].Name) {
|
||||
chosen = &release.Assets[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if chosen == nil {
|
||||
return fmt.Errorf("no asset matches %q (saw: %s)", spec.AssetRegex, assetNames(release.Assets))
|
||||
}
|
||||
uc.Log(fmt.Sprintf("downloading asset %s (%d bytes)", chosen.Name, chosen.Size))
|
||||
|
||||
direct := &modulepkg.UpdateProvider{
|
||||
Kind: "direct",
|
||||
URL: chosen.DownloadURL,
|
||||
TargetPath: spec.TargetPath,
|
||||
}
|
||||
return DirectProvider{}.Update(ctx, uc, direct)
|
||||
}
|
||||
|
||||
func assetNames(as []githubAsset) string {
|
||||
out := ""
|
||||
for _, a := range as {
|
||||
if out != "" {
|
||||
out += ", "
|
||||
}
|
||||
out += a.Name
|
||||
}
|
||||
if out == "" {
|
||||
out = "(none)"
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Package updater implements the Generic Update system: pluggable
|
||||
// providers that fetch new versions of game files from SteamCMD,
|
||||
// GitHub releases, or a direct URL, then lay them down under the
|
||||
// instance's data path (bind mount) or container volume.
|
||||
//
|
||||
// The panel's manifest declares one or more update_providers per module;
|
||||
// the operator picks one to trigger. Each provider runs on the *Agent*
|
||||
// (close to the data) and streams progress back as LogLine envelopes
|
||||
// so the dashboard renders updates in the same events pane as normal
|
||||
// server output.
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// Context is the environment an update runs in.
|
||||
type Context struct {
|
||||
InstanceID string
|
||||
Manifest *modulepkg.Manifest
|
||||
ContainerID string // empty if container doesn't exist; present for running + stopped
|
||||
BrowseableRoot string // container-absolute install root, when applicable
|
||||
DataPath string // host bind-mount root, fallback
|
||||
Runtime runtime.Runtime // for container file ops + sidecar launches (future)
|
||||
Log func(line string) // streams a progress line upstream via LogLine
|
||||
|
||||
// Steam login credentials for SteamCMD apps that refuse +login
|
||||
// anonymous (DayZ, Arma, etc.). Populated by the controller from its
|
||||
// encrypted steam_credentials store, forwarded via the mTLS-protected
|
||||
// gRPC stream. The agent redacts SteamPassword from log output before
|
||||
// it ever hits the bus; see steamcmd.go.
|
||||
SteamUsername string
|
||||
SteamPassword string
|
||||
}
|
||||
|
||||
// Provider performs an update according to its kind.
|
||||
type Provider interface {
|
||||
Name() string
|
||||
Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error
|
||||
}
|
||||
|
||||
// Get returns the Provider implementation for the given kind, or an
|
||||
// error if the kind is unknown.
|
||||
func Get(kind string) (Provider, error) {
|
||||
switch kind {
|
||||
case "direct":
|
||||
return &DirectProvider{}, nil
|
||||
case "github":
|
||||
return &GitHubProvider{}, nil
|
||||
case "steamcmd":
|
||||
return &SteamCMDProvider{}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown update provider kind %q", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// writeToInstance puts data at the instance-relative path. Prefers
|
||||
// container-path ops when the instance has a BrowseableRoot and a live
|
||||
// container; otherwise falls back to writing under DataPath.
|
||||
func writeToInstance(ctx context.Context, uc *Context, targetPath string, data []byte) error {
|
||||
if targetPath == "" {
|
||||
return fmt.Errorf("target_path is required")
|
||||
}
|
||||
if uc.BrowseableRoot != "" && uc.ContainerID != "" {
|
||||
abs := path.Join(uc.BrowseableRoot, targetPath)
|
||||
// Ensure parent dir exists inside the container via runtime exec.
|
||||
// We don't have a helper in the interface; we just try to write
|
||||
// and rely on CopyFileToContainer returning a clear error if the
|
||||
// parent is missing. (itzg and most images pre-create /data.)
|
||||
return uc.Runtime.CopyFileToContainer(ctx, uc.ContainerID, abs, data)
|
||||
}
|
||||
if uc.DataPath == "" {
|
||||
return fmt.Errorf("no storage available: container has no BrowseableRoot and instance has no DataPath")
|
||||
}
|
||||
abs := filepath.Join(uc.DataPath, filepath.FromSlash(targetPath))
|
||||
if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir %s: %w", filepath.Dir(abs), err)
|
||||
}
|
||||
if err := os.WriteFile(abs, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write %s: %w", abs, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
agentmodule "github.com/dbledeez/panel/agent/internal/module"
|
||||
"github.com/dbledeez/panel/agent/internal/runtime"
|
||||
modulepkg "github.com/dbledeez/panel/pkg/module"
|
||||
)
|
||||
|
||||
// SteamCMDProvider runs `steamcmd +force_install_dir <path> +login
|
||||
// anonymous +app_update <appid> [-beta <branch>] validate +quit` in a
|
||||
// one-shot sidecar container that mounts the target instance's volumes.
|
||||
// The sidecar writes game files into the same Docker volumes the main
|
||||
// container reads from, so "update then start" just works.
|
||||
//
|
||||
// Preconditions:
|
||||
// - Target instance's container must NOT be running (Steam holds file
|
||||
// locks that conflict with a live server). Error clearly otherwise.
|
||||
// - Manifest must declare at least one volume; install_path resolves
|
||||
// to spec.InstallPath → BrowseableRoot → first volume's container path.
|
||||
type SteamCMDProvider struct{}
|
||||
|
||||
// Name identifies this provider kind in logs / manifest.
|
||||
func (SteamCMDProvider) Name() string { return "steamcmd" }
|
||||
|
||||
// sidecarImage is the official Valve-maintained SteamCMD image. Its
|
||||
// ENTRYPOINT is `steamcmd`, so we pass steamcmd's `+...` flags as Cmd.
|
||||
const sidecarImage = "steamcmd/steamcmd:latest"
|
||||
|
||||
// redactSteamPassword replaces a password in a cmdline arg slice with
|
||||
// "[REDACTED]" so the Console tab never echoes it. Used before logging
|
||||
// steamcmd argv.
|
||||
func redactSteamPassword(args []string, password string) []string {
|
||||
if password == "" {
|
||||
return args
|
||||
}
|
||||
out := make([]string, len(args))
|
||||
for i, a := range args {
|
||||
if a == password {
|
||||
out[i] = "[REDACTED]"
|
||||
continue
|
||||
}
|
||||
out[i] = a
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// redactPasswordLine strips the password substring from a log line if it
|
||||
// ever appears (e.g. SteamCMD echoing `+login user PASS` back). Returns
|
||||
// the original string when password is empty so anonymous/public flows
|
||||
// are zero-cost.
|
||||
func redactPasswordLine(line, password string) string {
|
||||
if password == "" || len(password) < 4 {
|
||||
return line
|
||||
}
|
||||
if !strings.Contains(line, password) {
|
||||
return line
|
||||
}
|
||||
return strings.ReplaceAll(line, password, "[REDACTED]")
|
||||
}
|
||||
|
||||
func (SteamCMDProvider) Update(ctx context.Context, uc *Context, spec *modulepkg.UpdateProvider) error {
|
||||
if spec.AppID == "" {
|
||||
return fmt.Errorf("steamcmd provider: app_id is required")
|
||||
}
|
||||
|
||||
// Refuse if the main container is still running — Steam locks files.
|
||||
mainName := "panel-" + uc.InstanceID
|
||||
if state, err := uc.Runtime.InspectByName(ctx, mainName); err == nil && state.Status == "running" {
|
||||
return fmt.Errorf("stop the instance first — container %q is running; SteamCMD conflicts with a live server", mainName)
|
||||
}
|
||||
|
||||
installPath := spec.InstallPath
|
||||
if installPath == "" {
|
||||
installPath = uc.BrowseableRoot
|
||||
}
|
||||
if installPath == "" {
|
||||
return fmt.Errorf("steamcmd provider: install_path missing and module has no browseable_root/volumes to default from")
|
||||
}
|
||||
|
||||
// Mount the same volumes as the target instance so SteamCMD writes
|
||||
// where the main container will later read.
|
||||
volumes := agentmodule.ResolveVolumes(uc.Manifest, uc.InstanceID, uc.DataPath)
|
||||
if len(volumes) == 0 {
|
||||
return fmt.Errorf("steamcmd provider: module declares no volumes to mount")
|
||||
}
|
||||
|
||||
// Platform override MUST come before +login — SteamCMD evaluates its
|
||||
// forced-platform state at login time, not at app_update time. Used for
|
||||
// apps that ship only a Windows depot (run via Wine on Linux hosts).
|
||||
args := []string{}
|
||||
if spec.Platform != "" {
|
||||
args = append(args, "+@sSteamCmdForcePlatformType", spec.Platform)
|
||||
}
|
||||
args = append(args, "+force_install_dir", installPath)
|
||||
|
||||
// Non-anonymous Steam login for paid apps (DayZ, Arma). If the module
|
||||
// declared requires_steam_login and the controller forwarded creds,
|
||||
// use them. Otherwise fall back to anonymous (most free-to-download
|
||||
// dedicated servers work anonymously).
|
||||
//
|
||||
// CRITICAL for UX: pass ONLY the username when possible. SteamCMD's
|
||||
// `+login <user>` uses the cached refresh token from config.vdf
|
||||
// (persisted in panel-steamcmd-auth) — no Steam Guard push, no
|
||||
// password round-trip. Passing `+login <user> <password>` forces a
|
||||
// fresh password auth every time, which triggers a 2FA push on every
|
||||
// call even when a perfectly valid cached token is already on disk.
|
||||
//
|
||||
// First-ever login ceremony happens in the controller (steamauth.go)
|
||||
// with the full password+guard flow to prime the cache. After that,
|
||||
// this path rides the cache. If the cache is stale/expired, SteamCMD
|
||||
// exits non-zero — the retry loop below re-runs which will hit the
|
||||
// same error; the operator then clicks Update, which the controller
|
||||
// can short-circuit to re-run the login modal (the `steam_login_required`
|
||||
// response path).
|
||||
needsLogin := spec.RequiresSteamLogin && uc.SteamUsername != ""
|
||||
if needsLogin {
|
||||
args = append(args, "+login", uc.SteamUsername)
|
||||
} else {
|
||||
args = append(args, "+login", "anonymous")
|
||||
}
|
||||
updateArgs := []string{"+app_update", spec.AppID}
|
||||
if spec.Beta != "" {
|
||||
updateArgs = append(updateArgs, "-beta", spec.Beta)
|
||||
}
|
||||
if !spec.SkipValidate {
|
||||
updateArgs = append(updateArgs, "validate")
|
||||
}
|
||||
args = append(args, updateArgs...)
|
||||
args = append(args, "+quit")
|
||||
|
||||
uc.Log(fmt.Sprintf("steamcmd: image=%s install_path=%s app_id=%s beta=%s", sidecarImage, installPath, spec.AppID, spec.Beta))
|
||||
uc.Log(fmt.Sprintf("steamcmd argv: %v", redactSteamPassword(args, uc.SteamPassword)))
|
||||
|
||||
// SteamCMD is notoriously flaky on first invocation — exit 8 ("missing
|
||||
// configuration"), exit 2 (generic "retry later"), and Steam auth races
|
||||
// are all recoverable by simply running the same command again. AMP,
|
||||
// LinuxGSM, and Valve's own Steam client all retry on these codes. We
|
||||
// mirror that behaviour so operators don't have to click Update twice.
|
||||
const maxAttempts = 4
|
||||
var lastExit int
|
||||
var ranSuccessfully bool
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
exitCode, err := runSteamCMDOnce(ctx, uc, sidecarImage, args, volumes, attempt)
|
||||
lastExit = exitCode
|
||||
if err == nil && exitCode == 0 {
|
||||
uc.Log(fmt.Sprintf("steamcmd: completed successfully (app_id=%s, attempts=%d)", spec.AppID, attempt))
|
||||
ranSuccessfully = true
|
||||
break
|
||||
}
|
||||
// Transient exit codes SteamCMD is known to recover from. Anything
|
||||
// else we surface immediately (missing disk space, bad app id, etc.).
|
||||
transient := err == nil && (exitCode == 2 || exitCode == 5 || exitCode == 8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transient || attempt == maxAttempts {
|
||||
break
|
||||
}
|
||||
backoff := time.Duration(attempt*attempt*3) * time.Second // 3s, 12s, 27s
|
||||
uc.Log(fmt.Sprintf("steamcmd: exit %d — transient, retrying in %s (attempt %d/%d)", exitCode, backoff, attempt+1, maxAttempts))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
|
||||
// Finalize: SteamCMD with `+force_install_dir <p>` stages downloads
|
||||
// at <p>/steamapps/downloading/<appid>/ and atomically renames into
|
||||
// <p>/ on success. When SteamCMD exits with state 0x602 (often after
|
||||
// concurrent runs interrupt finalization, or a `validate` pass races
|
||||
// the move) the bytes are on disk but never get renamed — operator
|
||||
// sees an empty install dir and an exit-8 error from the panel.
|
||||
//
|
||||
// This step runs unconditionally after the retry loop. If the staging
|
||||
// dir is empty or absent (the happy path), it's a cheap no-op. If it
|
||||
// has content, we cp -a it into place and the install completes.
|
||||
finalized, ferr := finalizeStagingDir(ctx, uc, installPath, spec.AppID, volumes)
|
||||
if ferr != nil {
|
||||
uc.Log(fmt.Sprintf("steamcmd: finalize check failed: %s (continuing anyway)", ferr.Error()))
|
||||
} else if finalized {
|
||||
uc.Log("steamcmd: recovered staged install (state 0x602 workaround)")
|
||||
return nil
|
||||
}
|
||||
|
||||
if ranSuccessfully {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("steamcmd exited %d after %d attempt(s)", lastExit, maxAttempts)
|
||||
}
|
||||
|
||||
// finalizeStagingDir checks for a stranded SteamCMD staging directory
|
||||
// at <installPath>/steamapps/downloading/<appid>/ and, if present and
|
||||
// non-empty, moves its contents into <installPath>/. Runs as a one-shot
|
||||
// alpine sidecar with the same volume mounts as the SteamCMD run so
|
||||
// it has access to the install path. Returns true if a recovery move
|
||||
// actually happened, false if there was nothing to do.
|
||||
//
|
||||
// The script is intentionally defensive:
|
||||
// - mv is preferred over cp+rm because it's atomic on the same FS.
|
||||
// - If both source and destination have entries with the same name,
|
||||
// mv -f overwrites (rare; happens when steamcmd partially renamed
|
||||
// before dying).
|
||||
// - We exit 0 on "no staging dir" so this is a no-op for the happy
|
||||
// path.
|
||||
func finalizeStagingDir(ctx context.Context, uc *Context, installPath, appID string, volumes []runtime.VolumeSpec) (bool, error) {
|
||||
const stagingMarker = "/__staging__"
|
||||
script := fmt.Sprintf(`set -e
|
||||
DL=%q/steamapps/downloading/%s
|
||||
DEST=%q
|
||||
if [ ! -d "$DL" ] || [ -z "$(ls -A "$DL" 2>/dev/null)" ]; then
|
||||
echo "no-staging"
|
||||
exit 0
|
||||
fi
|
||||
echo "finalizing $DL -> $DEST"
|
||||
mkdir -p "$DEST"
|
||||
# mv every top-level entry; -f overwrites partial renames.
|
||||
for f in "$DL"/* "$DL"/.[!.]* "$DL"/..?*; do
|
||||
[ -e "$f" ] || continue
|
||||
mv -f "$f" "$DEST/" 2>&1 || cp -a "$f" "$DEST/" && rm -rf "$f"
|
||||
done
|
||||
rm -rf "$DL"
|
||||
touch %q
|
||||
echo "finalized"
|
||||
`, installPath, appID, installPath, installPath+stagingMarker)
|
||||
|
||||
sidecarSpec := runtime.InstanceSpec{
|
||||
InstanceID: fmt.Sprintf("%s-steamcmd-finalize", uc.InstanceID),
|
||||
IsSidecar: true,
|
||||
Image: "alpine:latest",
|
||||
Command: []string{"sh", "-c", script},
|
||||
Volumes: volumes,
|
||||
RestartPolicy: "no",
|
||||
}
|
||||
contID, err := uc.Runtime.Create(ctx, sidecarSpec)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("create finalize sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = uc.Runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := uc.Runtime.Start(ctx, contID); err != nil {
|
||||
return false, fmt.Errorf("start finalize sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
var lastLine string
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = uc.Runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
lastLine = strings.TrimSpace(line)
|
||||
uc.Log("finalize: " + lastLine)
|
||||
})
|
||||
}()
|
||||
exitCode, err := uc.Runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("wait finalize sidecar: %w", err)
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return false, fmt.Errorf("finalize sidecar exit %d", exitCode)
|
||||
}
|
||||
return lastLine == "finalized", nil
|
||||
}
|
||||
|
||||
// runSteamCMDOnce is a single run of the steamcmd sidecar; called in a loop
|
||||
// by the retry wrapper. Returns the container's exit code on clean
|
||||
// lifecycle (even non-zero) or an error on infrastructure failure.
|
||||
func runSteamCMDOnce(ctx context.Context, uc *Context, image string, args []string, volumes []runtime.VolumeSpec, attempt int) (int, error) {
|
||||
sidecarID := fmt.Sprintf("%s-steamcmd-%d", uc.InstanceID, attempt)
|
||||
// Mount the panel-wide SteamCMD auth volume at /root/.local/share/Steam
|
||||
// (the real data dir of the steamcmd:latest image — NOT /root/Steam).
|
||||
// This persists config/config.vdf (login refresh token) + ssfn sentry,
|
||||
// which lets subsequent `+login <user>` runs skip the 2FA push. Anon
|
||||
// logins ignore this dir — no harm.
|
||||
volumesWithAuth := append([]runtime.VolumeSpec(nil), volumes...)
|
||||
volumesWithAuth = append(volumesWithAuth, runtime.VolumeSpec{
|
||||
VolumeName: "panel-steamcmd-auth",
|
||||
ContainerPath: "/root/.local/share/Steam",
|
||||
Type: "volume",
|
||||
})
|
||||
sidecarSpec := runtime.InstanceSpec{
|
||||
InstanceID: sidecarID,
|
||||
IsSidecar: true,
|
||||
Image: image,
|
||||
Command: args,
|
||||
Volumes: volumesWithAuth,
|
||||
RestartPolicy: "no",
|
||||
// Docker 29.4+ default seccomp ENOSYS's a syscall the 32-bit
|
||||
// Steam runtime needs ("CreateBoundSocket error 38"). Older
|
||||
// Docker (29.1.x on princess/ivy) doesn't filter it. Unconfine
|
||||
// for the sidecar so install/validate works across hosts.
|
||||
SecurityOpts: []string{"seccomp=unconfined"},
|
||||
}
|
||||
contID, err := uc.Runtime.Create(ctx, sidecarSpec)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("create sidecar: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
rmCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
_ = uc.Runtime.Remove(rmCtx, contID)
|
||||
}()
|
||||
if err := uc.Runtime.Start(ctx, contID); err != nil {
|
||||
return 0, fmt.Errorf("start sidecar: %w", err)
|
||||
}
|
||||
logCtx, cancelLogs := context.WithCancel(ctx)
|
||||
defer cancelLogs()
|
||||
logDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(logDone)
|
||||
_ = uc.Runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
|
||||
uc.Log(redactPasswordLine(line, uc.SteamPassword))
|
||||
})
|
||||
}()
|
||||
exitCode, err := uc.Runtime.Wait(ctx, contID)
|
||||
cancelLogs()
|
||||
<-logDone
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("wait sidecar: %w", err)
|
||||
}
|
||||
return exitCode, nil
|
||||
}
|
||||
Reference in New Issue
Block a user