panel public release

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-15 01:26:41 -07:00
commit 295eb22826
2165 changed files with 301492 additions and 0 deletions
@@ -0,0 +1,505 @@
+package dispatch
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	agentmodule "github.com/dbledeez/panel/agent/internal/module"
+	"github.com/dbledeez/panel/agent/internal/runtime"
+	modulepkg "github.com/dbledeez/panel/pkg/module"
+	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
+)
+
+// buildBackupTarCmd assembles the tar command run inside the sidecar.
+// Manifest BackupSpec wins when Include is set: the tar runs against
+// the listed paths (each relative to BrowseableRoot, missing ones
+// silently skipped). Otherwise it falls back to a whole-tree archive
+// of BrowseableRoot. Excludes are passed through tar's --exclude.
+//
+// Resulting tarball preserves the directory layout under BrowseableRoot
+// so restore can extract straight into the same root.
+func buildBackupTarCmd(manifest *modulepkgManifest, root, fileName string, keep int) string {
+	if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
+		root = manifest.Backup.Root
+	}
+	var base string
+	if manifest == nil || manifest.Backup == nil || len(manifest.Backup.Include) == 0 {
+		// Fallback: whole tree.
+		base = fmt.Sprintf("cd %q && tar czf /backup/%s . && stat -c %%s /backup/%s",
+			root, shellEscape(fileName), shellEscape(fileName))
+	} else {
+		// Per-game include list. Missing paths are non-fatal — players will
+		// create new instances on a fresh server before the save dir exists,
+		// and we don't want their first Back-up-now to fail the run.
+		excludes := ""
+		for _, e := range manifest.Backup.Exclude {
+			excludes += " --exclude=" + shellEscape(e)
+		}
+		// Build "test -e <path> && echo <path>" for each include — only
+		// existing paths get added to the tar argv, so the operator doesn't
+		// see a confusing "Cannot stat: No such file" on first run.
+		//
+		// Newline-delimited (NOT NUL-delimited) because the sidecar image is
+		// alpine, which ships busybox tar — and busybox tar doesn't grok
+		// `--null`. Manifest paths never contain whitespace or newlines (we
+		// hand-curate them per game), so a plain newline list is safe.
+		var existsChecks []string
+		for _, p := range manifest.Backup.Include {
+			clean := strings.TrimPrefix(p, "./")
+			clean = strings.TrimPrefix(clean, "/")
+			existsChecks = append(existsChecks, fmt.Sprintf("[ -e %s ] && printf '%%s\\n' %s", shellEscape(clean), shellEscape(clean)))
+		}
+		listExpr := "{ " + strings.Join(existsChecks, " ; ") + " ; }"
+		base = fmt.Sprintf("cd %q && %s | tar czf /backup/%s%s -T - && stat -c %%s /backup/%s",
+			root, listExpr, shellEscape(fileName), excludes, shellEscape(fileName))
+	}
+	return base + backupPruneSuffix(keep)
+}
+
+// backupPruneSuffix appends a best-effort retention prune that keeps only the
+// newest `keep` *.tar.gz in /backup (by mtime) and deletes the rest. It runs in
+// the SAME root sidecar that just wrote the new tarball — the backup files are
+// root-owned, so the unprivileged agent can't prune them itself. keep<=0 leaves
+// pruning off. busybox-safe (alpine): ls -1t + tail -n +N + while-read — no
+// `head -n -N` (GNU-only) and no `xargs -r`.
+func backupPruneSuffix(keep int) string {
+	if keep <= 0 {
+		return ""
+	}
+	return fmt.Sprintf(" ; echo 'prune: keeping newest %d backups' ; ls -1t /backup/*.tar.gz 2>/dev/null | tail -n +%d | while IFS= read -r f; do echo \"prune: rm $f\" ; rm -f \"$f\" ; done",
+		keep, keep+1)
+}
+
+// readBackupKeep returns the backup retention for this instance: the integer in
+// <dataPath>/backup-keep when present, else the PANEL_BACKUP_KEEP env default,
+// else 10. A rolling cap stops the unbounded pileup that filled 90 GB on a busy
+// 7DTD season. Set per-instance by writing the file (the Backups UI exposes it).
+func readBackupKeep(dataPath string) int {
+	def := 10
+	if v := os.Getenv("PANEL_BACKUP_KEEP"); v != "" {
+		if n, e := strconv.Atoi(strings.TrimSpace(v)); e == nil && n > 0 {
+			def = n
+		}
+	}
+	b, err := os.ReadFile(filepath.Join(dataPath, "backup-keep"))
+	if err != nil {
+		return def
+	}
+	if n, e := strconv.Atoi(strings.TrimSpace(string(b))); e == nil && n > 0 {
+		return n
+	}
+	return def
+}
+
+// modulepkgManifest is a local alias so the helper signature below
+// doesn't drag modulepkg into its return type at the file top.
+type modulepkgManifest = modulepkg.Manifest
+
+// Backup / restore via sidecar containers. Two operations:
+//
+//   - Backup:  tar -czf /backup/<file>.tar.gz -C /src .   (volume readonly at /src)
+//   - Restore: tar -xzf /backup/<file>.tar.gz -C /dst      (volume read-write at /dst)
+//
+// The sidecar uses alpine:3.21 + busybox tar. Backup files live on the
+// agent host under $BACKUP_DIR/<instance_id>/<ts>-<bkpID>.tar.gz; both
+// sidecars bind-mount $BACKUP_DIR/<instance_id> at /backup.
+//
+// For restore we refuse if the main container is running: extracting
+// files under a live server would corrupt world state. Operator stops
+// first. (Symmetrical to the SteamCMD updater's running-container check.)
+
+const (
+	backupImage      = "alpine:3.21"
+	backupSidecarTTL = 30 * time.Minute
+)
+
+// ---- Backup ----
+
+func (d *Dispatcher) handleBackup(ctx context.Context, corrID string, req *panelv1.BackupRequest) {
+	log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
+
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendBackupResult(corrID, "", "", 0, err.Error())
+		return
+	}
+	if rec.BrowseableRoot == "" {
+		d.sendBackupResult(corrID, "", "", 0, "instance has no BrowseableRoot to tar (module declares no volumes)")
+		return
+	}
+	if d.backupDir == "" {
+		d.sendBackupResult(corrID, "", "", 0, "agent has no --backup-dir configured")
+		return
+	}
+
+	// Prepare the agent-side target directory.
+	bkpID := "bkp_" + randHex(8)
+	hostDir := filepath.Join(d.backupDir, req.InstanceId)
+	if err := os.MkdirAll(hostDir, 0o755); err != nil {
+		d.sendBackupResult(corrID, "", "", 0, fmt.Sprintf("mkdir %s: %s", hostDir, err.Error()))
+		return
+	}
+	ts := time.Now().UTC().Format("20060102-150405")
+	fileName := fmt.Sprintf("%s-%s.tar.gz", ts, bkpID)
+	hostFile := filepath.Join(hostDir, fileName)
+
+	log.Info("backup starting", "backup_id", bkpID, "host_file", hostFile)
+
+	// Build sidecar spec that mounts the same volumes as the instance,
+	// plus the host backup directory at /backup.
+	volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
+	// Mark volumes read-only for backup safety.
+	for i := range volumes {
+		volumes[i].ReadOnly = true
+	}
+	// Add the host backup dir as a (read-write) bind.
+	volumes = append(volumes, runtime.VolumeSpec{
+		Type:          "bind",
+		HostPath:      hostDir,
+		ContainerPath: "/backup",
+	})
+
+	// Per-game backup paths. If the manifest declares Backup.Include,
+	// only those paths get archived (saves-only — keeps ARK SA tarballs
+	// at ~250 MB instead of the 18 GB whole-install). Without a spec,
+	// fall back to the original "tar everything under BrowseableRoot"
+	// behavior so older modules don't regress.
+	manifest := d.modulesMustGet(rec.ModuleID)
+	keep := readBackupKeep(rec.DataPath)
+	tarCmd := buildBackupTarCmd(manifest, rec.BrowseableRoot, fileName, keep)
+	if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
+		log.Info("backup using manifest-declared paths", "include_count", len(manifest.Backup.Include))
+	} else {
+		log.Warn("backup falling back to whole BrowseableRoot — module manifest has no `backup.include`",
+			"module_id", rec.ModuleID, "root", rec.BrowseableRoot)
+	}
+	spec := runtime.InstanceSpec{
+		InstanceID:    req.InstanceId + "-backup",
+		IsSidecar:     true,
+		Image:         backupImage,
+		Entrypoint:    []string{"sh"},
+		Command:       []string{"-c", tarCmd},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		d.sendBackupResult(corrID, bkpID, "", 0, "create sidecar: "+err.Error())
+		return
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		d.sendBackupResult(corrID, bkpID, "", 0, "start sidecar: "+err.Error())
+		return
+	}
+	// Stream log lines for visibility.
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: rec.InstanceID, Stream: "backup",
+					At: timestamppb.Now(), Line: line,
+				}},
+			})
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		d.sendBackupResult(corrID, bkpID, "", 0, "wait: "+err.Error())
+		return
+	}
+	if exitCode != 0 {
+		d.sendBackupResult(corrID, bkpID, "", 0, fmt.Sprintf("sidecar exited %d", exitCode))
+		return
+	}
+
+	info, err := os.Stat(hostFile)
+	if err != nil {
+		d.sendBackupResult(corrID, bkpID, hostFile, 0, "stat result: "+err.Error())
+		return
+	}
+	log.Info("backup complete", "backup_id", bkpID, "size_bytes", info.Size())
+	d.sendBackupResult(corrID, bkpID, hostFile, info.Size(), "")
+}
+
+func (d *Dispatcher) sendBackupResult(corrID, bkpID, path string, size int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_BackupResult{
+			BackupResult: &panelv1.BackupResult{
+				BackupId:  bkpID,
+				Path:      path,
+				SizeBytes: size,
+				Error:     errMsg,
+			},
+		},
+	})
+}
+
+// ---- Wipe paths (used by the env-config recreate flow) ----
+//
+// Spawns an alpine sidecar that mounts the same volumes the instance
+// uses, runs `rm -rf` on each path (relative to BrowseableRoot), then
+// exits. Designed for the ARK map-switch flow — operator picked a new
+// SERVER_MAP and checked "wipe existing world data" so the old map's
+// SavedArks/<MapName>/ goes away. Generic enough that any future
+// env-config field can declare its own wipe paths.
+func (d *Dispatcher) runWipeSidecar(ctx context.Context, manifest *modulepkg.Manifest, instanceID, dataPath string, paths []string) error {
+	if manifest == nil {
+		return errors.New("nil manifest")
+	}
+	root := resolveBrowseableRoot(manifest)
+	if root == "" {
+		return errors.New("module declares no browseable_root — wipe needs a base path")
+	}
+	volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
+	// Build the rm -rf command. Each path is rooted under
+	// BrowseableRoot. Missing paths are silently skipped (first run).
+	var rmParts []string
+	for _, p := range paths {
+		clean := p
+		// Defensive: reject absolute paths and parent traversal so a
+		// rogue config_values value can't `rm -rf /` the container.
+		if strings.HasPrefix(clean, "/") || strings.Contains(clean, "..") {
+			return fmt.Errorf("refusing wipe of suspicious path %q", p)
+		}
+		full := root + "/" + clean
+		rmParts = append(rmParts, fmt.Sprintf("rm -rf %s", shellEscape(full)))
+	}
+	cmd := strings.Join(rmParts, " && ") + " && echo wiped"
+
+	spec := runtime.InstanceSpec{
+		InstanceID:    instanceID + "-wipe",
+		IsSidecar:     true,
+		Image:         backupImage,
+		Entrypoint:    []string{"sh"},
+		Command:       []string{"-c", cmd},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		return fmt.Errorf("create wipe sidecar: %w", err)
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		return fmt.Errorf("start wipe sidecar: %w", err)
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: instanceID, Stream: "wipe",
+					At: timestamppb.Now(), Line: line,
+				}},
+			})
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		return fmt.Errorf("wait wipe sidecar: %w", err)
+	}
+	if exitCode != 0 {
+		return fmt.Errorf("wipe sidecar exited %d", exitCode)
+	}
+	return nil
+}
+
+// ---- Restore ----
+
+func (d *Dispatcher) handleRestore(ctx context.Context, corrID string, req *panelv1.RestoreRequest) {
+	log := d.log.With("instance_id", req.InstanceId, "backup_id", req.BackupId, "correlation_id", corrID)
+
+	rec, err := d.lookupRecord(req.InstanceId)
+	if err != nil {
+		d.sendRestoreResult(corrID, 0, err.Error())
+		return
+	}
+	if rec.BrowseableRoot == "" {
+		d.sendRestoreResult(corrID, 0, "instance has no BrowseableRoot to restore into")
+		return
+	}
+	if req.BackupPath == "" {
+		d.sendRestoreResult(corrID, 0, "backup_path required")
+		return
+	}
+	// Refuse if container is running.
+	if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil && state.Status == "running" {
+		d.sendRestoreResult(corrID, 0, "stop the instance first — main container is running")
+		return
+	}
+	info, err := os.Stat(req.BackupPath)
+	if err != nil || info.IsDir() {
+		d.sendRestoreResult(corrID, 0, "backup file missing or is a directory")
+		return
+	}
+
+	volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
+	// Add the backup file's parent as a read-only bind at /backup.
+	hostDir := filepath.Dir(req.BackupPath)
+	fileName := filepath.Base(req.BackupPath)
+	volumes = append(volumes, runtime.VolumeSpec{
+		Type:          "bind",
+		HostPath:      hostDir,
+		ContainerPath: "/backup",
+		ReadOnly:      true,
+	})
+
+	// Extract strategy depends on whether the backup is per-game-paths
+	// or a whole-tree archive. When the manifest declares Backup.Include,
+	// only THOSE paths are wiped before extraction — preserves the
+	// 18 GB game install while still giving a clean save dir. Without a
+	// spec, fall back to wiping all of BrowseableRoot (legacy whole-tree
+	// behavior).
+	manifest := d.modulesMustGet(rec.ModuleID)
+	// Resolve the SAME root the backup wrote against. buildBackupTarCmd
+	// archives relative to manifest.Backup.Root when set (e.g. 7DTD's
+	// /game-saves, where world data actually lives — distinct from the
+	// /game install volume that is BrowseableRoot). Restore MUST extract
+	// into that same root or the tarball lands in the wrong volume and the
+	// live server never sees it (silent data loss: restore "succeeds" but
+	// players' bases are gone). Mirror the backup's resolution exactly.
+	restoreRoot := rec.BrowseableRoot
+	if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
+		restoreRoot = manifest.Backup.Root
+	}
+	var wipeCmd string
+	if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
+		// Wipe only the manifest-declared include paths so we don't
+		// nuke unrelated install files. Each path is rm -rf'd; missing
+		// ones are skipped silently (first-run case).
+		var parts []string
+		for _, p := range manifest.Backup.Include {
+			clean := strings.TrimPrefix(strings.TrimPrefix(p, "./"), "/")
+			parts = append(parts, fmt.Sprintf("rm -rf %s/%s", shellEscape(restoreRoot), shellEscape(clean)))
+		}
+		wipeCmd = strings.Join(parts, " ; ") + " 2>/dev/null"
+		log.Info("restore: wiping manifest-declared paths only", "root", restoreRoot, "include_count", len(manifest.Backup.Include))
+	} else {
+		wipeCmd = fmt.Sprintf("rm -rf %q/* %q/.[!.]* 2>/dev/null", restoreRoot, restoreRoot)
+		log.Warn("restore: wiping entire root (whole-tree backup)", "root", restoreRoot)
+	}
+	extractCmd := fmt.Sprintf(
+		"%s ; tar xzf /backup/%s -C %q && du -sb %q | cut -f1",
+		wipeCmd,
+		shellEscape(fileName),
+		restoreRoot,
+		restoreRoot,
+	)
+	spec := runtime.InstanceSpec{
+		InstanceID:    req.InstanceId + "-restore",
+		IsSidecar:     true,
+		Image:         backupImage,
+		Entrypoint:    []string{"sh"},
+		Command:       []string{"-c", extractCmd},
+		Volumes:       volumes,
+		RestartPolicy: "no",
+	}
+	contID, err := d.runtime.Create(ctx, spec)
+	if err != nil {
+		d.sendRestoreResult(corrID, 0, "create sidecar: "+err.Error())
+		return
+	}
+	defer func() {
+		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = d.runtime.Remove(rmCtx, contID)
+	}()
+	if err := d.runtime.Start(ctx, contID); err != nil {
+		d.sendRestoreResult(corrID, 0, "start sidecar: "+err.Error())
+		return
+	}
+	logCtx, cancelLogs := context.WithCancel(ctx)
+	logDone := make(chan struct{})
+	go func() {
+		defer close(logDone)
+		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
+			d.sendEnv(&panelv1.AgentEnvelope{
+				SentAt: timestamppb.Now(),
+				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
+					InstanceId: rec.InstanceID, Stream: "restore",
+					At: timestamppb.Now(), Line: line,
+				}},
+			})
+		})
+	}()
+	exitCode, err := d.runtime.Wait(ctx, contID)
+	cancelLogs()
+	<-logDone
+	if err != nil {
+		d.sendRestoreResult(corrID, 0, "wait: "+err.Error())
+		return
+	}
+	if exitCode != 0 {
+		d.sendRestoreResult(corrID, 0, fmt.Sprintf("sidecar exited %d", exitCode))
+		return
+	}
+	log.Info("restore complete")
+	d.sendRestoreResult(corrID, info.Size(), "")
+}
+
+func (d *Dispatcher) sendRestoreResult(corrID string, bytesRestored int64, errMsg string) {
+	d.sendEnv(&panelv1.AgentEnvelope{
+		CorrelationId: corrID,
+		SentAt:        timestamppb.Now(),
+		Payload: &panelv1.AgentEnvelope_RestoreResult{
+			RestoreResult: &panelv1.RestoreResult{BytesRestored: bytesRestored, Error: errMsg},
+		},
+	})
+}
+
+// ---- helpers ----
+
+func randHex(n int) string {
+	b := make([]byte, n)
+	_, _ = rand.Read(b)
+	return hex.EncodeToString(b)
+}
+
+// shellEscape wraps a filename in single quotes for safe inclusion in an
+// sh -c command. Assumes filename never contains a single-quote (ours
+// never do — timestamps + bkp_<hex>).
+func shellEscape(s string) string { return "'" + s + "'" }
+
+// modulesMustGet returns the dispatcher's manifest for the given id, or
+// nil if it's gone missing (shouldn't happen for a live instance).
+func (d *Dispatcher) modulesMustGet(id string) *modulepkgManifest {
+	m, _ := d.modules.Get(id)
+	return m
+}
+
+// Keep errors imported for future error-wrapping.
+var _ = errors.New