panel/agent/internal/dispatch/backup.go

package dispatch

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"google.golang.org/protobuf/types/known/timestamppb"

	agentmodule "github.com/dbledeez/panel/agent/internal/module"
	"github.com/dbledeez/panel/agent/internal/runtime"
	modulepkg "github.com/dbledeez/panel/pkg/module"
	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
)

// buildBackupTarCmd assembles the tar command run inside the sidecar.
// Manifest BackupSpec wins when Include is set: the tar runs against
// the listed paths (each relative to BrowseableRoot, missing ones
// silently skipped). Otherwise it falls back to a whole-tree archive
// of BrowseableRoot. Excludes are passed through tar's --exclude.
//
// Resulting tarball preserves the directory layout under BrowseableRoot
// so restore can extract straight into the same root.
func buildBackupTarCmd(manifest *modulepkgManifest, root, fileName string, keep int) string {
	if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
		root = manifest.Backup.Root
	}
	var base string
	if manifest == nil || manifest.Backup == nil || len(manifest.Backup.Include) == 0 {
		// Fallback: whole tree.
		base = fmt.Sprintf("cd %q && tar czf /backup/%s . && stat -c %%s /backup/%s",
			root, shellEscape(fileName), shellEscape(fileName))
	} else {
		// Per-game include list. Missing paths are non-fatal — players will
		// create new instances on a fresh server before the save dir exists,
		// and we don't want their first Back-up-now to fail the run.
		excludes := ""
		for _, e := range manifest.Backup.Exclude {
			excludes += " --exclude=" + shellEscape(e)
		}
		// Build "test -e <path> && echo <path>" for each include — only
		// existing paths get added to the tar argv, so the operator doesn't
		// see a confusing "Cannot stat: No such file" on first run.
		//
		// Newline-delimited (NOT NUL-delimited) because the sidecar image is
		// alpine, which ships busybox tar — and busybox tar doesn't grok
		// `--null`. Manifest paths never contain whitespace or newlines (we
		// hand-curate them per game), so a plain newline list is safe.
		var existsChecks []string
		for _, p := range manifest.Backup.Include {
			clean := strings.TrimPrefix(p, "./")
			clean = strings.TrimPrefix(clean, "/")
			existsChecks = append(existsChecks, fmt.Sprintf("[ -e %s ] && printf '%%s\\n' %s", shellEscape(clean), shellEscape(clean)))
		}
		listExpr := "{ " + strings.Join(existsChecks, " ; ") + " ; }"
		base = fmt.Sprintf("cd %q && %s | tar czf /backup/%s%s -T - && stat -c %%s /backup/%s",
			root, listExpr, shellEscape(fileName), excludes, shellEscape(fileName))
	}
	return base + backupPruneSuffix(keep)
}

// backupPruneSuffix appends a best-effort retention prune that keeps only the
// newest `keep` *.tar.gz in /backup (by mtime) and deletes the rest. It runs in
// the SAME root sidecar that just wrote the new tarball — the backup files are
// root-owned, so the unprivileged agent can't prune them itself. keep<=0 leaves
// pruning off. busybox-safe (alpine): ls -1t + tail -n +N + while-read — no
// `head -n -N` (GNU-only) and no `xargs -r`.
func backupPruneSuffix(keep int) string {
	if keep <= 0 {
		return ""
	}
	return fmt.Sprintf(" ; echo 'prune: keeping newest %d backups' ; ls -1t /backup/*.tar.gz 2>/dev/null | tail -n +%d | while IFS= read -r f; do echo \"prune: rm $f\" ; rm -f \"$f\" ; done",
		keep, keep+1)
}

// readBackupKeep returns the backup retention for this instance: the integer in
// <dataPath>/backup-keep when present, else the PANEL_BACKUP_KEEP env default,
// else 10. A rolling cap stops the unbounded pileup that filled 90 GB on a busy
// 7DTD season. Set per-instance by writing the file (the Backups UI exposes it).
func readBackupKeep(dataPath string) int {
	def := 10
	if v := os.Getenv("PANEL_BACKUP_KEEP"); v != "" {
		if n, e := strconv.Atoi(strings.TrimSpace(v)); e == nil && n > 0 {
			def = n
		}
	}
	b, err := os.ReadFile(filepath.Join(dataPath, "backup-keep"))
	if err != nil {
		return def
	}
	if n, e := strconv.Atoi(strings.TrimSpace(string(b))); e == nil && n > 0 {
		return n
	}
	return def
}

// modulepkgManifest is a local alias so the helper signature below
// doesn't drag modulepkg into its return type at the file top.
type modulepkgManifest = modulepkg.Manifest

// Backup / restore via sidecar containers. Two operations:
//
//   - Backup:  tar -czf /backup/<file>.tar.gz -C /src .   (volume readonly at /src)
//   - Restore: tar -xzf /backup/<file>.tar.gz -C /dst      (volume read-write at /dst)
//
// The sidecar uses alpine:3.21 + busybox tar. Backup files live on the
// agent host under $BACKUP_DIR/<instance_id>/<ts>-<bkpID>.tar.gz; both
// sidecars bind-mount $BACKUP_DIR/<instance_id> at /backup.
//
// For restore we refuse if the main container is running: extracting
// files under a live server would corrupt world state. Operator stops
// first. (Symmetrical to the SteamCMD updater's running-container check.)

const (
	backupImage      = "alpine:3.21"
	backupSidecarTTL = 30 * time.Minute
)

// ---- Backup ----

func (d *Dispatcher) handleBackup(ctx context.Context, corrID string, req *panelv1.BackupRequest) {
	log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)

	rec, err := d.lookupRecord(req.InstanceId)
	if err != nil {
		d.sendBackupResult(corrID, "", "", 0, err.Error())
		return
	}
	if rec.BrowseableRoot == "" {
		d.sendBackupResult(corrID, "", "", 0, "instance has no BrowseableRoot to tar (module declares no volumes)")
		return
	}
	if d.backupDir == "" {
		d.sendBackupResult(corrID, "", "", 0, "agent has no --backup-dir configured")
		return
	}

	// Prepare the agent-side target directory.
	bkpID := "bkp_" + randHex(8)
	hostDir := filepath.Join(d.backupDir, req.InstanceId)
	if err := os.MkdirAll(hostDir, 0o755); err != nil {
		d.sendBackupResult(corrID, "", "", 0, fmt.Sprintf("mkdir %s: %s", hostDir, err.Error()))
		return
	}
	ts := time.Now().UTC().Format("20060102-150405")
	fileName := fmt.Sprintf("%s-%s.tar.gz", ts, bkpID)
	hostFile := filepath.Join(hostDir, fileName)

	log.Info("backup starting", "backup_id", bkpID, "host_file", hostFile)

	// Build sidecar spec that mounts the same volumes as the instance,
	// plus the host backup directory at /backup.
	volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
	// Mark volumes read-only for backup safety.
	for i := range volumes {
		volumes[i].ReadOnly = true
	}
	// Add the host backup dir as a (read-write) bind.
	volumes = append(volumes, runtime.VolumeSpec{
		Type:          "bind",
		HostPath:      hostDir,
		ContainerPath: "/backup",
	})

	// Per-game backup paths. If the manifest declares Backup.Include,
	// only those paths get archived (saves-only — keeps ARK SA tarballs
	// at ~250 MB instead of the 18 GB whole-install). Without a spec,
	// fall back to the original "tar everything under BrowseableRoot"
	// behavior so older modules don't regress.
	manifest := d.modulesMustGet(rec.ModuleID)
	keep := readBackupKeep(rec.DataPath)
	tarCmd := buildBackupTarCmd(manifest, rec.BrowseableRoot, fileName, keep)
	if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
		log.Info("backup using manifest-declared paths", "include_count", len(manifest.Backup.Include))
	} else {
		log.Warn("backup falling back to whole BrowseableRoot — module manifest has no `backup.include`",
			"module_id", rec.ModuleID, "root", rec.BrowseableRoot)
	}
	spec := runtime.InstanceSpec{
		InstanceID:    req.InstanceId + "-backup",
		IsSidecar:     true,
		Image:         backupImage,
		Entrypoint:    []string{"sh"},
		Command:       []string{"-c", tarCmd},
		Volumes:       volumes,
		RestartPolicy: "no",
	}

	contID, err := d.runtime.Create(ctx, spec)
	if err != nil {
		d.sendBackupResult(corrID, bkpID, "", 0, "create sidecar: "+err.Error())
		return
	}
	defer func() {
		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		_ = d.runtime.Remove(rmCtx, contID)
	}()

	if err := d.runtime.Start(ctx, contID); err != nil {
		d.sendBackupResult(corrID, bkpID, "", 0, "start sidecar: "+err.Error())
		return
	}
	// Stream log lines for visibility.
	logCtx, cancelLogs := context.WithCancel(ctx)
	logDone := make(chan struct{})
	go func() {
		defer close(logDone)
		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
			d.sendEnv(&panelv1.AgentEnvelope{
				SentAt: timestamppb.Now(),
				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
					InstanceId: rec.InstanceID, Stream: "backup",
					At: timestamppb.Now(), Line: line,
				}},
			})
		})
	}()
	exitCode, err := d.runtime.Wait(ctx, contID)
	cancelLogs()
	<-logDone
	if err != nil {
		d.sendBackupResult(corrID, bkpID, "", 0, "wait: "+err.Error())
		return
	}
	if exitCode != 0 {
		d.sendBackupResult(corrID, bkpID, "", 0, fmt.Sprintf("sidecar exited %d", exitCode))
		return
	}

	info, err := os.Stat(hostFile)
	if err != nil {
		d.sendBackupResult(corrID, bkpID, hostFile, 0, "stat result: "+err.Error())
		return
	}
	log.Info("backup complete", "backup_id", bkpID, "size_bytes", info.Size())
	d.sendBackupResult(corrID, bkpID, hostFile, info.Size(), "")
}

func (d *Dispatcher) sendBackupResult(corrID, bkpID, path string, size int64, errMsg string) {
	d.sendEnv(&panelv1.AgentEnvelope{
		CorrelationId: corrID,
		SentAt:        timestamppb.Now(),
		Payload: &panelv1.AgentEnvelope_BackupResult{
			BackupResult: &panelv1.BackupResult{
				BackupId:  bkpID,
				Path:      path,
				SizeBytes: size,
				Error:     errMsg,
			},
		},
	})
}

// ---- Wipe paths (used by the env-config recreate flow) ----
//
// Spawns an alpine sidecar that mounts the same volumes the instance
// uses, runs `rm -rf` on each path (relative to BrowseableRoot), then
// exits. Designed for the ARK map-switch flow — operator picked a new
// SERVER_MAP and checked "wipe existing world data" so the old map's
// SavedArks/<MapName>/ goes away. Generic enough that any future
// env-config field can declare its own wipe paths.
func (d *Dispatcher) runWipeSidecar(ctx context.Context, manifest *modulepkg.Manifest, instanceID, dataPath string, paths []string) error {
	if manifest == nil {
		return errors.New("nil manifest")
	}
	root := resolveBrowseableRoot(manifest)
	if root == "" {
		return errors.New("module declares no browseable_root — wipe needs a base path")
	}
	volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath)
	// Build the rm -rf command. Each path is rooted under
	// BrowseableRoot. Missing paths are silently skipped (first run).
	var rmParts []string
	for _, p := range paths {
		clean := p
		// Defensive: reject absolute paths and parent traversal so a
		// rogue config_values value can't `rm -rf /` the container.
		if strings.HasPrefix(clean, "/") || strings.Contains(clean, "..") {
			return fmt.Errorf("refusing wipe of suspicious path %q", p)
		}
		full := root + "/" + clean
		rmParts = append(rmParts, fmt.Sprintf("rm -rf %s", shellEscape(full)))
	}
	cmd := strings.Join(rmParts, " && ") + " && echo wiped"

	spec := runtime.InstanceSpec{
		InstanceID:    instanceID + "-wipe",
		IsSidecar:     true,
		Image:         backupImage,
		Entrypoint:    []string{"sh"},
		Command:       []string{"-c", cmd},
		Volumes:       volumes,
		RestartPolicy: "no",
	}
	contID, err := d.runtime.Create(ctx, spec)
	if err != nil {
		return fmt.Errorf("create wipe sidecar: %w", err)
	}
	defer func() {
		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		_ = d.runtime.Remove(rmCtx, contID)
	}()
	if err := d.runtime.Start(ctx, contID); err != nil {
		return fmt.Errorf("start wipe sidecar: %w", err)
	}
	logCtx, cancelLogs := context.WithCancel(ctx)
	logDone := make(chan struct{})
	go func() {
		defer close(logDone)
		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
			d.sendEnv(&panelv1.AgentEnvelope{
				SentAt: timestamppb.Now(),
				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
					InstanceId: instanceID, Stream: "wipe",
					At: timestamppb.Now(), Line: line,
				}},
			})
		})
	}()
	exitCode, err := d.runtime.Wait(ctx, contID)
	cancelLogs()
	<-logDone
	if err != nil {
		return fmt.Errorf("wait wipe sidecar: %w", err)
	}
	if exitCode != 0 {
		return fmt.Errorf("wipe sidecar exited %d", exitCode)
	}
	return nil
}

// ---- Restore ----

func (d *Dispatcher) handleRestore(ctx context.Context, corrID string, req *panelv1.RestoreRequest) {
	log := d.log.With("instance_id", req.InstanceId, "backup_id", req.BackupId, "correlation_id", corrID)

	rec, err := d.lookupRecord(req.InstanceId)
	if err != nil {
		d.sendRestoreResult(corrID, 0, err.Error())
		return
	}
	if rec.BrowseableRoot == "" {
		d.sendRestoreResult(corrID, 0, "instance has no BrowseableRoot to restore into")
		return
	}
	if req.BackupPath == "" {
		d.sendRestoreResult(corrID, 0, "backup_path required")
		return
	}
	// Refuse if container is running.
	if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil && state.Status == "running" {
		d.sendRestoreResult(corrID, 0, "stop the instance first — main container is running")
		return
	}
	info, err := os.Stat(req.BackupPath)
	if err != nil || info.IsDir() {
		d.sendRestoreResult(corrID, 0, "backup file missing or is a directory")
		return
	}

	volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath)
	// Add the backup file's parent as a read-only bind at /backup.
	hostDir := filepath.Dir(req.BackupPath)
	fileName := filepath.Base(req.BackupPath)
	volumes = append(volumes, runtime.VolumeSpec{
		Type:          "bind",
		HostPath:      hostDir,
		ContainerPath: "/backup",
		ReadOnly:      true,
	})

	// Extract strategy depends on whether the backup is per-game-paths
	// or a whole-tree archive. When the manifest declares Backup.Include,
	// only THOSE paths are wiped before extraction — preserves the
	// 18 GB game install while still giving a clean save dir. Without a
	// spec, fall back to wiping all of BrowseableRoot (legacy whole-tree
	// behavior).
	manifest := d.modulesMustGet(rec.ModuleID)
	// Resolve the SAME root the backup wrote against. buildBackupTarCmd
	// archives relative to manifest.Backup.Root when set (e.g. 7DTD's
	// /game-saves, where world data actually lives — distinct from the
	// /game install volume that is BrowseableRoot). Restore MUST extract
	// into that same root or the tarball lands in the wrong volume and the
	// live server never sees it (silent data loss: restore "succeeds" but
	// players' bases are gone). Mirror the backup's resolution exactly.
	restoreRoot := rec.BrowseableRoot
	if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" {
		restoreRoot = manifest.Backup.Root
	}
	var wipeCmd string
	if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 {
		// Wipe only the manifest-declared include paths so we don't
		// nuke unrelated install files. Each path is rm -rf'd; missing
		// ones are skipped silently (first-run case).
		var parts []string
		for _, p := range manifest.Backup.Include {
			clean := strings.TrimPrefix(strings.TrimPrefix(p, "./"), "/")
			parts = append(parts, fmt.Sprintf("rm -rf %s/%s", shellEscape(restoreRoot), shellEscape(clean)))
		}
		wipeCmd = strings.Join(parts, " ; ") + " 2>/dev/null"
		log.Info("restore: wiping manifest-declared paths only", "root", restoreRoot, "include_count", len(manifest.Backup.Include))
	} else {
		wipeCmd = fmt.Sprintf("rm -rf %q/* %q/.[!.]* 2>/dev/null", restoreRoot, restoreRoot)
		log.Warn("restore: wiping entire root (whole-tree backup)", "root", restoreRoot)
	}
	extractCmd := fmt.Sprintf(
		"%s ; tar xzf /backup/%s -C %q && du -sb %q | cut -f1",
		wipeCmd,
		shellEscape(fileName),
		restoreRoot,
		restoreRoot,
	)
	spec := runtime.InstanceSpec{
		InstanceID:    req.InstanceId + "-restore",
		IsSidecar:     true,
		Image:         backupImage,
		Entrypoint:    []string{"sh"},
		Command:       []string{"-c", extractCmd},
		Volumes:       volumes,
		RestartPolicy: "no",
	}
	contID, err := d.runtime.Create(ctx, spec)
	if err != nil {
		d.sendRestoreResult(corrID, 0, "create sidecar: "+err.Error())
		return
	}
	defer func() {
		rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		_ = d.runtime.Remove(rmCtx, contID)
	}()
	if err := d.runtime.Start(ctx, contID); err != nil {
		d.sendRestoreResult(corrID, 0, "start sidecar: "+err.Error())
		return
	}
	logCtx, cancelLogs := context.WithCancel(ctx)
	logDone := make(chan struct{})
	go func() {
		defer close(logDone)
		_ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) {
			d.sendEnv(&panelv1.AgentEnvelope{
				SentAt: timestamppb.Now(),
				Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{
					InstanceId: rec.InstanceID, Stream: "restore",
					At: timestamppb.Now(), Line: line,
				}},
			})
		})
	}()
	exitCode, err := d.runtime.Wait(ctx, contID)
	cancelLogs()
	<-logDone
	if err != nil {
		d.sendRestoreResult(corrID, 0, "wait: "+err.Error())
		return
	}
	if exitCode != 0 {
		d.sendRestoreResult(corrID, 0, fmt.Sprintf("sidecar exited %d", exitCode))
		return
	}
	log.Info("restore complete")
	d.sendRestoreResult(corrID, info.Size(), "")
}

func (d *Dispatcher) sendRestoreResult(corrID string, bytesRestored int64, errMsg string) {
	d.sendEnv(&panelv1.AgentEnvelope{
		CorrelationId: corrID,
		SentAt:        timestamppb.Now(),
		Payload: &panelv1.AgentEnvelope_RestoreResult{
			RestoreResult: &panelv1.RestoreResult{BytesRestored: bytesRestored, Error: errMsg},
		},
	})
}

// ---- helpers ----

func randHex(n int) string {
	b := make([]byte, n)
	_, _ = rand.Read(b)
	return hex.EncodeToString(b)
}

// shellEscape wraps a filename in single quotes for safe inclusion in an
// sh -c command. Assumes filename never contains a single-quote (ours
// never do — timestamps + bkp_<hex>).
func shellEscape(s string) string { return "'" + s + "'" }

// modulesMustGet returns the dispatcher's manifest for the given id, or
// nil if it's gone missing (shouldn't happen for a live instance).
func (d *Dispatcher) modulesMustGet(id string) *modulepkgManifest {
	m, _ := d.modules.Get(id)
	return m
}

// Keep errors imported for future error-wrapping.
var _ = errors.New