package dispatch import ( "context" "crypto/rand" "encoding/hex" "errors" "fmt" "os" "path/filepath" "strconv" "strings" "time" "google.golang.org/protobuf/types/known/timestamppb" agentmodule "github.com/dbledeez/panel/agent/internal/module" "github.com/dbledeez/panel/agent/internal/runtime" modulepkg "github.com/dbledeez/panel/pkg/module" panelv1 "github.com/dbledeez/panel/proto/panel/v1" ) // buildBackupTarCmd assembles the tar command run inside the sidecar. // Manifest BackupSpec wins when Include is set: the tar runs against // the listed paths (each relative to BrowseableRoot, missing ones // silently skipped). Otherwise it falls back to a whole-tree archive // of BrowseableRoot. Excludes are passed through tar's --exclude. // // Resulting tarball preserves the directory layout under BrowseableRoot // so restore can extract straight into the same root. func buildBackupTarCmd(manifest *modulepkgManifest, root, fileName string, keep int) string { if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" { root = manifest.Backup.Root } var base string if manifest == nil || manifest.Backup == nil || len(manifest.Backup.Include) == 0 { // Fallback: whole tree. base = fmt.Sprintf("cd %q && tar czf /backup/%s . && stat -c %%s /backup/%s", root, shellEscape(fileName), shellEscape(fileName)) } else { // Per-game include list. Missing paths are non-fatal — players will // create new instances on a fresh server before the save dir exists, // and we don't want their first Back-up-now to fail the run. excludes := "" for _, e := range manifest.Backup.Exclude { excludes += " --exclude=" + shellEscape(e) } // Build "test -e && echo " for each include — only // existing paths get added to the tar argv, so the operator doesn't // see a confusing "Cannot stat: No such file" on first run. // // Newline-delimited (NOT NUL-delimited) because the sidecar image is // alpine, which ships busybox tar — and busybox tar doesn't grok // `--null`. Manifest paths never contain whitespace or newlines (we // hand-curate them per game), so a plain newline list is safe. var existsChecks []string for _, p := range manifest.Backup.Include { clean := strings.TrimPrefix(p, "./") clean = strings.TrimPrefix(clean, "/") existsChecks = append(existsChecks, fmt.Sprintf("[ -e %s ] && printf '%%s\\n' %s", shellEscape(clean), shellEscape(clean))) } listExpr := "{ " + strings.Join(existsChecks, " ; ") + " ; }" base = fmt.Sprintf("cd %q && %s | tar czf /backup/%s%s -T - && stat -c %%s /backup/%s", root, listExpr, shellEscape(fileName), excludes, shellEscape(fileName)) } return base + backupPruneSuffix(keep) } // backupPruneSuffix appends a best-effort retention prune that keeps only the // newest `keep` *.tar.gz in /backup (by mtime) and deletes the rest. It runs in // the SAME root sidecar that just wrote the new tarball — the backup files are // root-owned, so the unprivileged agent can't prune them itself. keep<=0 leaves // pruning off. busybox-safe (alpine): ls -1t + tail -n +N + while-read — no // `head -n -N` (GNU-only) and no `xargs -r`. func backupPruneSuffix(keep int) string { if keep <= 0 { return "" } return fmt.Sprintf(" ; echo 'prune: keeping newest %d backups' ; ls -1t /backup/*.tar.gz 2>/dev/null | tail -n +%d | while IFS= read -r f; do echo \"prune: rm $f\" ; rm -f \"$f\" ; done", keep, keep+1) } // readBackupKeep returns the backup retention for this instance: the integer in // /backup-keep when present, else the PANEL_BACKUP_KEEP env default, // else 10. A rolling cap stops the unbounded pileup that filled 90 GB on a busy // 7DTD season. Set per-instance by writing the file (the Backups UI exposes it). func readBackupKeep(dataPath string) int { def := 10 if v := os.Getenv("PANEL_BACKUP_KEEP"); v != "" { if n, e := strconv.Atoi(strings.TrimSpace(v)); e == nil && n > 0 { def = n } } b, err := os.ReadFile(filepath.Join(dataPath, "backup-keep")) if err != nil { return def } if n, e := strconv.Atoi(strings.TrimSpace(string(b))); e == nil && n > 0 { return n } return def } // modulepkgManifest is a local alias so the helper signature below // doesn't drag modulepkg into its return type at the file top. type modulepkgManifest = modulepkg.Manifest // Backup / restore via sidecar containers. Two operations: // // - Backup: tar -czf /backup/.tar.gz -C /src . (volume readonly at /src) // - Restore: tar -xzf /backup/.tar.gz -C /dst (volume read-write at /dst) // // The sidecar uses alpine:3.21 + busybox tar. Backup files live on the // agent host under $BACKUP_DIR//-.tar.gz; both // sidecars bind-mount $BACKUP_DIR/ at /backup. // // For restore we refuse if the main container is running: extracting // files under a live server would corrupt world state. Operator stops // first. (Symmetrical to the SteamCMD updater's running-container check.) const ( backupImage = "alpine:3.21" backupSidecarTTL = 30 * time.Minute ) // ---- Backup ---- func (d *Dispatcher) handleBackup(ctx context.Context, corrID string, req *panelv1.BackupRequest) { log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID) rec, err := d.lookupRecord(req.InstanceId) if err != nil { d.sendBackupResult(corrID, "", "", 0, err.Error()) return } if rec.BrowseableRoot == "" { d.sendBackupResult(corrID, "", "", 0, "instance has no BrowseableRoot to tar (module declares no volumes)") return } if d.backupDir == "" { d.sendBackupResult(corrID, "", "", 0, "agent has no --backup-dir configured") return } // Prepare the agent-side target directory. bkpID := "bkp_" + randHex(8) hostDir := filepath.Join(d.backupDir, req.InstanceId) if err := os.MkdirAll(hostDir, 0o755); err != nil { d.sendBackupResult(corrID, "", "", 0, fmt.Sprintf("mkdir %s: %s", hostDir, err.Error())) return } ts := time.Now().UTC().Format("20060102-150405") fileName := fmt.Sprintf("%s-%s.tar.gz", ts, bkpID) hostFile := filepath.Join(hostDir, fileName) log.Info("backup starting", "backup_id", bkpID, "host_file", hostFile) // Build sidecar spec that mounts the same volumes as the instance, // plus the host backup directory at /backup. volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath) // Mark volumes read-only for backup safety. for i := range volumes { volumes[i].ReadOnly = true } // Add the host backup dir as a (read-write) bind. volumes = append(volumes, runtime.VolumeSpec{ Type: "bind", HostPath: hostDir, ContainerPath: "/backup", }) // Per-game backup paths. If the manifest declares Backup.Include, // only those paths get archived (saves-only — keeps ARK SA tarballs // at ~250 MB instead of the 18 GB whole-install). Without a spec, // fall back to the original "tar everything under BrowseableRoot" // behavior so older modules don't regress. manifest := d.modulesMustGet(rec.ModuleID) keep := readBackupKeep(rec.DataPath) tarCmd := buildBackupTarCmd(manifest, rec.BrowseableRoot, fileName, keep) if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 { log.Info("backup using manifest-declared paths", "include_count", len(manifest.Backup.Include)) } else { log.Warn("backup falling back to whole BrowseableRoot — module manifest has no `backup.include`", "module_id", rec.ModuleID, "root", rec.BrowseableRoot) } spec := runtime.InstanceSpec{ InstanceID: req.InstanceId + "-backup", IsSidecar: true, Image: backupImage, Entrypoint: []string{"sh"}, Command: []string{"-c", tarCmd}, Volumes: volumes, RestartPolicy: "no", } contID, err := d.runtime.Create(ctx, spec) if err != nil { d.sendBackupResult(corrID, bkpID, "", 0, "create sidecar: "+err.Error()) return } defer func() { rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = d.runtime.Remove(rmCtx, contID) }() if err := d.runtime.Start(ctx, contID); err != nil { d.sendBackupResult(corrID, bkpID, "", 0, "start sidecar: "+err.Error()) return } // Stream log lines for visibility. logCtx, cancelLogs := context.WithCancel(ctx) logDone := make(chan struct{}) go func() { defer close(logDone) _ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) { d.sendEnv(&panelv1.AgentEnvelope{ SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{ InstanceId: rec.InstanceID, Stream: "backup", At: timestamppb.Now(), Line: line, }}, }) }) }() exitCode, err := d.runtime.Wait(ctx, contID) cancelLogs() <-logDone if err != nil { d.sendBackupResult(corrID, bkpID, "", 0, "wait: "+err.Error()) return } if exitCode != 0 { d.sendBackupResult(corrID, bkpID, "", 0, fmt.Sprintf("sidecar exited %d", exitCode)) return } info, err := os.Stat(hostFile) if err != nil { d.sendBackupResult(corrID, bkpID, hostFile, 0, "stat result: "+err.Error()) return } log.Info("backup complete", "backup_id", bkpID, "size_bytes", info.Size()) d.sendBackupResult(corrID, bkpID, hostFile, info.Size(), "") } func (d *Dispatcher) sendBackupResult(corrID, bkpID, path string, size int64, errMsg string) { d.sendEnv(&panelv1.AgentEnvelope{ CorrelationId: corrID, SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_BackupResult{ BackupResult: &panelv1.BackupResult{ BackupId: bkpID, Path: path, SizeBytes: size, Error: errMsg, }, }, }) } // ---- Wipe paths (used by the env-config recreate flow) ---- // // Spawns an alpine sidecar that mounts the same volumes the instance // uses, runs `rm -rf` on each path (relative to BrowseableRoot), then // exits. Designed for the ARK map-switch flow — operator picked a new // SERVER_MAP and checked "wipe existing world data" so the old map's // SavedArks// goes away. Generic enough that any future // env-config field can declare its own wipe paths. func (d *Dispatcher) runWipeSidecar(ctx context.Context, manifest *modulepkg.Manifest, instanceID, dataPath string, paths []string) error { if manifest == nil { return errors.New("nil manifest") } root := resolveBrowseableRoot(manifest) if root == "" { return errors.New("module declares no browseable_root — wipe needs a base path") } volumes := agentmodule.ResolveVolumes(manifest, instanceID, dataPath) // Build the rm -rf command. Each path is rooted under // BrowseableRoot. Missing paths are silently skipped (first run). var rmParts []string for _, p := range paths { clean := p // Defensive: reject absolute paths and parent traversal so a // rogue config_values value can't `rm -rf /` the container. if strings.HasPrefix(clean, "/") || strings.Contains(clean, "..") { return fmt.Errorf("refusing wipe of suspicious path %q", p) } full := root + "/" + clean rmParts = append(rmParts, fmt.Sprintf("rm -rf %s", shellEscape(full))) } cmd := strings.Join(rmParts, " && ") + " && echo wiped" spec := runtime.InstanceSpec{ InstanceID: instanceID + "-wipe", IsSidecar: true, Image: backupImage, Entrypoint: []string{"sh"}, Command: []string{"-c", cmd}, Volumes: volumes, RestartPolicy: "no", } contID, err := d.runtime.Create(ctx, spec) if err != nil { return fmt.Errorf("create wipe sidecar: %w", err) } defer func() { rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = d.runtime.Remove(rmCtx, contID) }() if err := d.runtime.Start(ctx, contID); err != nil { return fmt.Errorf("start wipe sidecar: %w", err) } logCtx, cancelLogs := context.WithCancel(ctx) logDone := make(chan struct{}) go func() { defer close(logDone) _ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) { d.sendEnv(&panelv1.AgentEnvelope{ SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{ InstanceId: instanceID, Stream: "wipe", At: timestamppb.Now(), Line: line, }}, }) }) }() exitCode, err := d.runtime.Wait(ctx, contID) cancelLogs() <-logDone if err != nil { return fmt.Errorf("wait wipe sidecar: %w", err) } if exitCode != 0 { return fmt.Errorf("wipe sidecar exited %d", exitCode) } return nil } // ---- Restore ---- func (d *Dispatcher) handleRestore(ctx context.Context, corrID string, req *panelv1.RestoreRequest) { log := d.log.With("instance_id", req.InstanceId, "backup_id", req.BackupId, "correlation_id", corrID) rec, err := d.lookupRecord(req.InstanceId) if err != nil { d.sendRestoreResult(corrID, 0, err.Error()) return } if rec.BrowseableRoot == "" { d.sendRestoreResult(corrID, 0, "instance has no BrowseableRoot to restore into") return } if req.BackupPath == "" { d.sendRestoreResult(corrID, 0, "backup_path required") return } // Refuse if container is running. if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil && state.Status == "running" { d.sendRestoreResult(corrID, 0, "stop the instance first — main container is running") return } info, err := os.Stat(req.BackupPath) if err != nil || info.IsDir() { d.sendRestoreResult(corrID, 0, "backup file missing or is a directory") return } volumes := agentmodule.ResolveVolumes(d.modulesMustGet(rec.ModuleID), req.InstanceId, rec.DataPath) // Add the backup file's parent as a read-only bind at /backup. hostDir := filepath.Dir(req.BackupPath) fileName := filepath.Base(req.BackupPath) volumes = append(volumes, runtime.VolumeSpec{ Type: "bind", HostPath: hostDir, ContainerPath: "/backup", ReadOnly: true, }) // Extract strategy depends on whether the backup is per-game-paths // or a whole-tree archive. When the manifest declares Backup.Include, // only THOSE paths are wiped before extraction — preserves the // 18 GB game install while still giving a clean save dir. Without a // spec, fall back to wiping all of BrowseableRoot (legacy whole-tree // behavior). manifest := d.modulesMustGet(rec.ModuleID) // Resolve the SAME root the backup wrote against. buildBackupTarCmd // archives relative to manifest.Backup.Root when set (e.g. 7DTD's // /game-saves, where world data actually lives — distinct from the // /game install volume that is BrowseableRoot). Restore MUST extract // into that same root or the tarball lands in the wrong volume and the // live server never sees it (silent data loss: restore "succeeds" but // players' bases are gone). Mirror the backup's resolution exactly. restoreRoot := rec.BrowseableRoot if manifest != nil && manifest.Backup != nil && manifest.Backup.Root != "" { restoreRoot = manifest.Backup.Root } var wipeCmd string if manifest != nil && manifest.Backup != nil && len(manifest.Backup.Include) > 0 { // Wipe only the manifest-declared include paths so we don't // nuke unrelated install files. Each path is rm -rf'd; missing // ones are skipped silently (first-run case). var parts []string for _, p := range manifest.Backup.Include { clean := strings.TrimPrefix(strings.TrimPrefix(p, "./"), "/") parts = append(parts, fmt.Sprintf("rm -rf %s/%s", shellEscape(restoreRoot), shellEscape(clean))) } wipeCmd = strings.Join(parts, " ; ") + " 2>/dev/null" log.Info("restore: wiping manifest-declared paths only", "root", restoreRoot, "include_count", len(manifest.Backup.Include)) } else { wipeCmd = fmt.Sprintf("rm -rf %q/* %q/.[!.]* 2>/dev/null", restoreRoot, restoreRoot) log.Warn("restore: wiping entire root (whole-tree backup)", "root", restoreRoot) } extractCmd := fmt.Sprintf( "%s ; tar xzf /backup/%s -C %q && du -sb %q | cut -f1", wipeCmd, shellEscape(fileName), restoreRoot, restoreRoot, ) spec := runtime.InstanceSpec{ InstanceID: req.InstanceId + "-restore", IsSidecar: true, Image: backupImage, Entrypoint: []string{"sh"}, Command: []string{"-c", extractCmd}, Volumes: volumes, RestartPolicy: "no", } contID, err := d.runtime.Create(ctx, spec) if err != nil { d.sendRestoreResult(corrID, 0, "create sidecar: "+err.Error()) return } defer func() { rmCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = d.runtime.Remove(rmCtx, contID) }() if err := d.runtime.Start(ctx, contID); err != nil { d.sendRestoreResult(corrID, 0, "start sidecar: "+err.Error()) return } logCtx, cancelLogs := context.WithCancel(ctx) logDone := make(chan struct{}) go func() { defer close(logDone) _ = d.runtime.StreamLogs(logCtx, contID, func(_ runtime.LogStream, line string, _ time.Time) { d.sendEnv(&panelv1.AgentEnvelope{ SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_Log{Log: &panelv1.LogLine{ InstanceId: rec.InstanceID, Stream: "restore", At: timestamppb.Now(), Line: line, }}, }) }) }() exitCode, err := d.runtime.Wait(ctx, contID) cancelLogs() <-logDone if err != nil { d.sendRestoreResult(corrID, 0, "wait: "+err.Error()) return } if exitCode != 0 { d.sendRestoreResult(corrID, 0, fmt.Sprintf("sidecar exited %d", exitCode)) return } log.Info("restore complete") d.sendRestoreResult(corrID, info.Size(), "") } func (d *Dispatcher) sendRestoreResult(corrID string, bytesRestored int64, errMsg string) { d.sendEnv(&panelv1.AgentEnvelope{ CorrelationId: corrID, SentAt: timestamppb.Now(), Payload: &panelv1.AgentEnvelope_RestoreResult{ RestoreResult: &panelv1.RestoreResult{BytesRestored: bytesRestored, Error: errMsg}, }, }) } // ---- helpers ---- func randHex(n int) string { b := make([]byte, n) _, _ = rand.Read(b) return hex.EncodeToString(b) } // shellEscape wraps a filename in single quotes for safe inclusion in an // sh -c command. Assumes filename never contains a single-quote (ours // never do — timestamps + bkp_). func shellEscape(s string) string { return "'" + s + "'" } // modulesMustGet returns the dispatcher's manifest for the given id, or // nil if it's gone missing (shouldn't happen for a live instance). func (d *Dispatcher) modulesMustGet(id string) *modulepkgManifest { m, _ := d.modules.Get(id) return m } // Keep errors imported for future error-wrapping. var _ = errors.New