582b5a6b08
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
265 lines
9.0 KiB
Go
265 lines
9.0 KiB
Go
package dispatch
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/dbledeez/panel/pkg/regionmedic"
|
|
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
)
|
|
|
|
// Region Medic agent handlers — scan a 7DTD instance's active-world Region
|
|
// directory for .7rg corruption, and heal one region from the newest clean
|
|
// panel backup. Volume files are reached through the runtime's container-path
|
|
// ops (the agent runs unprivileged, so it cannot touch /var/lib/docker/volumes
|
|
// directly). The controller orchestrates stop→heal→start; handleRegionHeal
|
|
// itself only performs the file swap and refuses to run on a live container.
|
|
|
|
// activeRegionDir resolves the container-absolute Region directory of the
|
|
// instance's active world by reading GameWorld/GameName from the live
|
|
// serverconfig.xml. Works on a stopped container (CopyFileFromContainer does).
|
|
func (d *Dispatcher) activeRegionDir(ctx context.Context, container string) (string, error) {
|
|
raw, err := d.runtime.CopyFileFromContainer(ctx, container, "/game-saves/serverconfig.xml")
|
|
if err != nil {
|
|
return "", fmt.Errorf("read serverconfig.xml: %w", err)
|
|
}
|
|
world := xmlProp(string(raw), "GameWorld")
|
|
game := xmlProp(string(raw), "GameName")
|
|
if world == "" || game == "" {
|
|
return "", fmt.Errorf("serverconfig.xml missing GameWorld/GameName (got %q/%q)", world, game)
|
|
}
|
|
// HOME is pinned to /game-saves by the 7dtd entrypoint.
|
|
return path.Join("/game-saves/.local/share/7DaysToDie/Saves", world, game, "Region"), nil
|
|
}
|
|
|
|
// xmlProp pulls the value from a 7DTD serverconfig line:
|
|
//
|
|
// <property name="GameWorld" value="Navezgane" />
|
|
//
|
|
// Tolerant of attribute order and single/double quotes; skips commented lines.
|
|
func xmlProp(content, name string) string {
|
|
for _, raw := range strings.Split(content, "\n") {
|
|
line := strings.TrimSpace(raw)
|
|
if line == "" || strings.HasPrefix(line, "<!--") {
|
|
continue
|
|
}
|
|
if !strings.Contains(line, `name="`+name+`"`) && !strings.Contains(line, `name='`+name+`'`) {
|
|
continue
|
|
}
|
|
i := strings.Index(line, "value=")
|
|
if i < 0 {
|
|
continue
|
|
}
|
|
rest := line[i+len("value="):]
|
|
if rest == "" {
|
|
continue
|
|
}
|
|
q := rest[0]
|
|
rest = rest[1:]
|
|
if end := strings.IndexByte(rest, q); end >= 0 {
|
|
return strings.TrimSpace(rest[:end])
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// listRegionDirNames returns the file names in the container's Region dir.
|
|
// Uses ExecCapture when the container is running; nil + error otherwise (the
|
|
// scan path runs while the instance is up).
|
|
func (d *Dispatcher) listRegionDirNames(ctx context.Context, container, regionDir string) ([]string, error) {
|
|
stdout, _, _, err := d.runtime.ExecCapture(ctx, container, []string{"ls", "-1", regionDir})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var names []string
|
|
for _, n := range strings.Split(string(stdout), "\n") {
|
|
if n = strings.TrimSpace(n); n != "" {
|
|
names = append(names, n)
|
|
}
|
|
}
|
|
return names, nil
|
|
}
|
|
|
|
// handleRegionScan reports corruption evidence for an instance's active world:
|
|
// error_backup salvage files clustered into regions, plus structural validation
|
|
// of each affected region's .7rg file.
|
|
func (d *Dispatcher) handleRegionScan(ctx context.Context, corrID string, req *panelv1.RegionScanRequest) {
|
|
rec, err := d.lookupRecord(req.InstanceId)
|
|
if err != nil {
|
|
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "not_found", Message: err.Error()}})
|
|
return
|
|
}
|
|
container := "panel-" + rec.InstanceID
|
|
regionDir, err := d.activeRegionDir(ctx, container)
|
|
if err != nil {
|
|
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{Error: &panelv1.Error{Code: "resolve_region_dir", Message: err.Error()}})
|
|
return
|
|
}
|
|
names, err := d.listRegionDirNames(ctx, container, regionDir)
|
|
if err != nil {
|
|
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{RegionDir: regionDir, Error: &panelv1.Error{Code: "list_region_dir", Message: err.Error()}})
|
|
return
|
|
}
|
|
|
|
// Cluster error_backups by region; note which region files exist.
|
|
type agg struct {
|
|
files int
|
|
present bool
|
|
}
|
|
clusters := map[regionmedic.RegionCoord]*agg{}
|
|
fileSet := map[regionmedic.RegionCoord]bool{}
|
|
total := 0
|
|
for _, n := range names {
|
|
if cx, cz, ok := regionmedic.ParseErrorBackup(n); ok {
|
|
total++
|
|
rc := regionmedic.RegionForChunk(cx, cz)
|
|
a := clusters[rc]
|
|
if a == nil {
|
|
a = &agg{}
|
|
clusters[rc] = a
|
|
}
|
|
a.files++
|
|
continue
|
|
}
|
|
if rc, ok := regionmedic.ParseRegionFileName(n); ok {
|
|
fileSet[rc] = true
|
|
}
|
|
}
|
|
|
|
var affected []*panelv1.AffectedRegionMsg
|
|
for rc, a := range clusters {
|
|
msg := &panelv1.AffectedRegionMsg{
|
|
Region: rc.String(),
|
|
ErrorBackups: int32(a.files),
|
|
FilePresent: fileSet[rc],
|
|
}
|
|
// Validate the affected region file (read via container-path op).
|
|
if fileSet[rc] {
|
|
if b, rerr := d.runtime.CopyFileFromContainer(ctx, container, path.Join(regionDir, rc.FileName())); rerr == nil {
|
|
rep := regionmedic.ValidateRegionBytes(b, false)
|
|
msg.FileCorrupt = !rep.Healthy()
|
|
msg.BadChunks = int32(len(rep.BadChunks))
|
|
}
|
|
}
|
|
affected = append(affected, msg)
|
|
}
|
|
sort.Slice(affected, func(i, j int) bool { return affected[i].ErrorBackups > affected[j].ErrorBackups })
|
|
|
|
d.sendRegionScanResult(corrID, &panelv1.RegionScanResult{
|
|
ErrorBackups: int32(total),
|
|
Affected: affected,
|
|
RegionDir: regionDir,
|
|
})
|
|
}
|
|
|
|
// handleRegionHeal swaps one region file for the newest clean backup copy.
|
|
// It REQUIRES the instance to be stopped (the controller stops it first) so the
|
|
// running engine can't overwrite the swap. It snapshots the corrupt original
|
|
// next to the region file before overwriting. Orchestration (stop/start) is the
|
|
// controller's job.
|
|
func (d *Dispatcher) handleRegionHeal(ctx context.Context, corrID string, req *panelv1.RegionHealRequest) {
|
|
fail := func(code, msg string) {
|
|
d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{Region: req.Region, Error: &panelv1.Error{Code: code, Message: msg}})
|
|
}
|
|
rec, err := d.lookupRecord(req.InstanceId)
|
|
if err != nil {
|
|
fail("not_found", err.Error())
|
|
return
|
|
}
|
|
region, ok := regionmedic.ParseRegionFileName(regionFileArg(req.Region))
|
|
if !ok {
|
|
fail("bad_region", fmt.Sprintf("invalid region %q", req.Region))
|
|
return
|
|
}
|
|
container := "panel-" + rec.InstanceID
|
|
|
|
// Refuse to swap a region on a LIVE container (it would be overwritten on
|
|
// the next save). ExecCapture succeeds only when the container is running.
|
|
if _, _, _, exErr := d.runtime.ExecCapture(ctx, container, []string{"true"}); exErr == nil {
|
|
fail("instance_running", "instance must be stopped before healing a region")
|
|
return
|
|
}
|
|
|
|
regionDir, err := d.activeRegionDir(ctx, container)
|
|
if err != nil {
|
|
fail("resolve_region_dir", err.Error())
|
|
return
|
|
}
|
|
liveFile := path.Join(regionDir, region.FileName())
|
|
liveBytes, err := d.runtime.CopyFileFromContainer(ctx, container, liveFile)
|
|
if err != nil {
|
|
fail("read_live_region", err.Error())
|
|
return
|
|
}
|
|
|
|
// Find the newest clean backup copy of this region (backups are local to
|
|
// the agent at <backupDir>/<instance>/).
|
|
backups, err := regionmedic.ListPanelBackups(path.Join(d.backupDir, rec.InstanceID))
|
|
if err != nil {
|
|
fail("list_backups", err.Error())
|
|
return
|
|
}
|
|
entry := regionmedic.SaveRelEntry(regionDir, region.FileName())
|
|
best, _, err := regionmedic.FindLastGoodRegion(backups, entry, true, time.Time{})
|
|
if err != nil {
|
|
fail("find_backup", err.Error())
|
|
return
|
|
}
|
|
if !best.Found || !best.Clean() {
|
|
fail("no_clean_backup", "no backup contains a clean copy of this region")
|
|
return
|
|
}
|
|
|
|
// Snapshot the corrupt original alongside the region file (no new subdir, so
|
|
// CopyFileToContainer needs no parent-dir creation on a stopped container).
|
|
stamp := time.Now().UTC().Format("20060102-150405")
|
|
snapName := region.FileName() + ".medic-corrupt-" + stamp
|
|
snapPath := path.Join(regionDir, snapName)
|
|
if err := d.runtime.CopyFileToContainer(ctx, container, snapPath, liveBytes); err != nil {
|
|
fail("snapshot", err.Error())
|
|
return
|
|
}
|
|
// Swap in the clean copy (overwrites the live region file in place).
|
|
if err := d.runtime.CopyFileToContainer(ctx, container, liveFile, best.Bytes); err != nil {
|
|
fail("write_region", err.Error())
|
|
return
|
|
}
|
|
|
|
d.sendRegionHealResult(corrID, &panelv1.RegionHealResult{
|
|
Region: region.String(),
|
|
SourceBackup: best.Backup.ID,
|
|
BytesWritten: int64(len(best.Bytes)),
|
|
SnapshotPath: snapPath,
|
|
Healed: true,
|
|
})
|
|
}
|
|
|
|
// regionFileArg normalizes "r.X.Z" or "r.X.Z.7rg" to a .7rg filename.
|
|
func regionFileArg(s string) string {
|
|
if strings.HasSuffix(s, ".7rg") {
|
|
return s
|
|
}
|
|
return s + ".7rg"
|
|
}
|
|
|
|
func (d *Dispatcher) sendRegionScanResult(corrID string, res *panelv1.RegionScanResult) {
|
|
d.sendEnv(&panelv1.AgentEnvelope{
|
|
CorrelationId: corrID,
|
|
SentAt: timestamppb.Now(),
|
|
Payload: &panelv1.AgentEnvelope_RegionScanResult{RegionScanResult: res},
|
|
})
|
|
}
|
|
|
|
func (d *Dispatcher) sendRegionHealResult(corrID string, res *panelv1.RegionHealResult) {
|
|
d.sendEnv(&panelv1.AgentEnvelope{
|
|
CorrelationId: corrID,
|
|
SentAt: timestamppb.Now(),
|
|
Payload: &panelv1.AgentEnvelope_RegionHealResult{RegionHealResult: res},
|
|
})
|
|
}
|