172 lines
6.9 KiB
Go
172 lines
6.9 KiB
Go
package dispatch
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
|
|
"github.com/dbledeez/panel/agent/internal/runtime"
|
|
panelv1 "github.com/dbledeez/panel/proto/panel/v1"
|
|
)
|
|
|
|
// handleDelete tears down the instance record. If the container is still
|
|
// running, it's stopped with the requested grace first. Optionally
|
|
// removes every Docker volume labeled for this instance ("purge").
|
|
//
|
|
// Never deletes the host data_path directory — operators may want to
|
|
// keep configs/logs that landed on the bind mount. If they want a full
|
|
// wipe, they can delete the dir manually after.
|
|
func (d *Dispatcher) handleDelete(ctx context.Context, corrID string, req *panelv1.InstanceDelete) {
|
|
log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)
|
|
|
|
d.mu.Lock()
|
|
rec, exists := d.instances[req.InstanceId]
|
|
d.mu.Unlock()
|
|
|
|
// Cancel any in-flight steamcmd / updater goroutine FIRST. The
|
|
// updater spins up its own sidecar container (panel-<id>-steamcmd-<n>)
|
|
// that mounts the same data volumes. If we don't cancel it before
|
|
// trying to remove volumes, the sidecar holds them open and the
|
|
// volume-rm fails with "volume is in use" — operator sees the
|
|
// instance stuck on "Deleting…" while a download finishes in the
|
|
// background.
|
|
d.cancelUpdater(req.InstanceId)
|
|
// Whatever state the main container is in, any Files-tab helper that
|
|
// was running against this instance is now orphaned. Clean it up
|
|
// before touching anything else so its volume-holds don't interfere
|
|
// with the main container's stop/remove.
|
|
d.teardownFsHelper(req.InstanceId)
|
|
|
|
grace := time.Duration(req.GraceSeconds) * time.Second
|
|
if grace == 0 {
|
|
grace = 15 * time.Second
|
|
}
|
|
|
|
// If we have a record, use its container_id directly. Otherwise try
|
|
// to find the container by its canonical name (may exist from a
|
|
// previous session that skipped rehydrate).
|
|
containerID := ""
|
|
if exists {
|
|
containerID = rec.ContainerID
|
|
} else {
|
|
if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil {
|
|
containerID = state.ContainerID
|
|
}
|
|
}
|
|
|
|
if containerID != "" {
|
|
// Best-effort stop; ignore errors since container may already be stopped.
|
|
_ = d.runtime.Stop(ctx, containerID, grace)
|
|
removeErr := d.runtime.Remove(ctx, containerID)
|
|
// d.runtime.Remove already swallows not-found, so a non-nil
|
|
// removeErr means the remove genuinely failed. Before we tear down
|
|
// any bookkeeping (in-memory record + sidecar metadata) we must be
|
|
// SURE the container is gone — otherwise we orphan a live container
|
|
// the controller still routes to, and the agent forgets it on the
|
|
// next rehydrate. The classic trigger: a delete racing with agent
|
|
// shutdown, where the Docker connection dies mid-remove
|
|
// ("terminated signal received"). The remove errors, and the
|
|
// follow-up existence check ALSO errors — fate unknown, not gone.
|
|
if removeErr != nil {
|
|
exists, existErr := d.runtime.ContainerExists(ctx, "panel-"+req.InstanceId)
|
|
switch {
|
|
case existErr != nil:
|
|
// Couldn't even ask (daemon down / connection dropped /
|
|
// ctx cancelled). Do NOT delete the sidecar — bail and let
|
|
// the operator retry once the daemon is reachable. Leaving
|
|
// the sidecar means the next rehydrate re-adopts the
|
|
// instance instead of orphaning it.
|
|
log.Error("delete: container remove failed and existence is unknown — aborting to avoid orphan",
|
|
"remove_err", removeErr, "exists_err", existErr)
|
|
d.replyError(corrID, req.InstanceId, "container_remove",
|
|
fmt.Sprintf("container panel-%s remove failed and existence unverifiable: %s", req.InstanceId, removeErr.Error()))
|
|
return
|
|
case exists:
|
|
// Container is confirmed still present. The controller must
|
|
// learn — otherwise the DB row gets purged while a zombie
|
|
// container lives on, and the next create with the same id
|
|
// fails on a "name already in use" docker error.
|
|
log.Error("delete: container still exists after remove", "err", removeErr)
|
|
d.replyError(corrID, req.InstanceId, "container_remove",
|
|
fmt.Sprintf("container panel-%s still exists after remove: %s", req.InstanceId, removeErr.Error()))
|
|
return
|
|
default:
|
|
// Confirmed gone (e.g. autoremove fired between Stop and
|
|
// Remove). Safe to proceed with bookkeeping cleanup.
|
|
log.Warn("delete: remove returned error but container is confirmed gone", "err", removeErr)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Signal any live trackers / log streams to stop.
|
|
if exists {
|
|
if rec.TrackerCancel != nil {
|
|
rec.TrackerCancel()
|
|
}
|
|
if rec.LogCancel != nil {
|
|
rec.LogCancel()
|
|
}
|
|
}
|
|
|
|
// Purge volumes if requested.
|
|
if req.PurgeVolumes {
|
|
dr, ok := d.runtime.(*runtime.DockerRuntime)
|
|
if !ok {
|
|
log.Warn("delete: purge requested but runtime is not Docker; skipping volume cleanup")
|
|
} else {
|
|
// Sweep every container that mounts one of this instance's
|
|
// volumes — running OR exited. Without this, a stranded
|
|
// steamcmd / backup / fs-helper sidecar from a previous
|
|
// agent crash keeps the volume "in use" and volume-rm
|
|
// fails repeatedly. Operator sees the instance stuck
|
|
// "Deleting…" forever.
|
|
holders, err := dr.ListContainersHoldingInstanceVolumes(ctx, req.InstanceId)
|
|
if err != nil {
|
|
log.Warn("delete: scan for orphan sidecars failed", "err", err)
|
|
}
|
|
for _, hid := range holders {
|
|
if err := dr.ForceRemoveContainer(ctx, hid); err != nil {
|
|
log.Warn("delete: force-remove sidecar failed", "container_id", hid, "err", err)
|
|
} else {
|
|
log.Info("delete: removed orphan sidecar", "container_id", short(hid))
|
|
}
|
|
}
|
|
vols, err := dr.ListVolumesByInstance(ctx, req.InstanceId)
|
|
if err != nil {
|
|
log.Warn("delete: list volumes failed", "err", err)
|
|
}
|
|
for _, v := range vols {
|
|
// Force=true: there's no scenario where we want a panel
|
|
// volume to survive a purge=true delete after we've
|
|
// scrubbed every container that referenced it.
|
|
if err := dr.RemoveVolume(ctx, v, true); err != nil {
|
|
log.Warn("delete: remove volume failed", "volume", v, "err", err)
|
|
d.replyError(corrID, req.InstanceId, "volume_remove", fmt.Sprintf("volume %s: %s", v, err.Error()))
|
|
return
|
|
}
|
|
log.Info("delete: purged volume", "volume", v)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove in-memory record + sidecar metadata.
|
|
d.mu.Lock()
|
|
delete(d.instances, req.InstanceId)
|
|
d.mu.Unlock()
|
|
d.deleteMeta(req.InstanceId)
|
|
|
|
// Leave host data_path alone (see doc comment); operators can rm -rf
|
|
// manually. We do NOT attempt os.RemoveAll here — too destructive to
|
|
// do automatically on a shared multi-instance directory layout.
|
|
|
|
d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "deleted")
|
|
d.replyOK(corrID)
|
|
log.Info("instance deleted", "purge_volumes", req.PurgeVolumes)
|
|
_ = os.Stat // kept imported for future host-data_path removal flag
|
|
}
|
|
|
|
// sendDelete-specific correlation reply uses the existing replyOK/err pattern.
|
|
var _ = timestamppb.Now // keep import hint while ctx-based future expansions land |