panel/agent/internal/dispatch/delete.go

package dispatch

import (
	"context"
	"fmt"
	"os"
	"time"

	"google.golang.org/protobuf/types/known/timestamppb"

	"github.com/dbledeez/panel/agent/internal/runtime"
	panelv1 "github.com/dbledeez/panel/proto/panel/v1"
)

// handleDelete tears down the instance record. If the container is still
// running, it's stopped with the requested grace first. Optionally
// removes every Docker volume labeled for this instance ("purge").
//
// Never deletes the host data_path directory — operators may want to
// keep configs/logs that landed on the bind mount. If they want a full
// wipe, they can delete the dir manually after.
func (d *Dispatcher) handleDelete(ctx context.Context, corrID string, req *panelv1.InstanceDelete) {
	log := d.log.With("instance_id", req.InstanceId, "correlation_id", corrID)

	d.mu.Lock()
	rec, exists := d.instances[req.InstanceId]
	d.mu.Unlock()

	// Cancel any in-flight steamcmd / updater goroutine FIRST. The
	// updater spins up its own sidecar container (panel-<id>-steamcmd-<n>)
	// that mounts the same data volumes. If we don't cancel it before
	// trying to remove volumes, the sidecar holds them open and the
	// volume-rm fails with "volume is in use" — operator sees the
	// instance stuck on "Deleting…" while a download finishes in the
	// background.
	d.cancelUpdater(req.InstanceId)
	// Whatever state the main container is in, any Files-tab helper that
	// was running against this instance is now orphaned. Clean it up
	// before touching anything else so its volume-holds don't interfere
	// with the main container's stop/remove.
	d.teardownFsHelper(req.InstanceId)

	grace := time.Duration(req.GraceSeconds) * time.Second
	if grace == 0 {
		grace = 15 * time.Second
	}

	// If we have a record, use its container_id directly. Otherwise try
	// to find the container by its canonical name (may exist from a
	// previous session that skipped rehydrate).
	containerID := ""
	if exists {
		containerID = rec.ContainerID
	} else {
		if state, err := d.runtime.InspectByName(ctx, "panel-"+req.InstanceId); err == nil {
			containerID = state.ContainerID
		}
	}

	if containerID != "" {
		// Best-effort stop; ignore errors since container may already be stopped.
		_ = d.runtime.Stop(ctx, containerID, grace)
		removeErr := d.runtime.Remove(ctx, containerID)
		// d.runtime.Remove already swallows not-found, so a non-nil
		// removeErr means the remove genuinely failed. Before we tear down
		// any bookkeeping (in-memory record + sidecar metadata) we must be
		// SURE the container is gone — otherwise we orphan a live container
		// the controller still routes to, and the agent forgets it on the
		// next rehydrate. The classic trigger: a delete racing with agent
		// shutdown, where the Docker connection dies mid-remove
		// ("terminated signal received"). The remove errors, and the
		// follow-up existence check ALSO errors — fate unknown, not gone.
		if removeErr != nil {
			exists, existErr := d.runtime.ContainerExists(ctx, "panel-"+req.InstanceId)
			switch {
			case existErr != nil:
				// Couldn't even ask (daemon down / connection dropped /
				// ctx cancelled). Do NOT delete the sidecar — bail and let
				// the operator retry once the daemon is reachable. Leaving
				// the sidecar means the next rehydrate re-adopts the
				// instance instead of orphaning it.
				log.Error("delete: container remove failed and existence is unknown — aborting to avoid orphan",
					"remove_err", removeErr, "exists_err", existErr)
				d.replyError(corrID, req.InstanceId, "container_remove",
					fmt.Sprintf("container panel-%s remove failed and existence unverifiable: %s", req.InstanceId, removeErr.Error()))
				return
			case exists:
				// Container is confirmed still present. The controller must
				// learn — otherwise the DB row gets purged while a zombie
				// container lives on, and the next create with the same id
				// fails on a "name already in use" docker error.
				log.Error("delete: container still exists after remove", "err", removeErr)
				d.replyError(corrID, req.InstanceId, "container_remove",
					fmt.Sprintf("container panel-%s still exists after remove: %s", req.InstanceId, removeErr.Error()))
				return
			default:
				// Confirmed gone (e.g. autoremove fired between Stop and
				// Remove). Safe to proceed with bookkeeping cleanup.
				log.Warn("delete: remove returned error but container is confirmed gone", "err", removeErr)
			}
		}
	}

	// Signal any live trackers / log streams to stop.
	if exists {
		if rec.TrackerCancel != nil {
			rec.TrackerCancel()
		}
		if rec.LogCancel != nil {
			rec.LogCancel()
		}
	}

	// Purge volumes if requested.
	if req.PurgeVolumes {
		dr, ok := d.runtime.(*runtime.DockerRuntime)
		if !ok {
			log.Warn("delete: purge requested but runtime is not Docker; skipping volume cleanup")
		} else {
			// Sweep every container that mounts one of this instance's
			// volumes — running OR exited. Without this, a stranded
			// steamcmd / backup / fs-helper sidecar from a previous
			// agent crash keeps the volume "in use" and volume-rm
			// fails repeatedly. Operator sees the instance stuck
			// "Deleting…" forever.
			holders, err := dr.ListContainersHoldingInstanceVolumes(ctx, req.InstanceId)
			if err != nil {
				log.Warn("delete: scan for orphan sidecars failed", "err", err)
			}
			for _, hid := range holders {
				if err := dr.ForceRemoveContainer(ctx, hid); err != nil {
					log.Warn("delete: force-remove sidecar failed", "container_id", hid, "err", err)
				} else {
					log.Info("delete: removed orphan sidecar", "container_id", short(hid))
				}
			}
			vols, err := dr.ListVolumesByInstance(ctx, req.InstanceId)
			if err != nil {
				log.Warn("delete: list volumes failed", "err", err)
			}
			for _, v := range vols {
				// Force=true: there's no scenario where we want a panel
				// volume to survive a purge=true delete after we've
				// scrubbed every container that referenced it.
				if err := dr.RemoveVolume(ctx, v, true); err != nil {
					log.Warn("delete: remove volume failed", "volume", v, "err", err)
					d.replyError(corrID, req.InstanceId, "volume_remove", fmt.Sprintf("volume %s: %s", v, err.Error()))
					return
				}
				log.Info("delete: purged volume", "volume", v)
			}
		}
	}

	// Remove in-memory record + sidecar metadata.
	d.mu.Lock()
	delete(d.instances, req.InstanceId)
	d.mu.Unlock()
	d.deleteMeta(req.InstanceId)

	// Leave host data_path alone (see doc comment); operators can rm -rf
	// manually. We do NOT attempt os.RemoveAll here — too destructive to
	// do automatically on a shared multi-instance directory layout.

	d.sendInstanceState(req.InstanceId, panelv1.InstanceStatus_INSTANCE_STATUS_STOPPED, 0, "deleted")
	d.replyOK(corrID)
	log.Info("instance deleted", "purge_volumes", req.PurgeVolumes)
	_ = os.Stat // kept imported for future host-data_path removal flag
}

// sendDelete-specific correlation reply uses the existing replyOK/err pattern.
var _ = timestamppb.Now // keep import hint while ctx-based future expansions land