mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-18 14:17:10 +00:00
feat(agents): 3-layer recovery for agents stuck in summoning state
Add ResetStuckSummoning store method (PG + SQLite) + startup sweep to
flip ghost rows to summon_failed on boot. Drop 409 guards on resummon
and regenerate to allow retry. Add POST /v1/agents/{id}/cancel-summon
endpoint + audit event agent.summon_cancelled + i18n (en/vi/zh).
This commit is contained in:
@@ -143,6 +143,17 @@ func runGateway() {
|
||||
}
|
||||
|
||||
pgStores, traceCollector, snapshotWorker := setupStoresAndTracing(cfg, dataDir, msgBus)
|
||||
|
||||
// Recover from crashes: flip ghost 'summoning' rows to 'summon_failed'.
|
||||
// Summon goroutines don't survive process restart; stale DB rows would trap the UI.
|
||||
if pgStores.Agents != nil {
|
||||
if n, err := pgStores.Agents.ResetStuckSummoning(context.Background()); err != nil {
|
||||
slog.Warn("agents.reset_stuck_summoning_failed", "err", err)
|
||||
} else if n > 0 {
|
||||
slog.Info("agents.reset_stuck_summoning", "count", n)
|
||||
}
|
||||
}
|
||||
|
||||
if traceCollector != nil {
|
||||
defer traceCollector.Stop()
|
||||
// OTel OTLP export: compiled via build tags. Build with 'go build -tags otel' to enable.
|
||||
|
||||
@@ -145,6 +145,7 @@ func (h *AgentsHandler) RegisterRoutes(mux *http.ServeMux) {
|
||||
// Agent operations (admin+)
|
||||
mux.HandleFunc("POST /v1/agents/{id}/regenerate", h.adminMiddleware(h.handleRegenerate))
|
||||
mux.HandleFunc("POST /v1/agents/{id}/resummon", h.adminMiddleware(h.handleResummon))
|
||||
mux.HandleFunc("POST /v1/agents/{id}/cancel-summon", h.adminMiddleware(h.handleCancelSummon))
|
||||
// Export (agent owner or system owner)
|
||||
mux.HandleFunc("GET /v1/agents/{id}/system-prompt-preview", h.adminMiddleware(h.handleSystemPromptPreview))
|
||||
mux.HandleFunc("GET /v1/agents/{id}/export/preview", h.authMiddleware(h.handleExportPreview))
|
||||
|
||||
@@ -5,8 +5,10 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/i18n"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
|
||||
)
|
||||
|
||||
func (h *AgentsHandler) handleListShares(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -139,10 +141,6 @@ func (h *AgentsHandler) handleRegenerate(w http.ResponseWriter, r *http.Request)
|
||||
writeJSON(w, http.StatusForbidden, map[string]string{"error": i18n.T(locale, i18n.MsgOwnerOnly, "regenerate agent")})
|
||||
return
|
||||
}
|
||||
if ag.Status == store.AgentStatusSummoning {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": i18n.T(locale, i18n.MsgAlreadySummoning)})
|
||||
return
|
||||
}
|
||||
if h.summoner == nil {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": i18n.T(locale, i18n.MsgSummoningUnavailable)})
|
||||
return
|
||||
@@ -191,10 +189,6 @@ func (h *AgentsHandler) handleResummon(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusForbidden, map[string]string{"error": i18n.T(locale, i18n.MsgOwnerOnly, "resummon agent")})
|
||||
return
|
||||
}
|
||||
if ag.Status == store.AgentStatusSummoning {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": i18n.T(locale, i18n.MsgAlreadySummoning)})
|
||||
return
|
||||
}
|
||||
if h.summoner == nil {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": i18n.T(locale, i18n.MsgSummoningUnavailable)})
|
||||
return
|
||||
@@ -217,4 +211,46 @@ func (h *AgentsHandler) handleResummon(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{"ok": "true", "status": store.AgentStatusSummoning})
|
||||
}
|
||||
|
||||
// handleCancelSummon force-transitions a stuck 'summoning' agent to 'summon_failed'.
|
||||
// Used when user wants to abort a hanging summon (UI Cancel button after 60s).
|
||||
func (h *AgentsHandler) handleCancelSummon(w http.ResponseWriter, r *http.Request) {
|
||||
userID := store.UserIDFromContext(r.Context())
|
||||
locale := store.LocaleFromContext(r.Context())
|
||||
id, err := uuid.Parse(r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidID, "agent")})
|
||||
return
|
||||
}
|
||||
|
||||
ag, err := h.agents.GetByID(r.Context(), id)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusNotFound, map[string]string{"error": i18n.T(locale, i18n.MsgNotFound, "agent", id.String())})
|
||||
return
|
||||
}
|
||||
if userID != "" && ag.OwnerID != userID && !h.isOwnerUser(userID) {
|
||||
writeJSON(w, http.StatusForbidden, map[string]string{"error": i18n.T(locale, i18n.MsgOwnerOnly, "cancel summon")})
|
||||
return
|
||||
}
|
||||
if ag.Status != store.AgentStatusSummoning {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": i18n.T(locale, i18n.MsgCannotCancel)})
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.agents.Update(r.Context(), id, map[string]any{"status": store.AgentStatusSummonFailed}); err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if h.msgBus != nil {
|
||||
bus.BroadcastForTenant(h.msgBus, protocol.EventAgentSummoning, store.TenantIDFromContext(r.Context()), map[string]any{
|
||||
"agent_id": id.String(),
|
||||
"type": "failed",
|
||||
"error": i18n.T(locale, i18n.MsgSummonCancelled),
|
||||
})
|
||||
}
|
||||
|
||||
emitAudit(h.msgBus, r, "agent.summon_cancelled", "agent", id.String())
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{"ok": "true", "status": store.AgentStatusSummonFailed})
|
||||
}
|
||||
|
||||
// writeJSON moved to response_helpers.go
|
||||
|
||||
@@ -73,6 +73,8 @@ func init() {
|
||||
MsgAlreadySummoning: "agent is already being summoned",
|
||||
MsgSummoningUnavailable: "summoning not available",
|
||||
MsgNoDescription: "agent has no description to resummon from",
|
||||
MsgSummonCancelled: "summon cancelled by user",
|
||||
MsgCannotCancel: "agent is not being summoned",
|
||||
MsgInvalidPath: "invalid path",
|
||||
|
||||
// Tenant backup / restore
|
||||
|
||||
@@ -73,6 +73,8 @@ func init() {
|
||||
MsgAlreadySummoning: "agent đang được triệu hồi",
|
||||
MsgSummoningUnavailable: "triệu hồi không khả dụng",
|
||||
MsgNoDescription: "agent không có mô tả để triệu hồi lại",
|
||||
MsgSummonCancelled: "đã huỷ triệu hồi",
|
||||
MsgCannotCancel: "agent không trong trạng thái đang triệu hồi",
|
||||
MsgInvalidPath: "đường dẫn không hợp lệ",
|
||||
|
||||
// Tenant backup / restore
|
||||
|
||||
@@ -73,6 +73,8 @@ func init() {
|
||||
MsgAlreadySummoning: "Agent正在被召唤中",
|
||||
MsgSummoningUnavailable: "召唤功能不可用",
|
||||
MsgNoDescription: "Agent没有可供重新召唤的描述",
|
||||
MsgSummonCancelled: "已取消召唤",
|
||||
MsgCannotCancel: "Agent 未处于召唤状态",
|
||||
MsgInvalidPath: "路径无效",
|
||||
|
||||
// Tenant backup / restore
|
||||
|
||||
@@ -74,6 +74,8 @@ const (
|
||||
MsgAlreadySummoning = "error.already_summoning" // "agent is already being summoned"
|
||||
MsgSummoningUnavailable = "error.summoning_unavailable" // "summoning not available"
|
||||
MsgNoDescription = "error.no_description" // "agent has no description to resummon from"
|
||||
MsgSummonCancelled = "info.summon_cancelled" // "summon cancelled by user"
|
||||
MsgCannotCancel = "error.cannot_cancel_summon" // "agent is not being summoned"
|
||||
MsgInvalidPath = "error.invalid_path" // "invalid path"
|
||||
|
||||
// --- Tenant backup / restore ---
|
||||
|
||||
@@ -608,6 +608,9 @@ type AgentCRUDStore interface {
|
||||
Delete(ctx context.Context, id uuid.UUID) error
|
||||
List(ctx context.Context, ownerID string) ([]AgentData, error)
|
||||
GetDefault(ctx context.Context) (*AgentData, error) // agent with is_default=true, or first available
|
||||
// ResetStuckSummoning flips rows with status='summoning' to 'summon_failed'.
|
||||
// Called at startup to recover from crashes where summon goroutine died mid-flight.
|
||||
ResetStuckSummoning(ctx context.Context) (int64, error)
|
||||
}
|
||||
|
||||
// AgentAccessStore manages agent sharing and access control.
|
||||
|
||||
@@ -609,6 +609,18 @@ func joinStrings(s []string, sep string) string {
|
||||
return result.String()
|
||||
}
|
||||
|
||||
// ResetStuckSummoning flips rows with status='summoning' to 'summon_failed'.
|
||||
// Called at startup to recover from crashes where summon goroutines died mid-flight.
|
||||
func (s *PGAgentStore) ResetStuckSummoning(ctx context.Context) (int64, error) {
|
||||
const q = `UPDATE agents SET status = $1 WHERE status = $2`
|
||||
res, err := s.db.ExecContext(ctx, q, store.AgentStatusSummonFailed, store.AgentStatusSummoning)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func replaceIDX(s, replacement string) string {
|
||||
var result strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
|
||||
@@ -327,3 +327,15 @@ func scanAgentRows(rows *sql.Rows) ([]store.AgentData, error) {
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
// ResetStuckSummoning flips rows with status='summoning' to 'summon_failed'.
|
||||
// Called at startup to recover from crashes where summon goroutines died mid-flight.
|
||||
func (s *SQLiteAgentStore) ResetStuckSummoning(ctx context.Context) (int64, error) {
|
||||
const q = `UPDATE agents SET status = ? WHERE status = ?`
|
||||
res, err := s.db.ExecContext(ctx, q, store.AgentStatusSummonFailed, store.AgentStatusSummoning)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
return n, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user