From 649b2754dce9eb18d24ee5f95839984745d50878 Mon Sep 17 00:00:00 2001 From: WangLeo <690854599@qq.com> Date: Tue, 21 Apr 2026 16:31:47 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96comfyui=E7=9A=84=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E6=B6=88=E6=81=AF=E5=9B=9E=E8=B0=83=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 1.txt | 227 ++---------------- backend/src/task-queue-client/index.js | 4 +- 任务队列debug.txt | 173 +++++++------ .../worker_threads/process/pollingTask.js | 11 +- 4 files changed, 134 insertions(+), 281 deletions(-) diff --git a/1.txt b/1.txt index 32abc27..1b95ac0 100644 --- a/1.txt +++ b/1.txt @@ -1,202 +1,25 @@ -root@2F-E3-Server6-04:/project/digitalHuman/taskBackend-v3# node app.js -BACKEND_API_URL: http://localhost:8787 -[2026-03-20T09:53:00.778Z] INFO: ***************初始化队列开始*************** -[2026-03-20T09:53:00.779Z] INFO: 开始初始化队列... -[2026-03-20T09:53:00.779Z] DEBUG: 等待队列创建成功: digitalHuman-test:comfyui:wait -[2026-03-20T09:53:00.779Z] DEBUG: 等待队列创建成功: digitalHuman-test:coze:wait -Redis 连接成功 -[2026-03-20T09:53:00.791Z] INFO: Redis已存在初始化信息，检查并更新配置 -[2026-03-20T09:53:00.793Z] DEBUG: 已更新平台 digitalHuman-test:comfyui 配置：MAX_CONCURRENT=0 -[2026-03-20T09:53:00.794Z] DEBUG: 已更新平台 digitalHuman-test:coze 配置：MAX_CONCURRENT=20 -[2026-03-20T09:53:00.794Z] INFO: 已更新各平台配置并清零WQtasks计数器 -[2026-03-20T09:53:00.794Z] INFO: 队列初始化完成 -[2026-03-20T09:53:00.794Z] INFO: ***************初始化队列完成*************** -======================================== -HTTP & WebSocket 服务器已启动 -服务地址: http://0.0.0.0:8087/ -======================================== -[2026-03-20T09:53:00.799Z] INFO: WebSocket server is running on the same port as HTTP server -[2026-03-20T09:53:01.207Z] [CallbackTimeout] INFO: 回调超时检测线程启动 -[2026-03-20T09:53:01.208Z] [CallbackTimeout] INFO: 回调超时时间: 3600秒 -Redis 连接成功 -[2026-03-20T09:53:01.221Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 25 -Redis 连接成功 -[2026-03-20T09:53:01.222Z] [CallbackResult] INFO: 回调结果处理线程启动 -Redis 连接成功 -Redis 连接成功 -[2026-03-20T09:53:01.232Z] INFO: 错误队列有可处理任务，数量: -Redis 连接成功 -[2026-03-20T09:53:01.234Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.235Z] DEBUG: 获取到的平台信息： -[2026-03-20T09:53:01.235Z] INFO: [process.js] 轮询线程启动 -[MDWebSocketServer] 从 Redis 读取算力信息: 6 -[2026-03-20T09:53:01.236Z] INFO: 错误队列有可处理任务，数量: -[MDWebSocketServer] 从 Redis 读取 JWT Token: 存在 -[2026-03-20T09:53:01.237Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.237Z] INFO: [judgConcurrency] 内部可用算力: 6, JWT: 存在 -[2026-03-20T09:53:01.238Z] DEBUG: [judgConcurrency] comfyui 平台总并发: 6 (外部0 + 内部6) -[2026-03-20T09:53:01.238Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.239Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.240Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.240Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.241Z] [CallbackResult] INFO: 回调结果队列有任务可处理，数量: 10 -[2026-03-20T09:53:01.241Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.242Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.243Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.243Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.242Z] [CallbackResult] DEBUG: 回调结果队列任务ID: ec666932-dc3d-4e70-b49c-c614878b7ffa, 10bc6ed3-fe85-4508-bf62-681ce477702a, 4b39e839-2059-4cd7-9c6f-203fe6762523, 3470b534-97e3-4a47-8b3b-3cf8e3e5ca87, faee26ba-07e6-415d-8bf4-7a2eb0541df9... -[2026-03-20T09:53:01.244Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.244Z] DEBUG: 错误队列任务ID: -Redis 连接成功 -[2026-03-20T09:53:01.245Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.245Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"ec666932-dc3d-4e70-b49c-c614878b7ffa\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.245Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.246Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"10bc6ed3-fe85-4508-bf62-681ce477702a\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.246Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.246Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"4b39e839-2059-4cd7-9c6f-203fe6762523\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.246Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.246Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"3470b534-97e3-4a47-8b3b-3cf8e3e5ca87\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.246Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.247Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"faee26ba-07e6-415d-8bf4-7a2eb0541df9\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.247Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.247Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"ec666932-dc3d-4e70-b49c-c614878b7ffa\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.247Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.247Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"10bc6ed3-fe85-4508-bf62-681ce477702a\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.247Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.247Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"faee26ba-07e6-415d-8bf4-7a2eb0541df9\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.247Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.248Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"4b39e839-2059-4cd7-9c6f-203fe6762523\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.248Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.248Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman2","message":"{\"taskId\":\"3470b534-97e3-4a47-8b3b-3cf8e3e5ca87\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[]}\"}"} -[2026-03-20T09:53:01.248Z] DEBUG: 未找到目标客户端或连接已关闭，保存消息到待发送队列，id: digitalHuman2 -[2026-03-20T09:53:01.245Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: ec666932-dc3d-4e70-b49c-c614878b7ffa -[2026-03-20T09:53:01.247Z] WARN: [CapacityManager] 检测到负值: digitalHuman-test:comfyui PQtasks = -6, 已修正为 0 -[2026-03-20T09:53:01.246Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.246Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.247Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.247Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.247Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.248Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 10bc6ed3-fe85-4508-bf62-681ce477702a -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 4b39e839-2059-4cd7-9c6f-203fe6762523 -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 3470b534-97e3-4a47-8b3b-3cf8e3e5ca87 -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: faee26ba-07e6-415d-8bf4-7a2eb0541df9 -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: ec666932-dc3d-4e70-b49c-c614878b7ffa -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 10bc6ed3-fe85-4508-bf62-681ce477702a -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: faee26ba-07e6-415d-8bf4-7a2eb0541df9 -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 4b39e839-2059-4cd7-9c6f-203fe6762523 -[2026-03-20T09:53:01.245Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 3470b534-97e3-4a47-8b3b-3cf8e3e5ca87 -[2026-03-20T09:53:01.247Z] DEBUG: [CapacityManager] digitalHuman-test:comfyui PQtasks: 4 -> 0 -[2026-03-20T09:53:01.247Z] DEBUG: 减少相关平台处理队列: AIGC=digitalHuman-test, Platform=comfyui, Count=10 -[2026-03-20T09:53:01.248Z] DEBUG: 减少回调队列任务数: 10 -[2026-03-20T09:53:01.248Z] [CallbackResult] INFO: 已处理10个回调结果任务，移除0个孤立任务 -[2026-03-20T09:53:01.248Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.249Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.249Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381246 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381246 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381246 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381247 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381247 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381247 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381247 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381247 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381248 -[2026-03-20T09:53:01.250Z] DEBUG: 保存待发送消息: backendId=digitalHuman2, messageKey=digitalHuman-test:pending:messages:digitalHuman2:1774000381248 -[2026-03-20T09:53:01.250Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.251Z] INFO: 消息已保存到待发送队列，等待重试: backendId=digitalHuman2 -[2026-03-20T09:53:01.249Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.250Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.251Z] DEBUG: [getSupportedPlatforms] 找到轮询队列键: [] -[2026-03-20T09:53:01.250Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.251Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.251Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.252Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.251Z] DEBUG: [getSupportedPlatforms] 轮询队列为空，从任务数据中获取平台信息 -[2026-03-20T09:53:01.252Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.252Z] DEBUG: [getSupportedPlatforms] 找到任务键: 30 个 -Redis 连接成功 -[2026-03-20T09:53:01.253Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.253Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.254Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.254Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.254Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.255Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.255Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.255Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.256Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.256Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.257Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.257Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.258Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.258Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.258Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.259Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.259Z] INFO: [getSupportedPlatforms] 最终支持的平台组合: [] -[2026-03-20T09:53:01.259Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.259Z] INFO: 支持的平台组合: -[2026-03-20T09:53:01.260Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.260Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.260Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.261Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.261Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.262Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.262Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.263Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.263Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.263Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.264Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.264Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.265Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.265Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.265Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.266Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.266Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.267Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.267Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.268Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.268Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.268Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.269Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.269Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.269Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.270Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.270Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.271Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.271Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.271Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.272Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.272Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.273Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.273Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.273Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.274Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.274Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.275Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.275Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.276Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.276Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.276Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.277Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.277Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.277Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.278Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.278Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.279Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.279Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.280Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.280Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.280Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.281Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.281Z] INFO: 错误队列有可处理任务，数量: -[2026-03-20T09:53:01.281Z] DEBUG: 错误队列任务ID: -[2026-03-20T09:53:01.282Z] INFO: 错误队列有可处理任务，数量: +2026-04-21 12:32:04 +08:00: 2026-04-21 12:32:04 [info]: [WebSocketClient] executing 消息: node=20, prompt_id=5c665958-abdf-4c6e-829f-eff14c667c3b +2026-04-21 12:32:04 +08:00: 2026-04-21 12:32:04 [info]: [WebSocketClient] 收到消息 from server-1-8194: type=executing, data={"node":"24","display_node":"24","prompt_id":"5c665958-abdf-4c6e-829f-eff14c667c3b"} +2026-04-21 12:32:04 +08:00: 2026-04-21 12:32:04 [info]: [WebSocketClient] executing 消息: node=24, prompt_id=5c665958-abdf-4c6e-829f-eff14c667c3b +2026-04-21 12:34:23 +08:00: 2026-04-21 12:34:23 [info]: [WebSocketClient] 收到消息 from server-1-8190: type=executing, data={"node":"55","display_node":"55","prompt_id":"e511838f-46b5-47a7-80b1-2168598bf4b1"} +2026-04-21 12:34:23 +08:00: 2026-04-21 12:34:23 [info]: [WebSocketClient] executing 消息: node=55, prompt_id=e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:34:28 +08:00: 2026-04-21 12:34:28 [info]: [WebSocketClient] 收到消息 from server-1-8190: type=executing, data={"node":"127","display_node":"127","prompt_id":"e511838f-46b5-47a7-80b1-2168598bf4b1"} +2026-04-21 12:34:28 +08:00: 2026-04-21 12:34:28 [info]: [WebSocketClient] executing 消息: node=127, prompt_id=e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:34:28 +08:00: 2026-04-21 12:34:28 [info]: [WebSocketClient] 收到消息 from server-1-8190: type=executing, data={"node":"126","display_node":"126","prompt_id":"e511838f-46b5-47a7-80b1-2168598bf4b1"} +2026-04-21 12:34:28 +08:00: 2026-04-21 12:34:28 [info]: [WebSocketClient] executing 消息: node=126, prompt_id=e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:34:28 +08:00: 2026-04-21 12:34:28 [info]: [WebSocketClient] 收到消息 from server-1-8190: type=executing, data={"node":"54","display_node":"54","prompt_id":"e511838f-46b5-47a7-80b1-2168598bf4b1"} +2026-04-21 12:34:28 +08:00: 2026-04-21 12:34:28 [info]: [WebSocketClient] executing 消息: node=54, prompt_id=e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: [WebSocketClient] 收到消息 from server-1-8190: type=execution_error, data={"prompt_id":"e511838f-46b5-47a7-80b1-2168598bf4b1","node_id":"54","node_type":"LatentSyncNode","executed":["55","122","132","130","128","127","129","126","131"],"exception_message":"Face not detected +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [error]: [WebSocketClient] execution_error 消息: {"prompt_id":"e511838f-46b5-47a7-80b1-2168598bf4b1","node_id":"54","node_type":"LatentSyncNode","executed":["55","122","132","130","128","127","129","126","131"],"exception_message":"Face not detected\n","exception_type":"RuntimeError","traceback":[" File \"/root/ComfyUI/execution.py\", line 524, in execute\n output_data, output_ui, has_subgraph, has_pending_tasks = await get_output_data(prompt_id, unique_id, obj, input_data_all, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb, v3_data=v3_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"," File \"/root/ComfyUI/execution.py\", line 333, in get_output_data\n return_values = await _async_map_node_over_list(prompt_id, unique_id, obj, input_data_all, obj.FUNCTION, allow_interrupt=True, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb, v3_data=v3_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"," File \"/root/ComfyUI/execution.py\", line 307, in _async_map_node_over_list\n await process_inputs(input_dict, i)\n"," File \"/root/ComfyUI/execution.py\", line 295, in process_inputs\n result = f(**inputs)\n ^^^^^^^^^^^\n"," File \"/root/ComfyUI/custom_nodes/ComfyUI-LatentSyncWrapper/nodes.py\", line 672, in inference\n inference_module.main(config, args)\n"," File \"/root/ComfyUI/custom_nodes/ComfyUI-LatentSyncWrapper/scripts/inference.py\", line 148, in main\n pipeline(\n"," File \"/opt/miniconda3/lib/python3.12/site-packages/torch/utils/_contextlib.py\", line 124, in decorate_context\n return func(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^\n"," File \"/root/ComfyUI/custom_nodes/ComfyUI-LatentSyncWrapper/latentsync/pipelines/lipsync_pipeline.py\", line 373, in __call__\n video_frames, faces, boxes, affine_matrices = self.loop_video(whisper_chunks, video_frames)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"," File \"/root/ComfyUI/custom_nodes/ComfyUI-LatentSyncWrapper/latentsync/pipelines/lipsync_pipeline.py\", line 286, in loop_video\n faces, boxes, affine_matrices = self.affine_transform_video(video_frames)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"," File \"/root/ComfyUI/custom_nodes/ComfyUI-LatentSyncWrapper/latentsync/pipelines/lipsync_pipeline.py\", line 260, in affine_transform_video\n face, box, affine_matrix = self.image_processor.affine_transform(frame)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"," File \"/root/ComfyUI/custom_nodes/ComfyUI-LatentSyncWrapper/latentsync/utils/image_processor.py\", line 59, in affine_transform\n raise RuntimeError(\"Face not detected\")\n"],"current_inputs":{"seed":[507990733],"lips_expression":[1.5],"inference_steps":[25],"images":["tensor([[[[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9608, 0.9490, 0.9804],\n [0.9569, 0.9451, 0.9725],\n [0.9569, 0.9451, 0.9686]],\n\n ...,\n\n [[0.6627, 0.5333, 0.5176],\n [0.6588, 0.5294, 0.5137],\n [0.6510, 0.5216, 0.5059],\n ...,\n [0.6392, 0.5137, 0.4824],\n [0.6353, 0.5098, 0.4784],\n [0.6275, 0.5020, 0.4706]],\n\n [[0.6549, 0.5255, 0.5098],\n [0.6549, 0.5255, 0.5098],\n [0.6471, 0.5216, 0.5059],\n ...,\n [0.6392, 0.5137, 0.4824],\n [0.6392, 0.5137, 0.4824],\n [0.6353, 0.5098, 0.4784]],\n\n [[0.6353, 0.5059, 0.4902],\n [0.6353, 0.5059, 0.4902],\n [0.6353, 0.5098, 0.4941],\n ...,\n [0.6510, 0.5255, 0.4941],\n [0.6471, 0.5216, 0.4902],\n [0.6392, 0.5137, 0.4824]]],\n\n\n [[[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9608, 0.9490, 0.9804],\n [0.9569, 0.9451, 0.9725],\n [0.9569, 0.9451, 0.9686]],\n\n ...,\n\n [[0.6549, 0.5216, 0.5176],\n [0.6588, 0.5255, 0.5176],\n [0.6588, 0.5294, 0.5098],\n ...,\n [0.6392, 0.5137, 0.4824],\n [0.6353, 0.5098, 0.4784],\n [0.6275, 0.5020, 0.4706]],\n\n [[0.6510, 0.5176, 0.5137],\n [0.6471, 0.5137, 0.5098],\n [0.6549, 0.5255, 0.5059],\n ...,\n [0.6392, 0.5137, 0.4824],\n [0.6392, 0.5137, 0.4824],\n [0.6353, 0.5098, 0.4784]],\n\n [[0.6353, 0.5020, 0.4980],\n [0.6392, 0.5059, 0.5020],\n [0.6431, 0.5137, 0.4980],\n ...,\n [0.6510, 0.5255, 0.4941],\n [0.6471, 0.5216, 0.4902],\n [0.6392, 0.5137, 0.4824]]],\n\n\n [[[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9608, 0.9490, 0.9804],\n [0.9569, 0.9451, 0.9725],\n [0.9569, 0.9451, 0.9686]],\n\n ...,\n\n [[0.6627, 0.5333, 0.5176],\n [0.6510, 0.5216, 0.5059],\n [0.6471, 0.5176, 0.5020],\n ...,\n [0.6392, 0.5137, 0.4824],\n [0.6353, 0.5098, 0.4784],\n [0.6275, 0.5020, 0.4706]],\n\n [[0.6588, 0.5294, 0.5137],\n [0.6471, 0.5176, 0.5020],\n [0.6471, 0.5176, 0.5020],\n ...,\n [0.6392, 0.5137, 0.4824],\n [0.6392, 0.5137, 0.4824],\n [0.6353, 0.5098, 0.4784]],\n\n [[0.6471, 0.5176, 0.5020],\n [0.6431, 0.5137, 0.4980],\n [0.6431, 0.5137, 0.4980],\n ...,\n [0.6510, 0.5255, 0.4941],\n [0.6471, 0.5216, 0.4902],\n [0.6392, 0.5137, 0.4824]]],\n\n\n ...,\n\n\n [[[0.8392, 0.8275, 0.8667],\n [0.8392, 0.8275, 0.8667],\n [0.8392, 0.8275, 0.8667],\n ...,\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8392, 0.8275, 0.8667],\n [0.8392, 0.8275, 0.8667],\n [0.8392, 0.8275, 0.8667],\n ...,\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8392, 0.8275, 0.8667],\n [0.8392, 0.8275, 0.8667],\n [0.8392, 0.8275, 0.8667],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n ...,\n\n [[0.9333, 0.8941, 0.9176],\n [0.8510, 0.8039, 0.8275],\n [0.7490, 0.6980, 0.7098],\n ...,\n [0.6353, 0.5098, 0.4706],\n [0.6353, 0.5098, 0.4706],\n [0.6314, 0.5059, 0.4667]],\n\n [[0.6980, 0.6157, 0.6392],\n [0.6196, 0.5412, 0.5608],\n [0.5686, 0.4824, 0.4941],\n ...,\n [0.6353, 0.5176, 0.4784],\n [0.6314, 0.5176, 0.4745],\n [0.6275, 0.5137, 0.4706]],\n\n [[0.5451, 0.4549, 0.4784],\n [0.5451, 0.4510, 0.4745],\n [0.5686, 0.4667, 0.4784],\n ...,\n [0.6392, 0.5255, 0.4824],\n [0.6353, 0.5216, 0.4784],\n [0.6314, 0.5176, 0.4745]]],\n\n\n [[[0.8353, 0.8314, 0.8667],\n [0.8353, 0.8314, 0.8667],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8353, 0.8314, 0.8667],\n [0.8353, 0.8314, 0.8667],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8353, 0.8314, 0.8667],\n [0.8353, 0.8314, 0.8667],\n [0.8431, 0.8314, 0.8706],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9765]],\n\n ...,\n\n [[0.9333, 0.8941, 0.9098],\n [0.8549, 0.8078, 0.8157],\n [0.7412, 0.6784, 0.6824],\n ...,\n [0.6353, 0.5098, 0.4706],\n [0.6275, 0.5137, 0.4706],\n [0.6235, 0.5098, 0.4667]],\n\n [[0.7020, 0.6235, 0.6353],\n [0.6314, 0.5451, 0.5529],\n [0.5725, 0.4745, 0.4784],\n ...,\n [0.6353, 0.5176, 0.4784],\n [0.6314, 0.5176, 0.4745],\n [0.6275, 0.5137, 0.4706]],\n\n [[0.5490, 0.4549, 0.4667],\n [0.5529, 0.4588, 0.4706],\n [0.5882, 0.4706, 0.4824],\n ...,\n [0.6392, 0.5255, 0.4824],\n [0.6353, 0.5216, 0.4784],\n [0.6314, 0.5176, 0.4745]]],\n\n\n [[[0.8353, 0.8314, 0.8667],\n [0.8353, 0.8314, 0.8667],\n [0.8392, 0.8275, 0.8667],\n ...,\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8353, 0.8314, 0.8667],\n [0.8353, 0.8314, 0.8667],\n [0.8392, 0.8275, 0.8667],\n ...,\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686],\n [0.9569, 0.9451, 0.9686]],\n\n [[0.8353, 0.8314, 0.8667],\n [0.8353, 0.8314, 0.8667],\n [0.8392, 0.8275, 0.8667],\n ...,\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9765],\n [0.9569, 0.9451, 0.9765]],\n\n ...,\n\n [[0.9373, 0.8980, 0.9098],\n [0.8510, 0.8039, 0.8118],\n [0.7373, 0.6745, 0.6784],\n ...,\n [0.6353, 0.5098, 0.4706],\n [0.6275, 0.5137, 0.4706],\n [0.6235, 0.5098, 0.4667]],\n\n [[0.6902, 0.6078, 0.6196],\n [0.6235, 0.5412, 0.5529],\n [0.5843, 0.4824, 0.4902],\n ...,\n [0.6314, 0.5176, 0.4745],\n [0.6314, 0.5176, 0.4745],\n [0.6275, 0.5137, 0.4706]],\n\n [[0.5529, 0.4588, 0.4706],\n [0.5529, 0.4588, 0.4706],\n [0.5882, 0.4745, 0.4863],\n ...,\n [0.6392, 0.5255, 0.4824],\n [0.6353, 0.5216, 0.4784],\n [0.6314, 0.5176, 0.4745]]]])"],"audio":["{'waveform': tensor([[[0.0009, 0.0009, 0.0007, ..., 0.0000, 0.0000, 0.0000]]]), 'sample_rate': 22050}"]},"current_outputs":["55","122","132","130","54","128","127","129","126","81","131"],"timestamp":1776746120290} +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [error]: [TaskForwarder] 收到 execution_error 事件: instanceId=server-1-8190, promptId=e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [error]: [TaskForwarder] handleExecutionError: instanceId=server-1-8190, promptId=e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [error]: 任务 e511838f-46b5-47a7-80b1-2168598bf4b1 执行失败: Face not detected +2026-04-21 12:35:20 +08:00: +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: 发送Webhook回调到: https://www.whjbjm.com/taskCallback/callback/all +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: [WebSocketClient] 收到消息 from server-1-8190: type=status, data={"status":{"exec_info":{"queue_remaining":0}}} +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: [WebSocketClient] status 消息: {"status":{"exec_info":{"queue_remaining":0}}} +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: [WebSocketClient] 收到消息 from server-1-8190: type=executing, data={"node":null,"prompt_id":"e511838f-46b5-47a7-80b1-2168598bf4b1"} +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: [WebSocketClient] executing 消息: node=null, prompt_id=e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: Webhook回调发送成功: e511838f-46b5-47a7-80b1-2168598bf4b1 +2026-04-21 12:35:20 +08:00: 2026-04-21 12:35:20 [info]: [MessageDispatcher] 发送回调到：https://www.whjbjm.com/taskCallback/callback/all +2026-04-21 12:35:21 +08:00: 2026-04-21 12:35:21 [info]: [MessageDispatcher] 回调发送成功：e511838f-46b5-47a7-80b1-2168598bf4b1 \ No newline at end of file diff --git a/backend/src/task-queue-client/index.js b/backend/src/task-queue-client/index.js index 581c01d..4d8c877 100644 --- a/backend/src/task-queue-client/index.js +++ b/backend/src/task-queue-client/index.js @@ -364,8 +364,8 @@ class MessageDispatcherClient extends EventEmitter { this.pendingTasks.delete(taskId); } - notifyTaskComplete(taskId, result) { - this.sendTaskEndCallback(taskId, result).catch(err => { + notifyTaskComplete(taskId, result, error = null) { + this.sendTaskEndCallback(taskId, result, error).catch(err => { logger.error('[MessageDispatcher] 处理任务完成通知失败:', err); }); } diff --git a/任务队列debug.txt b/任务队列debug.txt index c18d9ca..2d6ce81 100644 --- a/任务队列debug.txt +++ b/任务队列debug.txt @@ -1,74 +1,101 @@ -9|digitalH | 2026-04-17 09:31:04 +08:00: [2026-04-17T01:31:04.485Z] INFO: [getSupportedPlatforms] 最终支持的平台组合: [] -9|digitalH | 2026-04-17 09:31:17 +08:00: [2026-04-17T01:31:17.407Z] INFO: [waiting] 内部算力状态变更: 容量=7, JWT=存在 -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.687Z] [CallbackResult] INFO: 回调结果队列有任务可处理，数量: 2 -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.687Z] [CallbackResult] DEBUG: 回调结果队列任务ID: c050dc69-2cb6-46ca-a0f7-793659bd6f8b, c050dc69-2cb6-46ca-a0f7-793659bd6f8b -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman","message":"{\"taskId\":\"c050dc69-2cb6-46ca-a0f7-793659bd6f8b\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[{\\\"fileUrl\\\":\\\"https://www.whjbjm.com/api/file/internal/generated/1776389469957_2641d986-43f3-4cf3-bbc9-4a3d9e0e4d93_AnimateDiff_01464-audio.mp4\\\",\\\"fileType\\\":\\\"output\\\",\\\"taskCostTime\\\":0,\\\"nodeId\\\":\\\"81\\\"}]}\"}"} -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] DEBUG: 成功发送消息到客户端，id: digitalHuman, 消息: {"taskId":"c050dc69-2cb6-46ca-a0f7-793659bd6f8b","... -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman","message":"{\"taskId\":\"c050dc69-2cb6-46ca-a0f7-793659bd6f8b\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[{\\\"fileUrl\\\":\\\"https://www.whjbjm.com/api/file/internal/generated/1776389469957_2641d986-43f3-4cf3-bbc9-4a3d9e0e4d93_AnimateDiff_01464-audio.mp4\\\",\\\"fileType\\\":\\\"output\\\",\\\"taskCostTime\\\":0,\\\"nodeId\\\":\\\"81\\\"}]}\"}"} -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] DEBUG: 成功发送消息到客户端，id: digitalHuman, 消息: {"taskId":"c050dc69-2cb6-46ca-a0f7-793659bd6f8b","... -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: c050dc69-2cb6-46ca-a0f7-793659bd6f8b -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: c050dc69-2cb6-46ca-a0f7-793659bd6f8b -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] DEBUG: [CapacityManager] digitalHuman-v3:comfyui PQtasks: 0 -> 0 -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.688Z] DEBUG: 减少相关平台处理队列: AIGC=digitalHuman-v3, Platform=comfyui, Count=2 -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.689Z] DEBUG: 减少回调队列任务数: 2 -9|digitalH | 2026-04-17 09:31:18 +08:00: [2026-04-17T01:31:18.689Z] [CallbackResult] INFO: 已处理2个回调结果任务，移除0个孤立任务 -9|digitalH | 2026-04-17 09:31:31 +08:00: [2026-04-17T01:31:31.974Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 1 +root@2F-E3-Server6-04:/project/digitalHuman/taskBackend-v3# node checkQueue -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:01 +08:00: [2026-04-17T01:32:01.995Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:03 +08:00: [2026-04-17T01:32:03.607Z] INFO: 任务已加入排队队列：231eac28-25de-4890-b068-f7d63cdb60f0，并设置了2小时过期时间 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:03 +08:00: [2026-04-17T01:32:03.607Z] DEBUG: 增加平台等待队列任务数: AIGC=digitalHuman-v3, Platform=runninghub, Count=1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:04 +08:00: [2026-04-17T01:32:04.533Z] DEBUG: [process.js] 检查新增的平台组合 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:04 +08:00: [2026-04-17T01:32:04.533Z] DEBUG: [getSupportedPlatforms] 找到轮询队列键: [] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:04 +08:00: [2026-04-17T01:32:04.533Z] DEBUG: [getSupportedPlatforms] 轮询队列为空，从任务数据中获取平台信息 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:04 +08:00: [2026-04-17T01:32:04.534Z] DEBUG: [getSupportedPlatforms] 找到任务键: 9 个 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:04 +08:00: [2026-04-17T01:32:04.535Z] INFO: [getSupportedPlatforms] 最终支持的平台组合: [] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:17 +08:00: [2026-04-17T01:32:17.433Z] DEBUG: 增加相关平台处理队列: AIGC=digitalHuman-v3, Platform=runninghub, Count=1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:32 +08:00: [2026-04-17T01:32:32.016Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:46 +08:00: [2026-04-17T01:32:46.896Z] INFO: 任务已加入排队队列：a6332e95-b4d2-4862-981f-901112efeda8，并设置了2小时过期时间 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:46 +08:00: [2026-04-17T01:32:46.896Z] DEBUG: 增加平台等待队列任务数: AIGC=digitalHuman-v3, Platform=comfyui, Count=1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:47 +08:00: [2026-04-17T01:32:47.446Z] DEBUG: 增加相关平台处理队列: AIGC=digitalHuman-v3, Platform=comfyui, Count=1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:47 +08:00: [2026-04-17T01:32:47.447Z] INFO: [waiting] 开始处理 1 个任务 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:47 +08:00: [externalPostRequest] 任务提交成功: taskId=a6332e95-b4d2-4862-981f-901112efeda8, externalTaskId=[object Object] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:47 +08:00: [generatTask] 任务映射: taskId=a6332e95-b4d2-4862-981f-901112efeda8, externalTaskId=2d7af7ff-2c91-4752-a106-3d80f1923679, platform=comfyui -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:47 +08:00: [2026-04-17T01:32:47.464Z] DEBUG: 添加回调等待任务: remoteTaskId=2d7af7ff-2c91-4752-a106-3d80f1923679, taskId=a6332e95-b4d2-4862-981f-901112efeda8 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:47 +08:00: [2026-04-17T01:32:47.465Z] INFO: [waiting] 任务批次处理完成 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:32:47 +08:00: [generatTask] 批次处理完成: 成功=1 个平台, 错误=0 个 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:02 +08:00: [2026-04-17T01:33:02.037Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 2 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:02 +08:00: [2026-04-17T01:33:02.454Z] INFO: [waiting] 内部算力状态变更: 容量=6, JWT=存在 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:04 +08:00: [2026-04-17T01:33:04.583Z] DEBUG: [process.js] 检查新增的平台组合 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:04 +08:00: [2026-04-17T01:33:04.583Z] DEBUG: [getSupportedPlatforms] 找到轮询队列键: [] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:04 +08:00: [2026-04-17T01:33:04.583Z] DEBUG: [getSupportedPlatforms] 轮询队列为空，从任务数据中获取平台信息 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:04 +08:00: [2026-04-17T01:33:04.584Z] DEBUG: [getSupportedPlatforms] 找到任务键: 10 个 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:04 +08:00: [2026-04-17T01:33:04.585Z] INFO: [getSupportedPlatforms] 最终支持的平台组合: [] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.792Z] [CallbackResult] INFO: 回调结果队列有任务可处理，数量: 2 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.792Z] [CallbackResult] DEBUG: 回调结果队列任务ID: 7e843ac9-b266-488c-8fb0-3305638ac253, 7e843ac9-b266-488c-8fb0-3305638ac253 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman","message":"{\"taskId\":\"7e843ac9-b266-488c-8fb0-3305638ac253\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[{\\\"fileUrl\\\":\\\"https://www.whjbjm.com/api/file/internal/generated/1776389586505_66091ab4-1c7b-458b-9844-f0a9001c4472_AnimateDiff_01463-audio.mp4\\\",\\\"fileType\\\":\\\"output\\\",\\\"taskCostTime\\\":0,\\\"nodeId\\\":\\\"80\\\"}]}\"}"} -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] DEBUG: 成功发送消息到客户端，id: digitalHuman, 消息: {"taskId":"7e843ac9-b266-488c-8fb0-3305638ac253","... -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] DEBUG: 收到callback_result worker消息: {"type":"success","backendId":"digitalHuman","message":"{\"taskId\":\"7e843ac9-b266-488c-8fb0-3305638ac253\",\"result\":\"{\\\"code\\\":0,\\\"msg\\\":\\\"success\\\",\\\"data\\\":[{\\\"fileUrl\\\":\\\"https://www.whjbjm.com/api/file/internal/generated/1776389586505_66091ab4-1c7b-458b-9844-f0a9001c4472_AnimateDiff_01463-audio.mp4\\\",\\\"fileType\\\":\\\"output\\\",\\\"taskCostTime\\\":0,\\\"nodeId\\\":\\\"80\\\"}]}\"}"} -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] DEBUG: 成功发送消息到客户端，id: digitalHuman, 消息: {"taskId":"7e843ac9-b266-488c-8fb0-3305638ac253","... -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 7e843ac9-b266-488c-8fb0-3305638ac253 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] [CallbackResult] DEBUG: 成功发送结果给客户端，taskId: 7e843ac9-b266-488c-8fb0-3305638ac253 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] WARN: [CapacityManager] 检测到负值: digitalHuman-v3:comfyui PQtasks = -1, 已修正为 0 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] DEBUG: [CapacityManager] digitalHuman-v3:comfyui PQtasks: 1 -> 0 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.793Z] DEBUG: 减少相关平台处理队列: AIGC=digitalHuman-v3, Platform=comfyui, Count=2 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.794Z] DEBUG: 减少回调队列任务数: 2 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:09 +08:00: [2026-04-17T01:33:09.794Z] [CallbackResult] INFO: 已处理2个回调结果任务，移除0个孤立任务 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:17 +08:00: [2026-04-17T01:33:17.459Z] INFO: [waiting] 内部算力状态变更: 容量=7, JWT=存在 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:33:32 +08:00: [2026-04-17T01:33:32.057Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:02 +08:00: [2026-04-17T01:34:02.077Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:04 +08:00: [2026-04-17T01:34:04.633Z] DEBUG: [process.js] 检查新增的平台组合 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:04 +08:00: [2026-04-17T01:34:04.633Z] DEBUG: [getSupportedPlatforms] 找到轮询队列键: [] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:04 +08:00: [2026-04-17T01:34:04.633Z] DEBUG: [getSupportedPlatforms] 轮询队列为空，从任务数据中获取平台信息 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:04 +08:00: [2026-04-17T01:34:04.634Z] DEBUG: [getSupportedPlatforms] 找到任务键: 10 个 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:04 +08:00: [2026-04-17T01:34:04.635Z] INFO: [getSupportedPlatforms] 最终支持的平台组合: [] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:20 +08:00: [2026-04-17T01:34:20.098Z] INFO: 任务已加入排队队列：4e04b7da-dfb6-4366-bb89-ef841b0ae026，并设置了2小时过期时间 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:20 +08:00: [2026-04-17T01:34:20.099Z] DEBUG: 增加平台等待队列任务数: AIGC=digitalHuman-v3, Platform=comfyui, Count=1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [2026-04-17T01:34:32.098Z] [CallbackTimeout] DEBUG: 当前等待回调的任务数: 1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [2026-04-17T01:34:32.492Z] DEBUG: 增加相关平台处理队列: AIGC=digitalHuman-v3, Platform=comfyui, Count=1 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [2026-04-17T01:34:32.492Z] INFO: [waiting] 开始处理 1 个任务 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [externalPostRequest] 任务提交成功: taskId=4e04b7da-dfb6-4366-bb89-ef841b0ae026, externalTaskId=[object Object] -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [generatTask] 任务映射: taskId=4e04b7da-dfb6-4366-bb89-ef841b0ae026, externalTaskId=616c91dd-24e9-46cf-804a-c85c01e3552f, platform=comfyui -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [2026-04-17T01:34:32.510Z] DEBUG: 添加回调等待任务: remoteTaskId=616c91dd-24e9-46cf-804a-c85c01e3552f, taskId=4e04b7da-dfb6-4366-bb89-ef841b0ae026 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [2026-04-17T01:34:32.510Z] INFO: [waiting] 任务批次处理完成 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:32 +08:00: [generatTask] 批次处理完成: 成功=1 个平台, 错误=0 个 -9|digitalHuman-websocketTask-v3 | 2026-04-17 09:34:47 +08:00: [2026-04-17T01:34:47.499Z] INFO: [waiting] 内部算力状态变更: 容量=6, JWT=存在 +======================================== + Redis 队列检测脚本 + 项目前缀: digitalHuman-v3 + 检测时间: 2026/4/20 10:48:40 +======================================== + + Redis 连接成功 + + +========== 检测等待队列 ========== + ✓ 等待队列 [digitalHuman-v3:comfyui:wait] 为空 + ✓ 等待队列 [digitalHuman-v3:runninghub:wait] 为空 + ⚠️ 等待队列 [digitalHuman-v3:coze:wait] 有 11 个任务积压 + +========== 检测处理队列 ========== + ✓ 处理队列 [digitalHuman-v3:process:Polling] 为空 + ✓ 回调处理队列 [digitalHuman-v3:process:callback] 为空 + +========== 检测结果队列 ========== + ✓ 结果队列 [digitalHuman-v3:result:queue] 为空 + ✓ 结果列表 [digitalHuman-v3:result:list] 为空 + +========== 检测错误队列 ========== + ✓ 错误队列 [digitalHuman-v3:error:queue] 为空 + ⚠️ 错误列表 [digitalHuman-v3:error:list] 有 11 个任务 + +========== 检测回调等待任务 ========== + 发现 2 个回调等待任务 + ✓ 所有回调等待任务都在正常时间内 + +========== 检测待处理消息 ========== + ✓ 没有待处理消息 + +========== 检测计数器 ========== + +全局计数器: + PQtasksALL (处理队列总任务数): 19 + RQtasksALL (结果队列总任务数): 0 + CQtasksALL (回调队列总任务数): 2 + EQtaskALL (错误队列总任务数): -9 + +平台计数器: + [digitalHuman-v3:comfyui] + WQtasks (等待队列任务数): 0 + PQtasks (处理队列任务数): 2 + MAX_CONCURRENT (最大并发数): 0 + ✗ 平台 [digitalHuman-v3:comfyui] 处理队列任务数 (2) 超过最大并发数 (0) + [digitalHuman-v3:runninghub] + WQtasks (等待队列任务数): 0 + PQtasks (处理队列任务数): 5 + MAX_CONCURRENT (最大并发数): 13 + [digitalHuman-v3:coze] + WQtasks (等待队列任务数): 11 + PQtasks (处理队列任务数): 20 + MAX_CONCURRENT (最大并发数): 20 + ⚠️ 平台 [digitalHuman-v3:coze] 等待队列计数器不为零: 11 + ⚠️ 全局处理队列计数器不为零: 19 + ⚠️ 全局回调队列计数器不为零: 2 + +========== 检测队列与计数器一致性 ========== + ⚠️ PQtasksALL (19) 与实际队列长度 (0) 不一致 + ✓ CQtasksALL 与实际回调等待数一致 + ✓ 平台 [digitalHuman-v3:comfyui] WQtasks 与实际等待队列长度一致 + ✓ 平台 [digitalHuman-v3:runninghub] WQtasks 与实际等待队列长度一致 + ✓ 平台 [digitalHuman-v3:coze] WQtasks 与实际等待队列长度一致 + +======================================== + 检测报告汇总 + ======================================== + +共发现 7 个问题: + + +[wait_queue_backlog] - 1 个 + - 队列: digitalHuman-v3:coze:wait, 数量: 11 + +[error_list_has_tasks] - 1 个 + - 队列: digitalHuman-v3:error:list, 数量: 11 + +[process_counter_exceeds_concurrency] - 1 个 + - 平台: digitalHuman-v3:comfyui, 数量: 2 + +[wait_counter_not_zero] - 1 个 + - 平台: digitalHuman-v3:coze, 数量: 11 + +[global_process_counter_not_zero] - 1 个 + - {"type":"global_process_counter_not_zero","count":19} + +[global_callback_counter_not_zero] - 1 个 + - {"type":"global_callback_counter_not_zero","count":2} + +[counter_queue_mismatch] - 1 个 + - 计数器: PQtasksALL, 计数器值: 19, 实际值: 0 + +========== 建议操作 ========== + 1. 等待队列有积压，建议检查消费者是否正常运行 + 4. 计数器与实际队列不一致，建议重置计数器 + +Redis 连接已关闭 diff --git a/任务队列后端/worker_threads/process/pollingTask.js b/任务队列后端/worker_threads/process/pollingTask.js index cd5aab6..e2fbade 100644 --- a/任务队列后端/worker_threads/process/pollingTask.js +++ b/任务队列后端/worker_threads/process/pollingTask.js @@ -34,7 +34,7 @@ async function storeSuccessTasks(SuccessTasks) { const multi = redis.multi(); for (const task of SuccessTasks) { - const taskId = task.taskid || task.taskId; + const taskId = task.taskId; const remoteTaskId = task.remoteTaskId; const aigc = task.aigc; const platform = task.platform; @@ -79,14 +79,17 @@ async function storeSuccessTasks(SuccessTasks) { await multi.exec(); console.log(`[pollingTask] 已完成Redis批量操作，删除了轮询任务并存储到结果队列`); - // 更新平台计数（使用原子操作） + // 无论成功还是失败任务，都需要减少PQtasks计数器 if (taskCountMap.size > 0) { await initQueue.reducePlatformsProcess(taskCountMap); - console.log(`[pollingTask] 已更新平台计数: ${JSON.stringify(Array.from(taskCountMap.entries()))}`); + console.log(`[pollingTask] 已更新成功任务平台计数: ${JSON.stringify(Array.from(taskCountMap.entries()))}`); } - // 更新错误队列计数 + // 失败任务也需要减少PQtasks计数器，因为它们不再处于处理状态 if (taskErrorCountMap.size > 0) { + await initQueue.reducePlatformsProcess(taskErrorCountMap); + console.log(`[pollingTask] 已更新失败任务平台计数: ${JSON.stringify(Array.from(taskErrorCountMap.entries()))}`); + const totalErrorCount = Object.values(taskErrorCountMap).reduce((a, b) => a + b, 0); initQueue.addEQtaskALL(totalErrorCount); console.log(`[pollingTask] 已更新错误队列计数: ${JSON.stringify(Array.from(taskErrorCountMap.entries()))}`);