Compare commits
2 Commits
| Author | SHA1 | Date |
|---|---|---|
 王佑琳
|
649b2754dc | |
|
王佑琳
|
9f8bb07355 |
|
|
@ -0,0 +1,145 @@
|
|||
|
||||
# 日志分析脚本 - 检测僵尸任务
|
||||
|
||||
Write-Host "正在分析日志文件...`n" -ForegroundColor Cyan
|
||||
|
||||
$logFile = "message-dispatcher_debug.txt"
|
||||
|
||||
if (-not (Test-Path $logFile)) {
|
||||
Write-Host "错误: 找不到日志文件 $logFile" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
$lines = Get-Content $logFile
|
||||
$tasks = @{}
|
||||
$completedTasks = @{}
|
||||
$failedTasks = @{}
|
||||
$instanceLocks = @{}
|
||||
|
||||
for ($i = 0; $i -lt $lines.Count; $i++) {
|
||||
$line = $lines[$i].Trim()
|
||||
if (-not $line) { continue }
|
||||
|
||||
# 匹配任务ID
|
||||
if ($line -match "任务(直接分配|调度|确认|完成|失败|超时): ([a-f0-9-]{36})") {
|
||||
$taskId = $matches[2]
|
||||
$eventType = $matches[1]
|
||||
|
||||
if (-not $tasks.ContainsKey($taskId)) {
|
||||
$tasks[$taskId] = @{
|
||||
taskId = $taskId
|
||||
events = @()
|
||||
startTime = $null
|
||||
endTime = $null
|
||||
instanceId = $null
|
||||
completed = $false
|
||||
failed = $false
|
||||
}
|
||||
}
|
||||
|
||||
$tasks[$taskId].events += @{
|
||||
line = $i + 1
|
||||
content = $line
|
||||
}
|
||||
|
||||
if ($line -match "任务完成") {
|
||||
$tasks[$taskId].completed = $true
|
||||
$tasks[$taskId].endTime = $line
|
||||
$completedTasks[$taskId] = $true
|
||||
}
|
||||
elseif ($line -match "任务失败" -or $line -match "任务超时") {
|
||||
$tasks[$taskId].failed = $true
|
||||
$tasks[$taskId].endTime = $line
|
||||
$failedTasks[$taskId] = $true
|
||||
}
|
||||
elseif ($line -match "任务调度" -or $line -match "任务直接分配") {
|
||||
if ($line -match "实例 ([^,\s]+)") {
|
||||
$tasks[$taskId].instanceId = $matches[1]
|
||||
}
|
||||
$tasks[$taskId].startTime = $line
|
||||
}
|
||||
}
|
||||
|
||||
# 匹配实例锁定
|
||||
if ($line -match "实例已锁定: ([^,]+), taskId: ([a-f0-9-]{36})") {
|
||||
$instanceId = $matches[1]
|
||||
$taskId = $matches[2]
|
||||
$instanceLocks[$instanceId] = @{
|
||||
locked = $true
|
||||
taskId = $taskId
|
||||
line = $i + 1
|
||||
}
|
||||
if ($tasks.ContainsKey($taskId)) {
|
||||
$tasks[$taskId].instanceId = $instanceId
|
||||
}
|
||||
}
|
||||
|
||||
# 匹配实例释放
|
||||
if ($line -match "实例锁已释放: ([^,\s]+)") {
|
||||
$instanceId = $matches[1]
|
||||
$instanceLocks[$instanceId] = @{
|
||||
locked = $false
|
||||
line = $i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# 统计
|
||||
$activeTasks = @()
|
||||
$zombieTasks = @()
|
||||
|
||||
foreach ($task in $tasks.Values) {
|
||||
if (-not $task.completed -and -not $task.failed) {
|
||||
$activeTasks += $task
|
||||
if ($task.startTime) {
|
||||
$zombieTasks += $task
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$lockedInstances = @()
|
||||
foreach ($instanceId in $instanceLocks.Keys) {
|
||||
if ($instanceLocks[$instanceId].locked) {
|
||||
$lockedInstances += @{
|
||||
instanceId = $instanceId
|
||||
info = $instanceLocks[$instanceId]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# 输出结果
|
||||
Write-Host "=== 统计结果 ===" -ForegroundColor Yellow
|
||||
Write-Host "总任务数: $($tasks.Count)"
|
||||
Write-Host "已完成任务: $($completedTasks.Count)"
|
||||
Write-Host "失败/超时任务: $($failedTasks.Count)"
|
||||
Write-Host "活跃任务（未完成）: $($activeTasks.Count)"
|
||||
Write-Host "可能的僵尸任务（已开始但未完成）: $($zombieTasks.Count)`n"
|
||||
|
||||
if ($zombieTasks.Count -gt 0) {
|
||||
Write-Host "=== 可能的僵尸任务详情 ===" -ForegroundColor Red
|
||||
foreach ($task in $zombieTasks) {
|
||||
Write-Host "任务ID: $($task.taskId)" -ForegroundColor Red
|
||||
$instanceDisplay = if ($task.instanceId) { $task.instanceId } else { '未知' }
|
||||
Write-Host "实例ID: $instanceDisplay"
|
||||
Write-Host "事件记录:"
|
||||
$recentEvents = $task.events | Select-Object -Last 5
|
||||
foreach ($evt in $recentEvents) {
|
||||
Write-Host " 行$($evt.line): $($evt.content)"
|
||||
}
|
||||
Write-Host ""
|
||||
}
|
||||
} else {
|
||||
Write-Host "✅ 没有发现僵尸任务！" -ForegroundColor Green
|
||||
}
|
||||
|
||||
if ($lockedInstances.Count -gt 0) {
|
||||
Write-Host "`n=== 仍被锁定的实例 ===" -ForegroundColor Red
|
||||
foreach ($item in $lockedInstances) {
|
||||
Write-Host "实例: $($item.instanceId)" -ForegroundColor Red
|
||||
Write-Host "任务ID: $($item.info.taskId)"
|
||||
Write-Host "锁定位置: 行$($item.info.line)"
|
||||
Write-Host ""
|
||||
}
|
||||
} else {
|
||||
Write-Host "`n✅ 所有实例锁都已释放！" -ForegroundColor Green
|
||||
}
|
||||
|
|
@ -9,7 +9,7 @@ ADMIN_PASSWORD=2233..2233
|
|||
|
||||
MESSAGE_DISPATCHER_URL=wss://www.whjbjm.com/message-dispatcher
|
||||
INTERNAL_UPLOAD_URL=http://43.134.182.189:9000/api/internal/uploadGeneratedFile
|
||||
INTERNAL_API_TOKEN=123456/message-dispatcher
|
||||
INTERNAL_API_TOKEN=123456
|
||||
BRIDGE_ID=bridge-1
|
||||
WORKFLOW_RESOURCES_URL=http://117.72.204.159/AIGC/static/public/workflows
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ class ClusterManager {
|
|||
|
||||
/**
|
||||
* 初始化集群管理器
|
||||
* 初始化流程：
|
||||
* 1. 从配置加载所有实例
|
||||
* 2. 立即检查所有实例的健康状态（启动时检查）
|
||||
* 3. 启动定期健康检查定时器
|
||||
*/
|
||||
init() {
|
||||
const initialInstances = config.getAllInstances();
|
||||
|
|
@ -25,8 +29,16 @@ class ClusterManager {
|
|||
this.instances.set(instance.id, instance);
|
||||
}
|
||||
|
||||
logger.info(`集群管理器初始化完成，共 ${this.instances.size} 个实例，开始立即检查所有实例...`);
|
||||
|
||||
// 立即检查所有实例的健康状态
|
||||
this.checkAllInstancesHealth().then(() => {
|
||||
const onlineCount = this.getOnlineInstances().length;
|
||||
logger.info(`初始健康检查完成，在线实例: ${onlineCount}/${this.instances.size}`);
|
||||
});
|
||||
|
||||
// 启动定期健康检查
|
||||
this.startHealthCheck();
|
||||
logger.info(`集群管理器初始化完成，共 ${this.instances.size} 个实例`);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,6 +1,25 @@
|
|||
|
||||
/**
|
||||
* file-uploader模块 - 文件上传处理
|
||||
*
|
||||
* 职责：
|
||||
* 将文件上传到外部文件服务器，返回可访问的URL
|
||||
*
|
||||
* 上传目标服务器配置：
|
||||
* - INTERNAL_UPLOAD_URL: 外部文件服务器API地址
|
||||
* - INTERNAL_API_TOKEN: API认证Token
|
||||
*
|
||||
* 上传协议：
|
||||
* - 使用multipart/form-data格式
|
||||
* - 文件字段名为'file'
|
||||
* - 需要Authorization头携带Bearer Token
|
||||
*
|
||||
* 预期响应格式：
|
||||
* {
|
||||
* code: '0', // 成功状态码
|
||||
* data: {
|
||||
* url: 'https://...' // 上传后的文件访问URL
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
|
||||
import axios from 'axios';
|
||||
|
|
@ -9,50 +28,96 @@ import fs from 'fs';
|
|||
import path from 'path';
|
||||
import logger from '../logger/index.js';
|
||||
|
||||
// 本地临时上传目录（用于存放待上传的文件）
|
||||
const uploadDir = path.resolve(process.cwd(), 'uploads');
|
||||
|
||||
// 外部文件服务器配置（从环境变量读取）
|
||||
const INTERNAL_UPLOAD_URL = process.env.INTERNAL_UPLOAD_URL || 'http://43.134.182.189:9000/api/internal/uploadGeneratedFile';
|
||||
const INTERNAL_API_TOKEN = process.env.INTERNAL_API_TOKEN || '';
|
||||
const INTERNAL_API_TOKEN = process.env.INTERNAL_API_TOKEN || '123456';
|
||||
|
||||
// 确保上传目录存在
|
||||
if (!fs.existsSync(uploadDir)) {
|
||||
fs.mkdirSync(uploadDir, { recursive: true });
|
||||
}
|
||||
|
||||
class FileUploader {
|
||||
/**
|
||||
* 上传文件到外部服务器
|
||||
*
|
||||
* 上传流程：
|
||||
* 1. 创建FormData对象，添加文件流
|
||||
* 2. 设置请求头（Content-Type和Authorization）
|
||||
* 3. 发送POST请求到外部服务器
|
||||
* 4. 解析响应，提取文件URL
|
||||
* 5. 清理本地临时文件（无论上传成功或失败）
|
||||
*
|
||||
* @param {string} filePath - 本地文件路径
|
||||
* @param {string} originalFilename - 原始文件名（用于保持文件名一致性）
|
||||
* @returns {Object} 上传结果对象，包含url字段
|
||||
* @throws {Error} 上传失败时抛出异常
|
||||
*/
|
||||
async uploadToExternalServer(filePath, originalFilename) {
|
||||
// ========== 步骤1：构建FormData ==========
|
||||
// 使用流式读取，避免大文件占用过多内存
|
||||
const formData = new FormData();
|
||||
formData.append('file', fs.createReadStream(filePath), originalFilename);
|
||||
|
||||
// ========== 步骤2：设置请求头 ==========
|
||||
const headers = {
|
||||
'Content-Type': `multipart/form-data; boundary=${formData.getBoundary()}`
|
||||
};
|
||||
|
||||
// 添加认证Token（如果配置了）
|
||||
if (INTERNAL_API_TOKEN) {
|
||||
headers['Authorization'] = `Bearer ${INTERNAL_API_TOKEN}`;
|
||||
}
|
||||
|
||||
try {
|
||||
// ========== 步骤3：打印请求信息并发送上传请求 ==========
|
||||
logger.info(`正在上传文件到外部服务器: ${INTERNAL_UPLOAD_URL}, 文件名: ${originalFilename}`);
|
||||
|
||||
// 打印请求头信息（不含文件内容）
|
||||
logger.info(`[上传请求] URL: ${INTERNAL_UPLOAD_URL}`);
|
||||
logger.info(`[上传请求] Headers: ${JSON.stringify(headers, null, 2)}`);
|
||||
logger.info(`[上传请求] FormData字段: file (文件名: ${originalFilename})`);
|
||||
|
||||
const response = await axios.post(INTERNAL_UPLOAD_URL, formData, {
|
||||
headers,
|
||||
timeout: 60000
|
||||
timeout: 60000 // 60秒超时，适应大文件上传
|
||||
});
|
||||
|
||||
// 打印响应信息
|
||||
logger.info(`[上传响应] Status: ${response.status}`);
|
||||
logger.info(`[上传响应] Data: ${JSON.stringify(response.data, null, 2)}`);
|
||||
|
||||
// ========== 步骤4：解析响应 ==========
|
||||
// 检查响应格式是否正确
|
||||
if (response.data && response.data.code === '0' && response.data.data && response.data.data.url) {
|
||||
logger.info(`文件上传成功: ${response.data.data.url}`);
|
||||
return response.data.data;
|
||||
} else {
|
||||
// 响应格式不符合预期，记录错误
|
||||
logger.error(`文件上传失败: ${JSON.stringify(response.data)}`);
|
||||
throw new Error(response.data?.msg || '文件上传失败');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('文件上传出错:', error.message);
|
||||
// 打印完整错误信息
|
||||
if (error.response) {
|
||||
logger.error(`[上传错误] 响应状态: ${error.response.status}`);
|
||||
logger.error(`[上传错误] 响应数据: ${JSON.stringify(error.response.data, null, 2)}`);
|
||||
logger.error(`[上传错误] 响应头: ${JSON.stringify(error.response.headers, null, 2)}`);
|
||||
} else if (error.request) {
|
||||
logger.error(`[上传错误] 请求已发送但未收到响应`);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
// ========== 步骤5：清理临时文件 ==========
|
||||
// 无论上传成功或失败，都删除本地临时文件
|
||||
// 注意：这里会删除传入的文件，调用方需确保不再需要该文件
|
||||
// if (fs.existsSync(filePath)) {
|
||||
// fs.unlinkSync(filePath);
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class TaskForwarder {
|
|||
}
|
||||
|
||||
async submitTask(workflow, nodeInfoList = [], workflowId = null, instanceId = null, webhookUrl = null, queueTaskId = null) {
|
||||
const taskId = uuidv4();
|
||||
const taskId = queueTaskId || uuidv4();
|
||||
|
||||
let instance;
|
||||
if (instanceId) {
|
||||
|
|
@ -130,8 +130,7 @@ class TaskForwarder {
|
|||
const promptPayload = {
|
||||
prompt: task.workflow,
|
||||
prompt_id: task.id,
|
||||
client_id: wsClientId,
|
||||
front_end: "comfy"
|
||||
client_id: wsClientId
|
||||
};
|
||||
|
||||
logger.info(`[TaskForwarder] 发送的 prompt 消息结构: prompt_id=${task.id}, client_id=${wsClientId}, workflow节点数=${Object.keys(task.workflow || {}).length}`);
|
||||
|
|
@ -236,9 +235,24 @@ class TaskForwarder {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理节点执行完成事件
|
||||
*
|
||||
* 说明：
|
||||
* - 每个节点执行完成后，ComfyUI会发送executed WebSocket消息
|
||||
* - 如果节点有输出（如PreviewImage），会包含output字段
|
||||
* - 这里仅收集部分结果，完整结果在handleExecutionSuccess中处理
|
||||
*
|
||||
* @param {string} instanceId - ComfyUI实例ID
|
||||
* @param {Object} data - WebSocket消息数据
|
||||
* @param {string} data.prompt_id - 任务ID
|
||||
* @param {string} data.node - 节点ID
|
||||
* @param {Object} data.output - 节点输出（可选）
|
||||
*/
|
||||
async handleExecuted(instanceId, data) {
|
||||
logger.info(`[TaskForwarder] handleExecuted: instanceId=${instanceId}, promptId=${data?.prompt_id}, node=${data?.node}`);
|
||||
|
||||
// 检查是否有输出数据
|
||||
if (!data?.output) {
|
||||
logger.info(`[TaskForwarder] executed 消息无输出，跳过处理`);
|
||||
return;
|
||||
|
|
@ -250,9 +264,11 @@ class TaskForwarder {
|
|||
return;
|
||||
}
|
||||
|
||||
// 初始化部分结果数组
|
||||
if (!task.partialResults) {
|
||||
task.partialResults = [];
|
||||
}
|
||||
// 收集当前节点的输出结果
|
||||
task.partialResults.push(data);
|
||||
logger.info(`[TaskForwarder] 收集节点 ${data.node} 的输出结果`);
|
||||
}
|
||||
|
|
@ -282,6 +298,18 @@ class TaskForwarder {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理任务执行成功事件
|
||||
*
|
||||
* 这是文件上传流程的入口点：
|
||||
* 1. 收到ComfyUI的execution_success WebSocket消息后触发
|
||||
* 2. 调用ComfyUI的/history/{prompt_id} API获取任务输出信息
|
||||
* 3. 从history返回的outputs中提取文件信息
|
||||
* 4. 调用processHistoryOutputs处理并上传所有输出文件
|
||||
*
|
||||
* @param {string} instanceId - ComfyUI实例ID
|
||||
* @param {Object} data - WebSocket消息数据，包含prompt_id
|
||||
*/
|
||||
async handleExecutionSuccess(instanceId, data) {
|
||||
logger.info(`[TaskForwarder] handleExecutionSuccess: instanceId=${instanceId}, promptId=${data?.prompt_id}`);
|
||||
|
||||
|
|
@ -306,13 +334,17 @@ class TaskForwarder {
|
|||
}
|
||||
|
||||
try {
|
||||
// ========== 步骤1：调用ComfyUI history API获取任务输出信息 ==========
|
||||
// history API返回格式: { [prompt_id]: { outputs: { [nodeId]: { images: [...] } } } }
|
||||
const historyResponse = await axios.get(`${instance.apiUrl}/history/${promptId}`, {
|
||||
timeout: 10000
|
||||
});
|
||||
|
||||
const historyData = historyResponse.data;
|
||||
logger.info(`[TaskForwarder] 获取到历史记录`);
|
||||
logger.info(`[TaskForwarder] 获取到历史记录: ${JSON.stringify(historyData)}`);
|
||||
|
||||
// ========== 步骤2：从history数据中提取outputs ==========
|
||||
// outputs包含所有输出节点的文件信息
|
||||
let outputs = null;
|
||||
if (historyData && historyData[promptId] && historyData[promptId].outputs) {
|
||||
outputs = historyData[promptId].outputs;
|
||||
|
|
@ -320,9 +352,13 @@ class TaskForwarder {
|
|||
outputs = historyData.outputs;
|
||||
}
|
||||
|
||||
// ========== 步骤3：处理并上传输出文件 ==========
|
||||
if (outputs) {
|
||||
// 调用processHistoryOutputs处理所有输出文件
|
||||
// 这是上传流程的核心方法
|
||||
const resultData = await this.processHistoryOutputs(outputs, instanceId);
|
||||
|
||||
// 更新任务状态为完成
|
||||
task.status = 'completed';
|
||||
task.completedAt = new Date().toISOString();
|
||||
task.result = outputs;
|
||||
|
|
@ -330,10 +366,12 @@ class TaskForwarder {
|
|||
clusterManager.updateInstanceStatus(instanceId, 'online');
|
||||
logger.info(`任务 ${task.id} 执行完成，结果数量: ${resultData.length}`);
|
||||
|
||||
// 发送webhook回调通知调用方
|
||||
if (task.webhookUrl) {
|
||||
await this.sendWebhookCallback(task, { output: outputs }, null);
|
||||
}
|
||||
|
||||
// 通知任务队列客户端任务完成
|
||||
if (task.queueTaskId) {
|
||||
taskQueueClient.notifyTaskComplete(task.queueTaskId, resultData);
|
||||
}
|
||||
|
|
@ -356,6 +394,21 @@ class TaskForwarder {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理历史记录中的输出结果
|
||||
*
|
||||
* 流程说明：
|
||||
* 1. 从ComfyUI的history API获取的outputs中提取所有类型的媒体文件信息
|
||||
* 2. outputs结构: { [nodeId]: { images: [...], gifs: [...], audio: [...], ... } }
|
||||
* 3. 自动检测输出中的所有数组类型字段（images/gifs/audio/text等）
|
||||
* 4. 每个媒体文件包含: filename, subfolder, type 等信息
|
||||
* 5. 遍历所有节点的输出，逐个上传文件到外部服务器
|
||||
* 6. 非文件类型输出（如text, value等）会被跳过，不会作为文件处理
|
||||
*
|
||||
* @param {Object} outputs - ComfyUI history返回的输出对象
|
||||
* @param {string} instanceId - ComfyUI实例ID
|
||||
* @returns {Array} 上传结果数组，包含fileUrl、fileType等信息
|
||||
*/
|
||||
async processHistoryOutputs(outputs, instanceId) {
|
||||
const resultData = [];
|
||||
|
||||
|
|
@ -369,14 +422,66 @@ class TaskForwarder {
|
|||
return resultData;
|
||||
}
|
||||
|
||||
// 定义有效的文件输出类型（这些类型包含filename字段）
|
||||
const FILE_OUTPUT_TYPES = ['images', 'gifs', 'video', 'audio', 'files'];
|
||||
|
||||
// 定义非文件类型的输出（这些是元数据或简单值，不需要上传）
|
||||
const NON_FILE_OUTPUT_TYPES = ['text', 'value', 'string', 'number', 'boolean', 'object'];
|
||||
|
||||
for (const [nodeId, output] of Object.entries(outputs)) {
|
||||
if (!output) continue;
|
||||
|
||||
const mediaFiles = output.images || output.gifs || [];
|
||||
logger.info(`[processHistoryOutputs] 节点 ${nodeId} 找到 ${mediaFiles.length} 个媒体文件`);
|
||||
const mediaFiles = [];
|
||||
const typeCounts = {};
|
||||
const processedTypes = new Set();
|
||||
|
||||
for (const [key, value] of Object.entries(output)) {
|
||||
// 跳过已处理的类型
|
||||
if (processedTypes.has(key)) continue;
|
||||
processedTypes.add(key);
|
||||
|
||||
// 检查是否为数组类型
|
||||
if (!Array.isArray(value)) {
|
||||
// 如果是非文件类型的非数组值，记录但不处理
|
||||
if (NON_FILE_OUTPUT_TYPES.includes(key)) {
|
||||
typeCounts[key] = 'skipped (non-file type)';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 检查数组中的元素是否为有效的文件对象（包含filename字段）
|
||||
const validFiles = value.filter(item => item && typeof item === 'object' && 'filename' in item);
|
||||
|
||||
if (validFiles.length > 0) {
|
||||
// 有效的文件数组
|
||||
mediaFiles.push(...validFiles);
|
||||
typeCounts[key] = validFiles.length;
|
||||
} else if (value.length > 0) {
|
||||
// 数组但没有有效的filename，可能是text/value等类型
|
||||
typeCounts[key] = `skipped (${value.length} items, no filename)`;
|
||||
logger.debug(`[processHistoryOutputs] 节点 ${nodeId} 跳过非文件类型: ${key}, 数量: ${value.length}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 打印日志，显示各类型的数量
|
||||
const typeCountStr = Object.entries(typeCounts)
|
||||
.map(([type, count]) => `${type}=${count}`)
|
||||
.join(', ');
|
||||
|
||||
if (mediaFiles.length > 0) {
|
||||
logger.info(`[processHistoryOutputs] 节点 ${nodeId} 找到 ${mediaFiles.length} 个媒体文件 (${typeCountStr})`);
|
||||
} else {
|
||||
logger.debug(`[processHistoryOutputs] 节点 ${nodeId} 无有效媒体文件: ${typeCountStr}`);
|
||||
}
|
||||
|
||||
for (const media of mediaFiles) {
|
||||
try {
|
||||
// 再次检查filename是否存在
|
||||
if (!media.filename) {
|
||||
logger.warn(`[processHistoryOutputs] 跳过没有filename的媒体: nodeId=${nodeId}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.info(`[processHistoryOutputs] 正在上传文件: ${media.filename}`);
|
||||
const fileUrl = await this.uploadImage(media, instance);
|
||||
logger.info(`[processHistoryOutputs] 文件上传成功: ${fileUrl}`);
|
||||
|
|
@ -395,45 +500,118 @@ class TaskForwarder {
|
|||
return resultData;
|
||||
}
|
||||
|
||||
async uploadImage(image, instance) {
|
||||
/**
|
||||
* 上传单个媒体文件到外部服务器
|
||||
*
|
||||
* 支持的文件类型：
|
||||
* - 图片：images (png, jpg等)
|
||||
* - 动图：gifs
|
||||
* - 音频：audio (flac, mp3, wav等)
|
||||
*
|
||||
* 上传流程（两种方式）：
|
||||
*
|
||||
* 方式一：本地文件直接上传（优先）
|
||||
* - 条件：环境变量 COMFYUI_OUTPUT_DIR 已配置，且文件存在于本地
|
||||
* - 适用场景：bridge服务与ComfyUI在同一台服务器，可直接访问output目录
|
||||
* - 流程：直接读取本地文件 → 上传到外部服务器
|
||||
*
|
||||
* 方式二：通过ComfyUI HTTP API获取文件（回退方案）
|
||||
* - 条件：本地文件不存在或COMFYUI_OUTPUT_DIR未配置
|
||||
* - 适用场景：bridge服务与ComfyUI不在同一台服务器
|
||||
* - 流程：调用ComfyUI的/view API → 保存到本地临时目录 → 上传到外部服务器 → 删除临时文件
|
||||
*
|
||||
* @param {Object} media - 媒体文件信息对象
|
||||
* @param {string} media.filename - 文件名
|
||||
* @param {string} media.subfolder - 子文件夹路径（可选）
|
||||
* @param {string} media.type - 文件类型（output/temp/input）
|
||||
* @param {Object} instance - ComfyUI实例对象
|
||||
* @returns {string} 上传后的文件URL
|
||||
*/
|
||||
async uploadImage(media, instance) {
|
||||
const comfyuiOutputDir = process.env.COMFYUI_OUTPUT_DIR;
|
||||
|
||||
// ========== 方式一：本地文件直接上传 ==========
|
||||
if (comfyuiOutputDir) {
|
||||
let localFilePath = path.join(comfyuiOutputDir, image.filename);
|
||||
if (image.subfolder) {
|
||||
localFilePath = path.join(comfyuiOutputDir, image.subfolder, image.filename);
|
||||
// 构建本地文件完整路径
|
||||
let localFilePath = path.join(comfyuiOutputDir, media.filename);
|
||||
if (media.subfolder) {
|
||||
localFilePath = path.join(comfyuiOutputDir, media.subfolder, media.filename);
|
||||
}
|
||||
|
||||
// 检查本地文件是否存在
|
||||
if (fs.existsSync(localFilePath)) {
|
||||
logger.info(`从本地目录读取文件: ${localFilePath}`);
|
||||
const uploadResult = await fileUploader.uploadToExternalServer(localFilePath, image.filename);
|
||||
// 直接上传本地文件
|
||||
const uploadResult = await fileUploader.uploadToExternalServer(localFilePath, media.filename);
|
||||
return uploadResult.url || uploadResult.data?.url;
|
||||
} else {
|
||||
logger.warn(`本地文件不存在: ${localFilePath}, 回退到 HTTP API`);
|
||||
}
|
||||
}
|
||||
|
||||
const imageUrl = `${instance.apiUrl}/view?filename=${image.filename}&subfolder=${image.subfolder || ''}&type=${image.type}`;
|
||||
// ========== 方式二：通过HTTP API获取文件 ==========
|
||||
// 构建ComfyUI /view API的URL
|
||||
// API格式: http://host:port/view?filename=xxx&subfolder=xxx&type=output
|
||||
const mediaUrl = `${instance.apiUrl}/view?filename=${media.filename}&subfolder=${media.subfolder || ''}&type=${media.type}`;
|
||||
|
||||
const response = await axios.get(imageUrl, {
|
||||
// 通过HTTP请求获取文件内容（二进制数据）
|
||||
const response = await axios.get(mediaUrl, {
|
||||
responseType: 'arraybuffer'
|
||||
});
|
||||
|
||||
// 创建本地临时目录用于存放下载的文件
|
||||
const uploadsDir = path.resolve(process.cwd(), 'uploads');
|
||||
if (!fs.existsSync(uploadsDir)) {
|
||||
fs.mkdirSync(uploadsDir, { recursive: true });
|
||||
}
|
||||
|
||||
const tempPath = path.join(uploadsDir, `${uuidv4()}.${image.type}`);
|
||||
// 从文件名中提取扩展名，或使用media.type
|
||||
let ext = media.type;
|
||||
if (media.filename) {
|
||||
const match = media.filename.match(/\.([a-zA-Z0-9]+)$/);
|
||||
if (match) {
|
||||
ext = match[1];
|
||||
}
|
||||
}
|
||||
|
||||
// 生成唯一的临时文件名，避免冲突
|
||||
const tempPath = path.join(uploadsDir, `${uuidv4()}.${ext}`);
|
||||
|
||||
// 将下载的文件内容写入临时文件
|
||||
await fs.promises.writeFile(tempPath, response.data);
|
||||
|
||||
const uploadResult = await fileUploader.uploadToExternalServer(tempPath, image.filename);
|
||||
// 上传临时文件到外部服务器
|
||||
const uploadResult = await fileUploader.uploadToExternalServer(tempPath, media.filename);
|
||||
|
||||
// 上传完成后删除临时文件，释放磁盘空间
|
||||
await fs.promises.unlink(tempPath);
|
||||
|
||||
return uploadResult.url || uploadResult.data?.url;
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送Webhook回调通知
|
||||
*
|
||||
* 说明：
|
||||
* - 任务完成或失败时，通过webhook通知调用方
|
||||
* - 成功时会重新处理一遍输出文件并上传
|
||||
* - 失败时直接返回错误信息
|
||||
*
|
||||
* 回调消息格式：
|
||||
* {
|
||||
* event: 'TASK_END',
|
||||
* taskId: 'xxx',
|
||||
* eventData: JSON.stringify({
|
||||
* code: 0 | 1, // 0成功，1失败
|
||||
* msg: 'success' | 'error message',
|
||||
* data: [...] // 成功时包含文件URL列表
|
||||
* })
|
||||
* }
|
||||
*
|
||||
* @param {Object} task - 任务对象
|
||||
* @param {Object|null} resultData - 成功时的结果数据（包含output）
|
||||
* @param {string|null} error - 失败时的错误信息
|
||||
*/
|
||||
async sendWebhookCallback(task, resultData, error = null) {
|
||||
if (!task.webhookUrl) {
|
||||
return;
|
||||
|
|
@ -441,12 +619,14 @@ class TaskForwarder {
|
|||
|
||||
let eventData;
|
||||
if (error) {
|
||||
// 失败情况：返回错误信息
|
||||
eventData = JSON.stringify({
|
||||
code: 1,
|
||||
msg: error,
|
||||
data: []
|
||||
});
|
||||
} else {
|
||||
// 成功情况：重新处理输出文件并上传
|
||||
const processedData = await this.processHistoryOutputs(resultData?.output, task.instanceId);
|
||||
eventData = JSON.stringify({
|
||||
code: 0,
|
||||
|
|
@ -455,9 +635,11 @@ class TaskForwarder {
|
|||
});
|
||||
}
|
||||
|
||||
// 构建回调消息
|
||||
const callbackMessage = {
|
||||
event: 'TASK_END',
|
||||
taskId: task.queueTaskId || task.id,
|
||||
instanceId: task.instanceId, // 添加 instanceId 字段
|
||||
eventData
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -271,6 +271,7 @@ class MessageDispatcherClient extends EventEmitter {
|
|||
nodeInfoList,
|
||||
webhookUrl,
|
||||
requestId,
|
||||
instanceId, // 保存 instanceId
|
||||
status: 'pending',
|
||||
createdAt: new Date().toISOString()
|
||||
};
|
||||
|
|
@ -310,10 +311,10 @@ class MessageDispatcherClient extends EventEmitter {
|
|||
return;
|
||||
}
|
||||
|
||||
const { webhookUrl, requestId } = taskRecord;
|
||||
const { webhookUrl, requestId, instanceId } = taskRecord;
|
||||
|
||||
if (!webhookUrl) {
|
||||
logger.warn('[MessageDispatcher] 缺少webhookUrl，无法发送回调');
|
||||
logger.warn('[MessageDispatcher] 缺少 webhookUrl，无法发送回调');
|
||||
} else {
|
||||
let eventData;
|
||||
if (error) {
|
||||
|
|
@ -333,15 +334,16 @@ class MessageDispatcherClient extends EventEmitter {
|
|||
const callbackMessage = {
|
||||
event: 'TASK_END',
|
||||
taskId,
|
||||
instanceId, // 添加 instanceId 字段
|
||||
eventData
|
||||
};
|
||||
|
||||
try {
|
||||
logger.info(`[MessageDispatcher] 发送回调到: ${webhookUrl}`);
|
||||
logger.info(`[MessageDispatcher] 发送回调到：${webhookUrl}`);
|
||||
await axios.post(webhookUrl, callbackMessage, {
|
||||
timeout: 10000
|
||||
});
|
||||
logger.info(`[MessageDispatcher] 回调发送成功: ${taskId}`);
|
||||
logger.info(`[MessageDispatcher] 回调发送成功：${taskId}`);
|
||||
} catch (error) {
|
||||
logger.error('[MessageDispatcher] 发送回调失败:', error.message);
|
||||
}
|
||||
|
|
@ -352,6 +354,7 @@ class MessageDispatcherClient extends EventEmitter {
|
|||
data: {
|
||||
requestId,
|
||||
taskId,
|
||||
instanceId, // 添加 instanceId 字段
|
||||
result: resultData,
|
||||
error: error
|
||||
}
|
||||
|
|
@ -361,8 +364,8 @@ class MessageDispatcherClient extends EventEmitter {
|
|||
this.pendingTasks.delete(taskId);
|
||||
}
|
||||
|
||||
notifyTaskComplete(taskId, result) {
|
||||
this.sendTaskEndCallback(taskId, result).catch(err => {
|
||||
notifyTaskComplete(taskId, result, error = null) {
|
||||
this.sendTaskEndCallback(taskId, result, error).catch(err => {
|
||||
logger.error('[MessageDispatcher] 处理任务完成通知失败:', err);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,10 @@ class WebSocketClient extends EventEmitter {
|
|||
resolve(ws);
|
||||
});
|
||||
|
||||
ws.on('message', (data) => {
|
||||
ws.on('message', (data, isBinary) => {
|
||||
if (isBinary) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const message = JSON.parse(data.toString());
|
||||
this.handleMessage(instanceId, message);
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -12,3 +12,6 @@ ADMIN_PASSWORD=2233..2233
|
|||
TASK_QUEUE_WS_URL=ws://localhost:8088
|
||||
# 任务队列后端 token (与任务队列后端的 TOKEN_SECRET 保持一致)
|
||||
TASK_QUEUE_TOKEN=1Ag9BJJn0rXDnidCyXqu
|
||||
|
||||
# 任务超时时间 (毫秒)，默认1小时
|
||||
TASK_TIMEOUT=3600000
|
||||
|
|
|
|||
|
|
@ -8,6 +8,21 @@ import { authMiddleware } from '../auth/middleware.js';
|
|||
|
||||
const router = express.Router();
|
||||
|
||||
router.get('/capacity', authMiddleware, (req, res) => {
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
const stats = taskScheduler.getStats();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
...capacity,
|
||||
processingTasks: stats.processing,
|
||||
pendingTasks: stats.pending,
|
||||
availableForNewTasks: Math.max(0, capacity.available - stats.pending)
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/health', (req, res) => {
|
||||
res.json({
|
||||
success: true,
|
||||
|
|
@ -51,7 +66,7 @@ router.post('/task', authMiddleware, async (req, res) => {
|
|||
const requestId = uuidv4();
|
||||
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
taskScheduler.setCurrentCapacity(capacity.available);
|
||||
await taskScheduler.setCurrentCapacity(capacity.online, false);
|
||||
|
||||
const assignResult = taskScheduler.tryDirectAssign({
|
||||
requestId,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
import logger from '../logger/index.js';
|
||||
|
||||
let mdWebSocketClient = null;
|
||||
|
||||
export function setMDWebSocketClient(client) {
|
||||
mdWebSocketClient = client;
|
||||
}
|
||||
|
||||
class BridgeManager {
|
||||
constructor() {
|
||||
this.bridges = new Map();
|
||||
|
|
@ -7,7 +13,9 @@ class BridgeManager {
|
|||
this.instanceLocks = new Map();
|
||||
this.lockTimeouts = new Map();
|
||||
this.LOCK_TIMEOUT = 30000;
|
||||
this.LONG_LOCK_TIMEOUT = parseInt(process.env.LONG_LOCK_TIMEOUT) || 2 * 60 * 60 * 1000;
|
||||
this.roundRobinIndex = 0;
|
||||
this.lockCheckInterval = null;
|
||||
}
|
||||
|
||||
onBridgeChange(callback) {
|
||||
|
|
@ -91,7 +99,7 @@ class BridgeManager {
|
|||
|
||||
this.lockTimeouts.set(instanceId, timeoutId);
|
||||
|
||||
logger.info(`[BridgeManager] 实例已锁定: ${instanceId}, taskId: ${taskId}`);
|
||||
// logger.info(`[BridgeManager] 实例已锁定: ${instanceId}, taskId: ${taskId}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -104,17 +112,26 @@ class BridgeManager {
|
|||
this.lockTimeouts.delete(instanceId);
|
||||
}
|
||||
|
||||
logger.info(`[BridgeManager] 实例锁已释放: ${instanceId}`);
|
||||
logger.info(`[BridgeManager] 实例锁已释放：${instanceId}`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
||||
// 锁不存在时也返回 true（幂等操作）
|
||||
logger.debug(`[BridgeManager] 实例锁不存在，无需释放：${instanceId}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
confirmInstanceLock(instanceId) {
|
||||
if (this.lockTimeouts.has(instanceId)) {
|
||||
clearTimeout(this.lockTimeouts.get(instanceId));
|
||||
this.lockTimeouts.delete(instanceId);
|
||||
logger.info(`[BridgeManager] 实例锁已确认，取消超时定时器: ${instanceId}`);
|
||||
|
||||
const timeoutId = setTimeout(() => {
|
||||
logger.warn(`[BridgeManager] 实例锁长时间未释放，自动释放: ${instanceId}`);
|
||||
this.releaseInstanceLock(instanceId);
|
||||
}, this.LONG_LOCK_TIMEOUT);
|
||||
|
||||
this.lockTimeouts.set(instanceId, timeoutId);
|
||||
logger.info(`[BridgeManager] 实例锁已确认，设置长超时定时器: ${instanceId}`);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
|
|
@ -156,7 +173,7 @@ class BridgeManager {
|
|||
bestInstance = selected.instance;
|
||||
bestBridgeId = selected.bridgeId;
|
||||
|
||||
console.log(`[BridgeManager] 找到空闲实例: ${bestInstance.id}, 所属 bridge: ${bestBridgeId}`);
|
||||
// console.log(`[BridgeManager] 找到空闲实例: ${bestInstance.id}, 所属 bridge: ${bestBridgeId}`);
|
||||
|
||||
return { instance: bestInstance, bridgeId: bestBridgeId };
|
||||
}
|
||||
|
|
@ -230,7 +247,7 @@ class BridgeManager {
|
|||
return false;
|
||||
}
|
||||
try {
|
||||
console.log(`[分发] BridgeManager 发送到桥接器 ${bridgeId}:`, JSON.stringify(message, null, 2));
|
||||
// console.log(`[分发] BridgeManager 发送到桥接器 ${bridgeId}:`, JSON.stringify(message, null, 2));
|
||||
bridge.ws.send(JSON.stringify(message));
|
||||
return true;
|
||||
} catch (error) {
|
||||
|
|
@ -258,6 +275,40 @@ class BridgeManager {
|
|||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
startLockCheck() {
|
||||
if (this.lockCheckInterval) {
|
||||
clearInterval(this.lockCheckInterval);
|
||||
}
|
||||
|
||||
this.lockCheckInterval = setInterval(() => {
|
||||
this.checkAllInstanceLocks();
|
||||
}, 60000);
|
||||
}
|
||||
|
||||
stopLockCheck() {
|
||||
if (this.lockCheckInterval) {
|
||||
clearInterval(this.lockCheckInterval);
|
||||
this.lockCheckInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
checkAllInstanceLocks() {
|
||||
const now = Date.now();
|
||||
|
||||
for (const [instanceId, lockInfo] of this.instanceLocks) {
|
||||
const lockedAt = new Date(lockInfo.lockedAt).getTime();
|
||||
const elapsed = now - lockedAt;
|
||||
|
||||
if (elapsed > this.LONG_LOCK_TIMEOUT) {
|
||||
logger.warn(`[BridgeManager] 实例锁检测超时，强制释放: ${instanceId}, taskId: ${lockInfo.taskId}, 已锁定 ${Math.round(elapsed / 1000)}秒`);
|
||||
this.releaseInstanceLock(instanceId);
|
||||
if (mdWebSocketClient) {
|
||||
mdWebSocketClient.pushCapacityState();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new BridgeManager();
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import logger from './logger/index.js';
|
|||
import websocketServer from './websocket-server/index.js';
|
||||
import mdWebSocketClient from './md-websocket-client/index.js';
|
||||
import taskScheduler from './task-scheduler/index.js';
|
||||
import bridgeManager, { setMDWebSocketClient } from './bridge-manager/index.js';
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 4000;
|
||||
|
|
@ -40,8 +41,12 @@ server.headersTimeout = 66000;
|
|||
|
||||
websocketServer.start(server);
|
||||
|
||||
setMDWebSocketClient(mdWebSocketClient);
|
||||
|
||||
taskScheduler.init().then(() => {
|
||||
console.log('[TaskScheduler] 任务调度器已初始化');
|
||||
bridgeManager.startLockCheck();
|
||||
console.log('[BridgeManager] 实例锁检测已启动');
|
||||
return mdWebSocketClient.init();
|
||||
}).then(() => {
|
||||
console.log('[MDWebSocketClient] WebSocket 客户端已初始化');
|
||||
|
|
@ -52,6 +57,7 @@ taskScheduler.init().then(() => {
|
|||
process.on('SIGINT', async () => {
|
||||
console.log('正在关闭服务...');
|
||||
try {
|
||||
bridgeManager.stopLockCheck();
|
||||
await taskScheduler.shutdown();
|
||||
mdWebSocketClient.disconnect();
|
||||
|
||||
|
|
|
|||
|
|
@ -128,9 +128,9 @@ class MDWebSocketClient {
|
|||
);
|
||||
}
|
||||
|
||||
pushCapacityState() {
|
||||
pushCapacityState(pendingCount = 0) {
|
||||
const bridges = bridgeManager.getAllBridges();
|
||||
const summary = this.calculateCapacitySummary(bridges);
|
||||
const summary = this.calculateCapacitySummary(bridges, pendingCount);
|
||||
|
||||
const message = {
|
||||
type: 'CAPACITY_UPDATE',
|
||||
|
|
@ -148,22 +148,30 @@ class MDWebSocketClient {
|
|||
console.log('[MDWebSocketClient] 已推送算力状态:', summary);
|
||||
}
|
||||
|
||||
calculateCapacitySummary(bridges) {
|
||||
calculateCapacitySummary(bridges, pendingCount = 0) {
|
||||
let totalInstances = 0;
|
||||
let onlineInstances = 0;
|
||||
let busyInstances = 0;
|
||||
let lockedInstances = 0;
|
||||
let offlineInstances = 0;
|
||||
|
||||
for (const bridge of bridges) {
|
||||
if (bridge.info?.instances) {
|
||||
totalInstances += bridge.info.instances.length;
|
||||
onlineInstances += bridge.info.instances.filter(i => i.status === 'online').length;
|
||||
busyInstances += bridge.info.instances.filter(i => i.status === 'busy').length;
|
||||
offlineInstances += bridge.info.instances.filter(i => i.status === 'offline').length;
|
||||
for (const instance of bridge.info.instances) {
|
||||
if (instance.status === 'online') {
|
||||
onlineInstances++;
|
||||
if (bridgeManager.isInstanceLocked(instance.id)) {
|
||||
lockedInstances++;
|
||||
}
|
||||
} else {
|
||||
offlineInstances++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const availableCapacity = onlineInstances - busyInstances;
|
||||
const busyInstances = lockedInstances;
|
||||
const availableCapacity = Math.max(0, onlineInstances - busyInstances - pendingCount);
|
||||
|
||||
return {
|
||||
totalBridges: bridges.length,
|
||||
|
|
@ -171,6 +179,7 @@ class MDWebSocketClient {
|
|||
onlineInstances,
|
||||
busyInstances,
|
||||
offlineInstances,
|
||||
lockedInstances,
|
||||
availableCapacity
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import bridgeManager from '../bridge-manager/index.js';
|
||||
import websocketServer from '../websocket-server/index.js';
|
||||
import mdWebSocketClient from '../md-websocket-client/index.js';
|
||||
import logger from '../logger/index.js';
|
||||
|
||||
const TASK_STATES = {
|
||||
|
|
@ -23,26 +24,53 @@ class TaskScheduler {
|
|||
this.MAX_PENDING_QUEUE = 100;
|
||||
this.MAX_COMPLETED_HISTORY = 1000;
|
||||
this.MAX_FAILED_HISTORY = 100;
|
||||
this.TASK_TIMEOUT = 5 * 60 * 1000;
|
||||
this.TASK_TIMEOUT = parseInt(process.env.TASK_TIMEOUT) || 30 * 60 * 1000;
|
||||
this.RECOVERY_TIMEOUT = 60000;
|
||||
this.taskCallbacks = new Map();
|
||||
}
|
||||
|
||||
async init() {
|
||||
console.log('[TaskScheduler] 初始化任务调度器');
|
||||
logger.info('[TaskScheduler] 初始化任务调度器');
|
||||
this.startSchedulerLoop();
|
||||
this.startTimeoutCheck();
|
||||
|
||||
if (this.pendingTaskQueue.length > 0) {
|
||||
logger.info(`[TaskScheduler] 检测到队列中有 ${this.pendingTaskQueue.length} 个任务，开始调度`);
|
||||
await this.schedulePendingTasks();
|
||||
}
|
||||
}
|
||||
|
||||
setCurrentCapacity(capacity) {
|
||||
async setCurrentCapacity(capacity, shouldPushToBackend = true) {
|
||||
const oldCapacity = this.currentCapacity;
|
||||
this.currentCapacity = capacity;
|
||||
this.maxCapacity = Math.max(this.maxCapacity, capacity);
|
||||
|
||||
console.log(`[TaskScheduler] 容量更新: ${oldCapacity} -> ${capacity}`);
|
||||
const processingCount = this.processingTasks.size;
|
||||
const pendingCount = this.pendingTaskQueue.length;
|
||||
const availableCapacity = Math.max(0, capacity - processingCount - pendingCount);
|
||||
|
||||
if (capacity > oldCapacity) {
|
||||
this.schedulePendingTasks();
|
||||
logger.info(`[容量更新] 总算力：${capacity}, 已用：${processingCount}, 等待：${pendingCount}, 可用：${availableCapacity}`);
|
||||
|
||||
if (this.pendingTaskQueue.length > 0) {
|
||||
logger.info(`[任务调度] 检测到队列有 ${pendingCount} 个等待任务，开始调度`);
|
||||
await this.schedulePendingTasks();
|
||||
|
||||
const remainingPending = this.pendingTaskQueue.length;
|
||||
const newAvailableCapacity = Math.max(0, capacity - this.processingTasks.size - remainingPending);
|
||||
|
||||
if (remainingPending > 0) {
|
||||
logger.info(`[容量更新] 调度完成，仍有 ${remainingPending} 个任务等待，推送算力状态给后端 (可用：${newAvailableCapacity})`);
|
||||
} else {
|
||||
logger.info(`[容量更新] 队列已清空，推送算力状态给后端 (可用：${newAvailableCapacity})`);
|
||||
}
|
||||
|
||||
if (shouldPushToBackend && mdWebSocketClient) {
|
||||
mdWebSocketClient.pushCapacityState(remainingPending);
|
||||
}
|
||||
} else {
|
||||
if (shouldPushToBackend && mdWebSocketClient) {
|
||||
mdWebSocketClient.pushCapacityState(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -62,6 +90,7 @@ class TaskScheduler {
|
|||
const taskId = taskData.requestId || taskData.taskId;
|
||||
|
||||
if (this.pendingTaskQueue.length >= this.MAX_PENDING_QUEUE) {
|
||||
logger.warn(`[任务队列] 队列已满，拒绝任务：${taskId}, 当前等待数：${this.pendingTaskQueue.length}`);
|
||||
return {
|
||||
success: false,
|
||||
error: '任务队列已满',
|
||||
|
|
@ -78,7 +107,7 @@ class TaskScheduler {
|
|||
};
|
||||
|
||||
this.pendingTaskQueue.push(taskWithState);
|
||||
console.log(`[TaskScheduler] 任务已加入等待队列: ${taskId}, 当前等待数: ${this.pendingTaskQueue.length}`);
|
||||
logger.info(`[任务入队] 任务已加入等待队列：${taskId}, 当前等待数：${this.pendingTaskQueue.length}`);
|
||||
|
||||
this.schedulePendingTasks();
|
||||
|
||||
|
|
@ -123,8 +152,11 @@ class TaskScheduler {
|
|||
};
|
||||
|
||||
this.processingTasks.set(taskId, taskWithState);
|
||||
|
||||
console.log(`[TaskScheduler] 任务直接分配: ${taskId} -> 实例 ${instance.id}`);
|
||||
|
||||
logger.info(`[算力状态] 总算力：${capacity.online}, 已用：${capacity.used}, 可用：${capacity.available}`);
|
||||
if (mdWebSocketClient) {
|
||||
mdWebSocketClient.pushCapacityState(this.pendingTaskQueue.length);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
|
|
@ -145,13 +177,13 @@ class TaskScheduler {
|
|||
|
||||
const tasksToSchedule = this.pendingTaskQueue.splice(0, availableSlots);
|
||||
|
||||
console.log(`[TaskScheduler] 调度 ${tasksToSchedule.length} 个任务, 可用槽位: ${availableSlots}`);
|
||||
logger.info(`[任务调度] 计划调度 ${tasksToSchedule.length} 个任务，可用槽位：${availableSlots}`);
|
||||
|
||||
for (const task of tasksToSchedule) {
|
||||
const { instance, bridgeId } = bridgeManager.getAvailableInstanceAndLock(task.taskId);
|
||||
|
||||
if (!instance || !bridgeId) {
|
||||
console.warn(`[TaskScheduler] 无可用实例，任务 ${task.taskId} 返回队列`);
|
||||
logger.warn(`[任务调度] 无可用实例，任务返回队列：${task.taskId}`);
|
||||
this.pendingTaskQueue.unshift(task);
|
||||
break;
|
||||
}
|
||||
|
|
@ -162,9 +194,15 @@ class TaskScheduler {
|
|||
task.startedAt = new Date().toISOString();
|
||||
|
||||
this.processingTasks.set(task.taskId, task);
|
||||
|
||||
console.log(`[TaskScheduler] 任务调度: ${task.taskId} -> 实例 ${instance.id}`);
|
||||
|
||||
|
||||
logger.info(`[任务调度] 任务分配：${task.taskId} -> 实例 ${instance.id} (桥接器：${bridgeId})`);
|
||||
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
logger.info(`[算力状态] 总算力：${capacity.online}, 已用：${capacity.used}, 可用：${capacity.available}`);
|
||||
if (mdWebSocketClient) {
|
||||
mdWebSocketClient.pushCapacityState(this.pendingTaskQueue.length);
|
||||
}
|
||||
|
||||
try {
|
||||
await websocketServer.sendTaskToInstance(
|
||||
bridgeId,
|
||||
|
|
@ -176,8 +214,9 @@ class TaskScheduler {
|
|||
},
|
||||
task.taskId
|
||||
);
|
||||
logger.info(`[任务调度] 任务发送成功：${task.taskId}`);
|
||||
} catch (error) {
|
||||
console.error(`[TaskScheduler] 任务发送失败: ${task.taskId}`, error);
|
||||
logger.error(`[任务调度] 任务发送失败：${task.taskId}`, error);
|
||||
this.handleTaskFailure(task.taskId, error.message);
|
||||
}
|
||||
}
|
||||
|
|
@ -188,19 +227,17 @@ class TaskScheduler {
|
|||
if (task) {
|
||||
task.ackReceived = true;
|
||||
task.ackAt = new Date().toISOString();
|
||||
console.log(`[TaskScheduler] 任务确认: ${taskId}`);
|
||||
logger.info(`[任务确认] 任务已确认：${taskId} (实例：${instanceId})`);
|
||||
}
|
||||
}
|
||||
|
||||
handleTaskComplete(taskId, result) {
|
||||
async handleTaskComplete(taskId, result) {
|
||||
const task = this.processingTasks.get(taskId);
|
||||
if (task) {
|
||||
task.state = TASK_STATES.COMPLETED;
|
||||
task.result = result;
|
||||
task.completedAt = new Date().toISOString();
|
||||
|
||||
bridgeManager.releaseInstanceLock(task.instanceId);
|
||||
|
||||
this.processingTasks.delete(taskId);
|
||||
|
||||
this.completedTasks.push(task);
|
||||
|
|
@ -208,46 +245,78 @@ class TaskScheduler {
|
|||
this.completedTasks.shift();
|
||||
}
|
||||
|
||||
console.log(`[TaskScheduler] 任务完成: ${taskId}`);
|
||||
logger.info(`[任务完成] 任务已成功完成：${taskId}`);
|
||||
|
||||
this.schedulePendingTasks();
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
this.setCurrentCapacity(capacity.online, false);
|
||||
|
||||
await this.schedulePendingTasks();
|
||||
|
||||
if (this.pendingTaskQueue.length === 0) {
|
||||
if (mdWebSocketClient) {
|
||||
mdWebSocketClient.pushCapacityState(0);
|
||||
}
|
||||
} else {
|
||||
logger.info(`[容量更新] 队列仍有 ${this.pendingTaskQueue.length} 个任务，等待下次调度`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
handleTaskFailure(taskId, error) {
|
||||
async handleTaskFailure(taskId, error) {
|
||||
const task = this.processingTasks.get(taskId);
|
||||
if (task) {
|
||||
task.state = TASK_STATES.FAILED;
|
||||
task.error = error;
|
||||
task.failedAt = new Date().toISOString();
|
||||
|
||||
bridgeManager.releaseInstanceLock(task.instanceId);
|
||||
|
||||
|
||||
this.processingTasks.delete(taskId);
|
||||
|
||||
|
||||
this.failedTasks.push(task);
|
||||
if (this.failedTasks.length > this.MAX_FAILED_HISTORY) {
|
||||
this.failedTasks.shift();
|
||||
}
|
||||
|
||||
logger.error(`[任务失败] 任务执行失败：${taskId}, 错误：${error}`);
|
||||
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
this.setCurrentCapacity(capacity.online, false);
|
||||
|
||||
console.error(`[TaskScheduler] 任务失败: ${taskId}`, error);
|
||||
await this.schedulePendingTasks();
|
||||
|
||||
this.schedulePendingTasks();
|
||||
if (this.pendingTaskQueue.length === 0) {
|
||||
if (mdWebSocketClient) {
|
||||
mdWebSocketClient.pushCapacityState(0);
|
||||
}
|
||||
} else {
|
||||
logger.info(`[容量更新] 队列仍有 ${this.pendingTaskQueue.length} 个任务，等待下次调度`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
handleInstanceOffline(instanceId) {
|
||||
async handleInstanceOffline(instanceId) {
|
||||
const affectedTaskId = bridgeManager.handleInstanceOffline(instanceId);
|
||||
|
||||
|
||||
if (affectedTaskId) {
|
||||
const task = this.processingTasks.get(affectedTaskId);
|
||||
if (task) {
|
||||
console.warn(`[TaskScheduler] 实例离线，任务需要回收: ${affectedTaskId}`);
|
||||
logger.warn(`[实例离线] 实例 ${instanceId} 离线，任务需要回收：${affectedTaskId}`);
|
||||
this.recoverTask(affectedTaskId, 'instance_offline');
|
||||
}
|
||||
}
|
||||
|
||||
this.schedulePendingTasks();
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
this.setCurrentCapacity(capacity.online, false);
|
||||
|
||||
await this.schedulePendingTasks();
|
||||
|
||||
if (this.pendingTaskQueue.length === 0) {
|
||||
if (mdWebSocketClient) {
|
||||
logger.info('[容量更新] 实例离线处理完成，推送最终算力状态给后端');
|
||||
mdWebSocketClient.pushCapacityState(0);
|
||||
}
|
||||
} else {
|
||||
logger.info(`[容量更新] 队列仍有 ${this.pendingTaskQueue.length} 个任务，等待下次调度`);
|
||||
}
|
||||
}
|
||||
|
||||
recoverTask(taskId, reason) {
|
||||
|
|
@ -267,7 +336,7 @@ class TaskScheduler {
|
|||
this.executeRecovery(taskId);
|
||||
}, 1000);
|
||||
|
||||
console.log(`[TaskScheduler] 任务进入恢复流程: ${taskId}, 原因: ${reason}`);
|
||||
logger.info(`[任务恢复] 任务进入恢复流程：${taskId}, 原因：${reason}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -280,16 +349,16 @@ class TaskScheduler {
|
|||
task.retryCount = (task.retryCount || 0) + 1;
|
||||
|
||||
if (task.retryCount > 3) {
|
||||
console.error(`[TaskScheduler] 任务恢复失败次数过多: ${taskId}`);
|
||||
logger.error(`[任务恢复] 任务恢复失败次数过多，标记为失败：${taskId}`);
|
||||
this.recoveringTasks.delete(taskId);
|
||||
|
||||
task.state = TASK_STATES.FAILED;
|
||||
task.error = `任务恢复失败: 重试次数超过限制`;
|
||||
task.error = `任务恢复失败：重试次数超过限制`;
|
||||
this.failedTasks.push(task);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[TaskScheduler] 尝试恢复任务: ${taskId}, 重试次数: ${task.retryCount}`);
|
||||
logger.info(`[任务恢复] 尝试恢复任务：${taskId}, 重试次数：${task.retryCount}`);
|
||||
|
||||
this.recoveringTasks.delete(taskId);
|
||||
|
||||
|
|
@ -302,6 +371,7 @@ class TaskScheduler {
|
|||
};
|
||||
|
||||
this.pendingTaskQueue.unshift(retryTask);
|
||||
logger.info(`[任务恢复] 任务已重新加入队列：${taskId}`);
|
||||
|
||||
this.schedulePendingTasks();
|
||||
}
|
||||
|
|
@ -325,7 +395,7 @@ class TaskScheduler {
|
|||
if (task.startedAt) {
|
||||
const elapsed = now - new Date(task.startedAt).getTime();
|
||||
if (elapsed > this.TASK_TIMEOUT) {
|
||||
console.warn(`[TaskScheduler] 任务超时: ${taskId}, 已运行 ${Math.round(elapsed / 1000)}秒`);
|
||||
logger.warn(`[任务超时] 任务执行超时：${taskId}, 已运行 ${Math.round(elapsed / 1000)}秒`);
|
||||
this.handleTaskFailure(taskId, '任务执行超时');
|
||||
}
|
||||
}
|
||||
|
|
@ -335,7 +405,7 @@ class TaskScheduler {
|
|||
if (task.recoveryStartedAt) {
|
||||
const elapsed = now - new Date(task.recoveryStartedAt).getTime();
|
||||
if (elapsed > this.RECOVERY_TIMEOUT) {
|
||||
console.warn(`[TaskScheduler] 任务恢复超时: ${taskId}`);
|
||||
logger.warn(`[任务超时] 任务恢复超时：${taskId}`);
|
||||
this.recoveringTasks.delete(taskId);
|
||||
task.state = TASK_STATES.FAILED;
|
||||
task.error = '任务恢复超时';
|
||||
|
|
@ -392,7 +462,7 @@ class TaskScheduler {
|
|||
}
|
||||
|
||||
async shutdown() {
|
||||
console.log('[TaskScheduler] 正在关闭调度器');
|
||||
logger.info('[TaskScheduler] 正在关闭调度器');
|
||||
this.stopSchedulerLoop();
|
||||
|
||||
for (const [taskId, task] of this.processingTasks) {
|
||||
|
|
@ -401,6 +471,8 @@ class TaskScheduler {
|
|||
|
||||
this.processingTasks.clear();
|
||||
this.pendingTaskQueue.length = 0;
|
||||
|
||||
logger.info('[TaskScheduler] 调度器已关闭，所有实例锁已释放');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { WebSocketServer as WSServer } from 'ws';
|
|||
import logger from '../logger/index.js';
|
||||
import bridgeManager from '../bridge-manager/index.js';
|
||||
import taskScheduler from '../task-scheduler/index.js';
|
||||
import mdWebSocketClient from '../md-websocket-client/index.js';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
class WebSocketServer {
|
||||
|
|
@ -17,7 +18,7 @@ class WebSocketServer {
|
|||
server,
|
||||
keepalive: true
|
||||
});
|
||||
logger.info('WebSocket服务器已启动');
|
||||
logger.info('WebSocket 服务器已启动');
|
||||
|
||||
this.wss.on('connection', (ws) => {
|
||||
this.handleConnection(ws);
|
||||
|
|
@ -29,7 +30,7 @@ class WebSocketServer {
|
|||
let pingInterval = null;
|
||||
let pongTimeout = null;
|
||||
|
||||
logger.info('新的WebSocket连接已建立');
|
||||
logger.info('新的 WebSocket 连接已建立');
|
||||
|
||||
const PING_INTERVAL = 30000;
|
||||
const PONG_TIMEOUT = 10000;
|
||||
|
|
@ -41,7 +42,7 @@ class WebSocketServer {
|
|||
}
|
||||
ws.ping();
|
||||
pongTimeout = setTimeout(() => {
|
||||
logger.warn('PONG响应超时，关闭连接');
|
||||
logger.warn('PONG 响应超时，关闭连接');
|
||||
ws.terminate();
|
||||
}, PONG_TIMEOUT);
|
||||
};
|
||||
|
|
@ -69,7 +70,7 @@ class WebSocketServer {
|
|||
bridgeManager.unregisterBridge(bridgeId);
|
||||
this.cleanupPendingRequests(bridgeId);
|
||||
}
|
||||
logger.info(`WebSocket连接已关闭 (code: ${code})`);
|
||||
logger.info(`WebSocket 连接已关闭 (code: ${code})`);
|
||||
});
|
||||
|
||||
ws.on('error', (error) => {
|
||||
|
|
@ -77,11 +78,11 @@ class WebSocketServer {
|
|||
if (pongTimeout) {
|
||||
clearTimeout(pongTimeout);
|
||||
}
|
||||
logger.error('WebSocket连接错误:', error);
|
||||
logger.error('WebSocket 连接错误:', error);
|
||||
});
|
||||
}
|
||||
|
||||
handleBridgeDisconnect(bridgeId) {
|
||||
async handleBridgeDisconnect(bridgeId) {
|
||||
const bridge = bridgeManager.getBridge(bridgeId);
|
||||
if (bridge?.info?.instances) {
|
||||
for (const instance of bridge.info.instances) {
|
||||
|
|
@ -91,12 +92,14 @@ class WebSocketServer {
|
|||
}
|
||||
}
|
||||
}
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
await taskScheduler.setCurrentCapacity(capacity.online);
|
||||
}
|
||||
|
||||
handleMessage(ws, data, setBridgeId) {
|
||||
async handleMessage(ws, data, setBridgeId) {
|
||||
try {
|
||||
const message = JSON.parse(data.toString());
|
||||
logger.debug(`收到消息: ${message.type}`);
|
||||
logger.debug(`收到消息：${message.type}`);
|
||||
|
||||
switch (message.type) {
|
||||
case 'REGISTER':
|
||||
|
|
@ -127,10 +130,12 @@ class WebSocketServer {
|
|||
}
|
||||
}
|
||||
|
||||
handleRegister(ws, message, setBridgeId) {
|
||||
async handleRegister(ws, message, setBridgeId) {
|
||||
const bridgeId = message.data?.bridgeId || uuidv4();
|
||||
setBridgeId(bridgeId);
|
||||
bridgeManager.registerBridge(bridgeId, ws, message.data);
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
await taskScheduler.setCurrentCapacity(capacity.online);
|
||||
|
||||
const response = {
|
||||
type: 'REGISTER_ACK',
|
||||
|
|
@ -173,29 +178,43 @@ class WebSocketServer {
|
|||
this.handleBridgeResponse(message);
|
||||
}
|
||||
|
||||
handleTaskEnd(message) {
|
||||
async handleTaskEnd(message) {
|
||||
console.log(`[WebSocketServer] 收到 TASK_END 消息：`, JSON.stringify(message.data, null, 2));
|
||||
|
||||
const requestId = message.data?.requestId;
|
||||
const instanceId = message.data?.instanceId;
|
||||
const result = message.data?.result;
|
||||
const error = message.data?.error;
|
||||
|
||||
console.log(`[WebSocketServer] 解析参数：requestId=${requestId}, instanceId=${instanceId}`);
|
||||
|
||||
if (instanceId) {
|
||||
const existsInMap = this.instanceTaskMap.has(instanceId);
|
||||
console.log(`[WebSocketServer] 实例 ${instanceId} 在 instanceTaskMap 中：${existsInMap}`);
|
||||
|
||||
this.instanceTaskMap.delete(instanceId);
|
||||
bridgeManager.releaseInstanceLock(instanceId);
|
||||
const released = bridgeManager.releaseInstanceLock(instanceId);
|
||||
console.log(`[WebSocketServer] 实例锁释放 ${released ? '成功' : '失败'}: ${instanceId}`);
|
||||
|
||||
if (!released) {
|
||||
console.warn(`[WebSocketServer] 实例锁释放失败，可能原因：1) 锁已超时自动释放 2) 重复收到 TASK_END 消息`);
|
||||
}
|
||||
} else {
|
||||
console.warn(`[WebSocketServer] TASK_END 消息中缺少 instanceId，无法释放锁`);
|
||||
}
|
||||
|
||||
if (requestId) {
|
||||
if (error) {
|
||||
taskScheduler.handleTaskFailure(requestId, error);
|
||||
await taskScheduler.handleTaskFailure(requestId, error);
|
||||
} else {
|
||||
taskScheduler.handleTaskComplete(requestId, result);
|
||||
await taskScheduler.handleTaskComplete(requestId, result);
|
||||
}
|
||||
}
|
||||
|
||||
this.handleBridgeResponse(message);
|
||||
}
|
||||
|
||||
handleInstanceStatusUpdate(message) {
|
||||
async handleInstanceStatusUpdate(message) {
|
||||
const bridgeId = message.data?.bridgeId;
|
||||
const instances = message.data?.instances;
|
||||
|
||||
|
|
@ -208,6 +227,8 @@ class WebSocketServer {
|
|||
}
|
||||
}
|
||||
}
|
||||
const capacity = bridgeManager.getAvailableCapacity();
|
||||
await taskScheduler.setCurrentCapacity(capacity.online);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -270,7 +291,7 @@ class WebSocketServer {
|
|||
}
|
||||
};
|
||||
|
||||
console.log(`[分发] WebSocketServer 发送任务到实例: bridgeId=${bridgeId}, instanceId=${instanceId}, requestId=${requestId}`);
|
||||
// console.log(`[分发] WebSocketServer 发送任务到实例：bridgeId=${bridgeId}, instanceId=${instanceId}, requestId=${requestId}`);
|
||||
|
||||
const success = bridgeManager.sendToBridge(bridgeId, message);
|
||||
if (!success) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,196 @@
|
|||
11|comfyui消息分发 | 2026-04-10 19:18:50 +08:00: [MDWebSocketClient] WebSocket 客户端已初始化
|
||||
11|comfyui消息分发 | 2026-04-10 19:18:50 +08:00: 管理员密码已哈希
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: [2026-04-10 19:19:49] [INFO] 新的 WebSocket 连接已建立
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: [2026-04-10 19:19:49] [INFO] 桥接器已注册: bridge-1
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: [MDWebSocketClient] 已推送算力状态: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: totalBridges: 1,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: totalInstances: 8,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: onlineInstances: 8,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: busyInstances: 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: offlineInstances: 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: lockedInstances: 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: availableCapacity: 8
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: [TaskScheduler] 容量更新：0 -> 8
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: [TaskScheduler] 队列为空，直接更新算力给后端
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: [MDWebSocketClient] 已推送算力状态: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: totalBridges: 1,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: totalInstances: 8,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: onlineInstances: 8,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: busyInstances: 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: offlineInstances: 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: lockedInstances: 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: availableCapacity: 8
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:49 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: [WebSocketServer] 收到 TASK_END 消息： {
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "requestId": "5ac13f1e-fdac-4d18-9684-a1851de59502",
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "taskId": "5ac13f1e-fdac-4d18-9684-a1851de59502",
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "instanceId": "server-1-8194",
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "result": [
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "fileUrl": "https://www.whjbjm.com/api/file/internal/generated/1775819997504_a27c6cc8-410f-4eb9-b7b6-e31132672a89_AnimateDiff_00995-audio.mp4",
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "fileType": "output",
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "taskCostTime": 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "nodeId": "81"
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: ],
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: "error": null
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: [WebSocketServer] 解析参数：requestId=5ac13f1e-fdac-4d18-9684-a1851de59502, instanceId=server-1-8194
|
||||
11|comfyui消息分发 | 2026-04-10 19:19:58 +08:00: [WebSocketServer] 实例锁释放 失败: server-1-8194
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 容量更新：8 -> 8
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 队列为空，直接更新算力给后端
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [BridgeManager] 找到空闲实例: server-1-8190, 所属 bridge: bridge-1
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [2026-04-10 19:20:08] [INFO] [BridgeManager] 实例已锁定: server-1-8190, taskId: 59b10fe4-b2ae-4e73-9d25-ee1c07230bf5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 任务直接分配: 59b10fe4-b2ae-4e73-9d25-ee1c07230bf5 -> 实例 server-1-8190
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [2026-04-10 19:20:08] [INFO] 收到任务请求, 直接分配实例: server-1-8190, bridgeId: bridge-1, requestId: 59b10fe4-b2ae-4e73-9d25-ee1c07230bf5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [分发] WebSocketServer 发送任务到实例：bridgeId=bridge-1, instanceId=server-1-8190, requestId=59b10fe4-b2ae-4e73-9d25-ee1c07230bf5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [分发] BridgeManager 发送到桥接器 bridge-1: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "type": "TASK_ASSIGN",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "data": {
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "workflowId": "2032377173130088450",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "nodeInfoList": [
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "nodeId": "86",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "fieldName": "video",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "fieldValue": "https://www.whjbjm.com/api/file/51006a0f-9398-46de-936c-62140800b824/workflow/cca743d7-e6cd-4d24-adfb-82fa610ebfa7/video(1).mp4"
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: ],
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "webhookUrl": "https://www.whjbjm.com/taskCallback/callback/all",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "requestId": "59b10fe4-b2ae-4e73-9d25-ee1c07230bf5",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "instanceId": "server-1-8190"
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 容量更新：8 -> 7
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 队列为空，直接更新算力给后端
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [BridgeManager] 找到空闲实例: server-1-8192, 所属 bridge: bridge-1
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [2026-04-10 19:20:08] [INFO] [BridgeManager] 实例已锁定: server-1-8192, taskId: 0820eba4-3023-4ff0-8b2c-8a629d3f57d5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 任务直接分配: 0820eba4-3023-4ff0-8b2c-8a629d3f57d5 -> 实例 server-1-8192
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [2026-04-10 19:20:08] [INFO] 收到任务请求, 直接分配实例: server-1-8192, bridgeId: bridge-1, requestId: 0820eba4-3023-4ff0-8b2c-8a629d3f57d5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [分发] WebSocketServer 发送任务到实例：bridgeId=bridge-1, instanceId=server-1-8192, requestId=0820eba4-3023-4ff0-8b2c-8a629d3f57d5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [分发] BridgeManager 发送到桥接器 bridge-1: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "type": "TASK_ASSIGN",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "data": {
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "workflowId": "2032377173130088450",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "nodeInfoList": [
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "nodeId": "86",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "fieldName": "video",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "fieldValue": "https://www.whjbjm.com/api/file/51006a0f-9398-46de-936c-62140800b824/workflow/cca743d7-e6cd-4d24-adfb-82fa610ebfa7/video.mp4"
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: ],
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "webhookUrl": "https://www.whjbjm.com/taskCallback/callback/all",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "requestId": "0820eba4-3023-4ff0-8b2c-8a629d3f57d5",
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: "instanceId": "server-1-8192"
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 任务确认: 59b10fe4-b2ae-4e73-9d25-ee1c07230bf5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [2026-04-10 19:20:08] [INFO] [BridgeManager] 实例锁已确认，设置长超时定时器: server-1-8190
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [TaskScheduler] 任务确认: 0820eba4-3023-4ff0-8b2c-8a629d3f57d5
|
||||
11|comfyui消息分发 | 2026-04-10 19:20:08 +08:00: [2026-04-10 19:20:08] [INFO] [BridgeManager] 实例锁已确认，设置长超时定时器: server-1-8192
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 容量更新：7 -> 6
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 队列为空，直接更新算力给后端
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 容量更新：6 -> 6
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 队列为空，直接更新算力给后端
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [BridgeManager] 找到空闲实例: server-1-8194, 所属 bridge: bridge-1
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] [BridgeManager] 实例已锁定: server-1-8194, taskId: dba87553-3da1-4759-9439-e732a2d3877f
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 任务直接分配: dba87553-3da1-4759-9439-e732a2d3877f -> 实例 server-1-8194
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] 收到任务请求, 直接分配实例: server-1-8194, bridgeId: bridge-1, requestId: dba87553-3da1-4759-9439-e732a2d3877f
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [分发] WebSocketServer 发送任务到实例：bridgeId=bridge-1, instanceId=server-1-8194, requestId=dba87553-3da1-4759-9439-e732a2d3877f
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [分发] BridgeManager 发送到桥接器 bridge-1: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "type": "TASK_ASSIGN",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "data": {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "workflowId": "2032378523398184961",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeInfoList": [
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeId": "128",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldName": "audio_file",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldValue": "https://www.whjbjm.com/api/file/36367795-67ab-4a15-bbc3-337c0776770b/workflow/3da9184b-5695-4a78-b2b6-73db0afde5e6/voice.flac"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: },
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeId": "129",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldName": "video",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldValue": "https://www.whjbjm.com/api/file/36367795-67ab-4a15-bbc3-337c0776770b/workflow/3da9184b-5695-4a78-b2b6-73db0afde5e6/humanMedia.mp4"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: ],
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "webhookUrl": "https://www.whjbjm.com/taskCallback/callback/all",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "requestId": "dba87553-3da1-4759-9439-e732a2d3877f",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "instanceId": "server-1-8194"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [BridgeManager] 找到空闲实例: server-1-8196, 所属 bridge: bridge-1
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] [BridgeManager] 实例已锁定: server-1-8196, taskId: c9615180-3779-4891-b361-e9b6ed3a7471
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 任务直接分配: c9615180-3779-4891-b361-e9b6ed3a7471 -> 实例 server-1-8196
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] 收到任务请求, 直接分配实例: server-1-8196, bridgeId: bridge-1, requestId: c9615180-3779-4891-b361-e9b6ed3a7471
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [分发] WebSocketServer 发送任务到实例：bridgeId=bridge-1, instanceId=server-1-8196, requestId=c9615180-3779-4891-b361-e9b6ed3a7471
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [分发] BridgeManager 发送到桥接器 bridge-1: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "type": "TASK_ASSIGN",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "data": {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "workflowId": "2032378523398184961",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeInfoList": [
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeId": "128",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldName": "audio_file",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldValue": "https://www.whjbjm.com/api/file/36367795-67ab-4a15-bbc3-337c0776770b/workflow/d1a5a9d5-06fe-4764-96a7-656a5ae8b7e9/voice(1).flac"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: },
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeId": "129",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldName": "video",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldValue": "https://www.whjbjm.com/api/file/36367795-67ab-4a15-bbc3-337c0776770b/workflow/d1a5a9d5-06fe-4764-96a7-656a5ae8b7e9/humanMedia(1).mp4"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: ],
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "webhookUrl": "https://www.whjbjm.com/taskCallback/callback/all",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "requestId": "c9615180-3779-4891-b361-e9b6ed3a7471",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "instanceId": "server-1-8196"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 容量更新：6 -> 4
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 队列为空，直接更新算力给后端
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [BridgeManager] 找到空闲实例: server-1-8191, 所属 bridge: bridge-1
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] [BridgeManager] 实例已锁定: server-1-8191, taskId: 7fbc728c-bb16-4d79-9444-ef2d54dd01c2
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 任务直接分配: 7fbc728c-bb16-4d79-9444-ef2d54dd01c2 -> 实例 server-1-8191
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] 收到任务请求, 直接分配实例: server-1-8191, bridgeId: bridge-1, requestId: 7fbc728c-bb16-4d79-9444-ef2d54dd01c2
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [分发] WebSocketServer 发送任务到实例：bridgeId=bridge-1, instanceId=server-1-8191, requestId=7fbc728c-bb16-4d79-9444-ef2d54dd01c2
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [分发] BridgeManager 发送到桥接器 bridge-1: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "type": "TASK_ASSIGN",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "data": {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "workflowId": "2033817068948164610",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeInfoList": [
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeId": "23",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldName": "audio_file",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldValue": "https://www.whjbjm.com/api/file/36367795-67ab-4a15-bbc3-337c0776770b/workflow/853f1501-8a39-406b-b062-a1e24c222f4d/voiceSample.mp3"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: },
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "nodeId": "3",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldName": "prompt",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "fieldValue": "此类设备主要着眼于用户的日常健康管理。它们通过内置传感器持续追踪并记录一系列关键生理指标，如心率、血压、血氧饱和度、体温、呼吸频率以及睡眠质量等。采集到的数据通常由设备自身或配套的应用程序进行处理并呈现，可以帮助用户建立个人健康档案，并为医生提供评估健康状况和筛查潜在风险的参考依据。主要形式为智能手环、智能手表以及各种形式的智能贴片。优势在于佩戴相对无感、数据采集具有连续性且无需侵入性操作。"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: ],
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "webhookUrl": "https://www.whjbjm.com/taskCallback/callback/all",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "requestId": "7fbc728c-bb16-4d79-9444-ef2d54dd01c2",
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: "instanceId": "server-1-8191"
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 任务确认: dba87553-3da1-4759-9439-e732a2d3877f
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] [BridgeManager] 实例锁已确认，设置长超时定时器: server-1-8194
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 任务确认: c9615180-3779-4891-b361-e9b6ed3a7471
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] [BridgeManager] 实例锁已确认，设置长超时定时器: server-1-8196
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [TaskScheduler] 任务确认: 7fbc728c-bb16-4d79-9444-ef2d54dd01c2
|
||||
11|comfyui消息分发 | 2026-04-10 19:21:08 +08:00: [2026-04-10 19:21:08] [INFO] [BridgeManager] 实例锁已确认，设置长超时定时器: server-1-8191
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: [WebSocketServer] 收到 TASK_END 消息： {
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "requestId": "aae1af11-35db-4e07-a242-deec4993d26b",
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "taskId": "aae1af11-35db-4e07-a242-deec4993d26b",
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "instanceId": "server-1-8195",
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "result": [
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: {
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "fileUrl": "https://www.whjbjm.com/api/file/internal/generated/1775820215041_5929b917-5f6a-44fe-a6cb-9da22d800dae_AnimateDiff_00996-audio.mp4",
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "fileType": "output",
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "taskCostTime": 0,
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "nodeId": "81"
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: ],
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: "error": null
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: }
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: [WebSocketServer] 解析参数：requestId=aae1af11-35db-4e07-a242-deec4993d26b, instanceId=server-1-8195
|
||||
11|comfyui消息分发 | 2026-04-10 19:23:35 +08:00: [WebSocketServer] 实例锁释放 失败: server-1-8195
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
root@2F-E3-Server6-04:/project/digitalHuman/taskBackend-v3# node checkQueue
|
||||
|
||||
========================================
|
||||
Redis 队列检测脚本
|
||||
项目前缀: digitalHuman-v3
|
||||
检测时间: 2026/4/20 10:48:40
|
||||
========================================
|
||||
|
||||
Redis 连接成功
|
||||
|
||||
|
||||
========== 检测等待队列 ==========
|
||||
✓ 等待队列 [digitalHuman-v3:comfyui:wait] 为空
|
||||
✓ 等待队列 [digitalHuman-v3:runninghub:wait] 为空
|
||||
⚠️ 等待队列 [digitalHuman-v3:coze:wait] 有 11 个任务积压
|
||||
|
||||
========== 检测处理队列 ==========
|
||||
✓ 处理队列 [digitalHuman-v3:process:Polling] 为空
|
||||
✓ 回调处理队列 [digitalHuman-v3:process:callback] 为空
|
||||
|
||||
========== 检测结果队列 ==========
|
||||
✓ 结果队列 [digitalHuman-v3:result:queue] 为空
|
||||
✓ 结果列表 [digitalHuman-v3:result:list] 为空
|
||||
|
||||
========== 检测错误队列 ==========
|
||||
✓ 错误队列 [digitalHuman-v3:error:queue] 为空
|
||||
⚠️ 错误列表 [digitalHuman-v3:error:list] 有 11 个任务
|
||||
|
||||
========== 检测回调等待任务 ==========
|
||||
发现 2 个回调等待任务
|
||||
✓ 所有回调等待任务都在正常时间内
|
||||
|
||||
========== 检测待处理消息 ==========
|
||||
✓ 没有待处理消息
|
||||
|
||||
========== 检测计数器 ==========
|
||||
|
||||
全局计数器:
|
||||
PQtasksALL (处理队列总任务数): 19
|
||||
RQtasksALL (结果队列总任务数): 0
|
||||
CQtasksALL (回调队列总任务数): 2
|
||||
EQtaskALL (错误队列总任务数): -9
|
||||
|
||||
平台计数器:
|
||||
[digitalHuman-v3:comfyui]
|
||||
WQtasks (等待队列任务数): 0
|
||||
PQtasks (处理队列任务数): 2
|
||||
MAX_CONCURRENT (最大并发数): 0
|
||||
✗ 平台 [digitalHuman-v3:comfyui] 处理队列任务数 (2) 超过最大并发数 (0)
|
||||
[digitalHuman-v3:runninghub]
|
||||
WQtasks (等待队列任务数): 0
|
||||
PQtasks (处理队列任务数): 5
|
||||
MAX_CONCURRENT (最大并发数): 13
|
||||
[digitalHuman-v3:coze]
|
||||
WQtasks (等待队列任务数): 11
|
||||
PQtasks (处理队列任务数): 20
|
||||
MAX_CONCURRENT (最大并发数): 20
|
||||
⚠️ 平台 [digitalHuman-v3:coze] 等待队列计数器不为零: 11
|
||||
⚠️ 全局处理队列计数器不为零: 19
|
||||
⚠️ 全局回调队列计数器不为零: 2
|
||||
|
||||
========== 检测队列与计数器一致性 ==========
|
||||
⚠️ PQtasksALL (19) 与实际队列长度 (0) 不一致
|
||||
✓ CQtasksALL 与实际回调等待数一致
|
||||
✓ 平台 [digitalHuman-v3:comfyui] WQtasks 与实际等待队列长度一致
|
||||
✓ 平台 [digitalHuman-v3:runninghub] WQtasks 与实际等待队列长度一致
|
||||
✓ 平台 [digitalHuman-v3:coze] WQtasks 与实际等待队列长度一致
|
||||
|
||||
========================================
|
||||
检测报告汇总
|
||||
========================================
|
||||
|
||||
共发现 7 个问题:
|
||||
|
||||
|
||||
[wait_queue_backlog] - 1 个
|
||||
- 队列: digitalHuman-v3:coze:wait, 数量: 11
|
||||
|
||||
[error_list_has_tasks] - 1 个
|
||||
- 队列: digitalHuman-v3:error:list, 数量: 11
|
||||
|
||||
[process_counter_exceeds_concurrency] - 1 个
|
||||
- 平台: digitalHuman-v3:comfyui, 数量: 2
|
||||
|
||||
[wait_counter_not_zero] - 1 个
|
||||
- 平台: digitalHuman-v3:coze, 数量: 11
|
||||
|
||||
[global_process_counter_not_zero] - 1 个
|
||||
- {"type":"global_process_counter_not_zero","count":19}
|
||||
|
||||
[global_callback_counter_not_zero] - 1 个
|
||||
- {"type":"global_callback_counter_not_zero","count":2}
|
||||
|
||||
[counter_queue_mismatch] - 1 个
|
||||
- 计数器: PQtasksALL, 计数器值: 19, 实际值: 0
|
||||
|
||||
========== 建议操作 ==========
|
||||
1. 等待队列有积压，建议检查消费者是否正常运行
|
||||
4. 计数器与实际队列不一致，建议重置计数器
|
||||
|
||||
Redis 连接已关闭
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# 项目前缀
|
||||
PROJECT_PREFIX='digitalHuman-test'
|
||||
PROJECT_PREFIX='digitalHuman-v3'
|
||||
|
||||
# token 密钥
|
||||
TOKEN_SECRET='1Ag9BJJn0rXDnidCyXqu'
|
||||
|
|
@ -17,7 +17,7 @@ RunningHub_URL='https://www.runninghub.cn/task/openapi/create'
|
|||
BACKEND_API_URL='http://localhost:8787' # http://www.whjbjm.com/api
|
||||
|
||||
# 回调接口地址
|
||||
CALLBACK_URL='http://43.248.131.153:8087/callback/all'
|
||||
CALLBACK_URL='https://www.whjbjm.com/taskCallback/callback/all'
|
||||
# fNkecvcLonpHtFimE4G1BOjcB82yy4PqiQv9caknQqtQAwT1ZAJeWkG7YjY2YVBP
|
||||
# http://www.whjbjm.com/taskCallback/callback/all
|
||||
|
||||
|
|
@ -29,3 +29,6 @@ MESSAGE_DISPATCHER_URL=http://localhost:8078/api/task
|
|||
MESSAGE_DISPATCHER_WS_PORT=8088
|
||||
MESSAGE_DISPATCHER_ENABLED=true
|
||||
MESSAGE_DISPATCHER_TIMEOUT=30000
|
||||
|
||||
# 回调超时配置 (毫秒，默认10分钟)
|
||||
CALLBACK_TIMEOUT=3600000
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ const assessment = createWorker('./worker_threads/assessment/assessment.js');
|
|||
const wait = createWorker('./worker_threads/wait/waiting.js');
|
||||
const polling = createWorker('./worker_threads/process/process.js');
|
||||
const result = createWorker('./worker_threads/result/result.js');
|
||||
const callback_timeout = createWorker('./worker_threads/callback_timeout/callbackTimeout.js');
|
||||
const callback_result = createWorker('./worker_threads/callback_result/result.js');
|
||||
const error = createWorker('./worker_threads/error/error.js');
|
||||
|
||||
|
|
@ -222,7 +223,7 @@ function createWebSocketServer(server) {
|
|||
const heartbeatInterval = setInterval(() => {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
socket.send('ping');
|
||||
logger.debug(`向 ${id} 号后端发送心跳`);
|
||||
// logger.debug(`向 ${id} 号后端发送心跳`);
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,532 @@
|
|||
import { createClient } from 'redis';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const redis = createClient({
|
||||
RESP: 3,
|
||||
url: process.env.REDIS_URL || 'redis://localhost:16379',
|
||||
password: process.env.REDIS_PASSWORD || '654321',
|
||||
socket: {
|
||||
connectTimeout: 10000,
|
||||
keepAlive: 30000
|
||||
},
|
||||
legacyMode: false,
|
||||
enableReadyCheck: true,
|
||||
maxRetriesPerRequest: 3
|
||||
});
|
||||
|
||||
redis.on('error', (err) => {
|
||||
console.error('Redis 连接错误:', err);
|
||||
});
|
||||
|
||||
const PREFIX = process.env.PROJECT_PREFIX || 'digitalHuman-v3';
|
||||
const CALLBACK_TIMEOUT = parseInt(process.env.CALLBACK_TIMEOUT) || 3600000;
|
||||
|
||||
const QUEUE_NAMES = {
|
||||
initInfo: `${PREFIX}:InitInfo`,
|
||||
processPolling: `${PREFIX}:process:Polling`,
|
||||
processCallback: `${PREFIX}:process:callback`,
|
||||
resultQueue: `${PREFIX}:result:queue`,
|
||||
resultList: `${PREFIX}:result:list`,
|
||||
callback: `${PREFIX}:callback`,
|
||||
errorQueue: `${PREFIX}:error:queue`,
|
||||
errorList: `${PREFIX}:error:list`,
|
||||
pendingMessages: `${PREFIX}:pending:messages`,
|
||||
callbackPending: `${PREFIX}:callback:pending`
|
||||
};
|
||||
|
||||
const AIGC_TYPES = ['digitalHuman-v3'];
|
||||
const PLATFORMS = ['comfyui', 'runninghub', 'coze'];
|
||||
|
||||
const COLORS = {
|
||||
reset: '\x1b[0m',
|
||||
red: '\x1b[31m',
|
||||
green: '\x1b[32m',
|
||||
yellow: '\x1b[33m',
|
||||
blue: '\x1b[34m',
|
||||
cyan: '\x1b[36m'
|
||||
};
|
||||
|
||||
function log(color, ...args) {
|
||||
console.log(COLORS[color], ...args, COLORS.reset);
|
||||
}
|
||||
|
||||
function getWaitQueueName(aigc, platform) {
|
||||
return `${aigc}:${platform}:wait`;
|
||||
}
|
||||
|
||||
async function checkWaitQueues() {
|
||||
log('cyan', '\n========== 检测等待队列 ==========');
|
||||
const issues = [];
|
||||
|
||||
for (const aigc of AIGC_TYPES) {
|
||||
for (const platform of PLATFORMS) {
|
||||
const queueName = getWaitQueueName(aigc, platform);
|
||||
try {
|
||||
const length = await redis.lLen(queueName);
|
||||
if (length > 0) {
|
||||
const issue = `等待队列 [${queueName}] 有 ${length} 个任务积压`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'wait_queue_backlog', queue: queueName, count: length });
|
||||
} else {
|
||||
log('green', `✓ 等待队列 [${queueName}] 为空`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测等待队列 [${queueName}] 失败:`, error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function checkProcessQueues() {
|
||||
log('cyan', '\n========== 检测处理队列 ==========');
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const pollingLength = await redis.lLen(QUEUE_NAMES.processPolling);
|
||||
if (pollingLength > 0) {
|
||||
const issue = `处理队列 [${QUEUE_NAMES.processPolling}] 有 ${pollingLength} 个任务`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'process_queue_has_tasks', queue: QUEUE_NAMES.processPolling, count: pollingLength });
|
||||
} else {
|
||||
log('green', `✓ 处理队列 [${QUEUE_NAMES.processPolling}] 为空`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测处理队列失败:`, error.message);
|
||||
}
|
||||
|
||||
try {
|
||||
const callbackLength = await redis.lLen(QUEUE_NAMES.processCallback);
|
||||
if (callbackLength > 0) {
|
||||
const issue = `回调处理队列 [${QUEUE_NAMES.processCallback}] 有 ${callbackLength} 个任务`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'callback_queue_has_tasks', queue: QUEUE_NAMES.processCallback, count: callbackLength });
|
||||
} else {
|
||||
log('green', `✓ 回调处理队列 [${QUEUE_NAMES.processCallback}] 为空`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测回调处理队列失败:`, error.message);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function checkResultQueues() {
|
||||
log('cyan', '\n========== 检测结果队列 ==========');
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const queueLength = await redis.lLen(QUEUE_NAMES.resultQueue);
|
||||
if (queueLength > 0) {
|
||||
const issue = `结果队列 [${QUEUE_NAMES.resultQueue}] 有 ${queueLength} 个任务`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'result_queue_has_tasks', queue: QUEUE_NAMES.resultQueue, count: queueLength });
|
||||
} else {
|
||||
log('green', `✓ 结果队列 [${QUEUE_NAMES.resultQueue}] 为空`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测结果队列失败:`, error.message);
|
||||
}
|
||||
|
||||
try {
|
||||
const listLength = await redis.lLen(QUEUE_NAMES.resultList);
|
||||
if (listLength > 0) {
|
||||
const issue = `结果列表 [${QUEUE_NAMES.resultList}] 有 ${listLength} 个任务`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'result_list_has_tasks', queue: QUEUE_NAMES.resultList, count: listLength });
|
||||
} else {
|
||||
log('green', `✓ 结果列表 [${QUEUE_NAMES.resultList}] 为空`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测结果列表失败:`, error.message);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function checkErrorQueues() {
|
||||
log('cyan', '\n========== 检测错误队列 ==========');
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const queueLength = await redis.lLen(QUEUE_NAMES.errorQueue);
|
||||
if (queueLength > 0) {
|
||||
const issue = `错误队列 [${QUEUE_NAMES.errorQueue}] 有 ${queueLength} 个任务`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'error_queue_has_tasks', queue: QUEUE_NAMES.errorQueue, count: queueLength });
|
||||
} else {
|
||||
log('green', `✓ 错误队列 [${QUEUE_NAMES.errorQueue}] 为空`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测错误队列失败:`, error.message);
|
||||
}
|
||||
|
||||
try {
|
||||
const listLength = await redis.lLen(QUEUE_NAMES.errorList);
|
||||
if (listLength > 0) {
|
||||
const issue = `错误列表 [${QUEUE_NAMES.errorList}] 有 ${listLength} 个任务`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'error_list_has_tasks', queue: QUEUE_NAMES.errorList, count: listLength });
|
||||
} else {
|
||||
log('green', `✓ 错误列表 [${QUEUE_NAMES.errorList}] 为空`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测错误列表失败:`, error.message);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function checkCallbackPending() {
|
||||
log('cyan', '\n========== 检测回调等待任务 ==========');
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const tasks = await redis.hGetAll(QUEUE_NAMES.callbackPending);
|
||||
const taskEntries = Object.entries(tasks);
|
||||
const now = Date.now();
|
||||
const timeoutTasks = [];
|
||||
|
||||
if (taskEntries.length > 0) {
|
||||
log('blue', `发现 ${taskEntries.length} 个回调等待任务`);
|
||||
|
||||
for (const [remoteTaskId, taskJson] of taskEntries) {
|
||||
try {
|
||||
const task = JSON.parse(taskJson);
|
||||
const age = now - task.createdAt;
|
||||
|
||||
if (age > CALLBACK_TIMEOUT) {
|
||||
const ageMinutes = Math.floor(age / 60000);
|
||||
timeoutTasks.push({
|
||||
remoteTaskId,
|
||||
taskId: task.taskId,
|
||||
aigc: task.aigc,
|
||||
platform: task.platform,
|
||||
ageMinutes,
|
||||
createdAt: new Date(task.createdAt).toLocaleString('zh-CN')
|
||||
});
|
||||
}
|
||||
} catch (parseError) {
|
||||
log('red', `解析任务数据失败: ${remoteTaskId}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (timeoutTasks.length > 0) {
|
||||
log('red', `\n发现 ${timeoutTasks.length} 个超时的回调等待任务:`);
|
||||
for (const task of timeoutTasks) {
|
||||
log('red', ` - 任务ID: ${task.taskId}, 远程ID: ${task.remoteTaskId}, 平台: ${task.aigc}:${task.platform}, 已等待: ${task.ageMinutes}分钟, 创建时间: ${task.createdAt}`);
|
||||
}
|
||||
issues.push({ type: 'callback_pending_timeout', count: timeoutTasks.length, tasks: timeoutTasks });
|
||||
} else {
|
||||
log('green', `✓ 所有回调等待任务都在正常时间内`);
|
||||
}
|
||||
} else {
|
||||
log('green', `✓ 没有回调等待任务`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测回调等待任务失败:`, error.message);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function checkPendingMessages() {
|
||||
log('cyan', '\n========== 检测待处理消息 ==========');
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const messageKeys = await redis.lRange(QUEUE_NAMES.pendingMessages, 0, -1);
|
||||
|
||||
if (messageKeys.length > 0) {
|
||||
const issue = `待处理消息列表有 ${messageKeys.length} 个消息`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
|
||||
const messageDetails = [];
|
||||
for (const key of messageKeys.slice(0, 10)) {
|
||||
try {
|
||||
const data = await redis.hGetAll(key);
|
||||
if (data && data.taskId) {
|
||||
messageDetails.push({
|
||||
key,
|
||||
taskId: data.taskId,
|
||||
retryCount: data.retryCount || 0,
|
||||
timestamp: data.timestamp ? new Date(parseInt(data.timestamp)).toLocaleString('zh-CN') : 'unknown'
|
||||
});
|
||||
}
|
||||
} catch (parseError) {
|
||||
}
|
||||
}
|
||||
|
||||
if (messageDetails.length > 0) {
|
||||
log('yellow', '\n前10个待处理消息:');
|
||||
for (const msg of messageDetails) {
|
||||
log('yellow', ` - 任务ID: ${msg.taskId}, 重试次数: ${msg.retryCount}, 时间: ${msg.timestamp}`);
|
||||
}
|
||||
}
|
||||
|
||||
issues.push({ type: 'pending_messages_backlog', count: messageKeys.length, samples: messageDetails });
|
||||
} else {
|
||||
log('green', `✓ 没有待处理消息`);
|
||||
}
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测待处理消息失败:`, error.message);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function checkCounters() {
|
||||
log('cyan', '\n========== 检测计数器 ==========');
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const initInfo = await redis.json.get(QUEUE_NAMES.initInfo, { path: '$' });
|
||||
|
||||
if (!initInfo || !initInfo[0]) {
|
||||
log('yellow', `⚠️ 未找到初始化信息 [${QUEUE_NAMES.initInfo}]`);
|
||||
issues.push({ type: 'init_info_missing' });
|
||||
return issues;
|
||||
}
|
||||
|
||||
const info = initInfo[0];
|
||||
|
||||
log('blue', '\n全局计数器:');
|
||||
log('blue', ` PQtasksALL (处理队列总任务数): ${info.PQtasksALL || 0}`);
|
||||
log('blue', ` RQtasksALL (结果队列总任务数): ${info.RQtasksALL || 0}`);
|
||||
log('blue', ` CQtasksALL (回调队列总任务数): ${info.CQtasksALL || 0}`);
|
||||
log('blue', ` EQtaskALL (错误队列总任务数): ${info.EQtaskALL || 0}`);
|
||||
|
||||
if (info.platforms) {
|
||||
log('blue', '\n平台计数器:');
|
||||
for (const [key, platform] of Object.entries(info.platforms)) {
|
||||
const wqCount = platform.WQtasks ?? 0;
|
||||
const pqCount = platform.PQtasks ?? 0;
|
||||
const maxConcurrency = platform.MAX_CONCURRENT ?? 0;
|
||||
|
||||
log('blue', ` [${key}]`);
|
||||
log('blue', ` WQtasks (等待队列任务数): ${wqCount}`);
|
||||
log('blue', ` PQtasks (处理队列任务数): ${pqCount}`);
|
||||
log('blue', ` MAX_CONCURRENT (最大并发数): ${maxConcurrency}`);
|
||||
|
||||
if (wqCount < 0) {
|
||||
const issue = `平台 [${key}] 等待队列计数器为负值: ${wqCount}`;
|
||||
log('red', `✗ ${issue}`);
|
||||
issues.push({ type: 'wait_counter_negative', platform: key, count: wqCount });
|
||||
} else if (wqCount > 0) {
|
||||
const issue = `平台 [${key}] 等待队列计数器不为零: ${wqCount}`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'wait_counter_not_zero', platform: key, count: wqCount });
|
||||
}
|
||||
|
||||
if (pqCount < 0) {
|
||||
const issue = `平台 [${key}] 处理队列计数器为负值: ${pqCount}`;
|
||||
log('red', `✗ ${issue}`);
|
||||
issues.push({ type: 'process_counter_negative', platform: key, count: pqCount });
|
||||
} else if (pqCount > maxConcurrency) {
|
||||
const issue = `平台 [${key}] 处理队列任务数 (${pqCount}) 超过最大并发数 (${maxConcurrency})`;
|
||||
log('red', `✗ ${issue}`);
|
||||
issues.push({ type: 'process_counter_exceeds_concurrency', platform: key, count: pqCount, max: maxConcurrency });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (info.PQtasksALL > 0) {
|
||||
const issue = `全局处理队列计数器不为零: ${info.PQtasksALL}`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'global_process_counter_not_zero', count: info.PQtasksALL });
|
||||
} else if (info.PQtasksALL < 0) {
|
||||
const issue = `全局处理队列计数器为负值: ${info.PQtasksALL}`;
|
||||
log('red', `✗ ${issue}`);
|
||||
issues.push({ type: 'global_process_counter_negative', count: info.PQtasksALL });
|
||||
}
|
||||
|
||||
if (info.CQtasksALL > 0) {
|
||||
const issue = `全局回调队列计数器不为零: ${info.CQtasksALL}`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({ type: 'global_callback_counter_not_zero', count: info.CQtasksALL });
|
||||
} else if (info.CQtasksALL < 0) {
|
||||
const issue = `全局回调队列计数器为负值: ${info.CQtasksALL}`;
|
||||
log('red', `✗ ${issue}`);
|
||||
issues.push({ type: 'global_callback_counter_negative', count: info.CQtasksALL });
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测计数器失败:`, error.message);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function checkQueueCounterConsistency() {
|
||||
log('cyan', '\n========== 检测队列与计数器一致性 ==========');
|
||||
const issues = [];
|
||||
|
||||
try {
|
||||
const initInfo = await redis.json.get(QUEUE_NAMES.initInfo, { path: '$' });
|
||||
if (!initInfo || !initInfo[0]) {
|
||||
return issues;
|
||||
}
|
||||
|
||||
const info = initInfo[0];
|
||||
|
||||
const actualPollingLength = await redis.lLen(QUEUE_NAMES.processPolling);
|
||||
const actualCallbackLength = await redis.lLen(QUEUE_NAMES.processCallback);
|
||||
|
||||
const expectedPQtasksALL = actualPollingLength + actualCallbackLength;
|
||||
|
||||
if (info.PQtasksALL !== expectedPQtasksALL) {
|
||||
const issue = `PQtasksALL (${info.PQtasksALL}) 与实际队列长度 (${expectedPQtasksALL}) 不一致`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({
|
||||
type: 'counter_queue_mismatch',
|
||||
counter: 'PQtasksALL',
|
||||
counterValue: info.PQtasksALL,
|
||||
actualValue: expectedPQtasksALL
|
||||
});
|
||||
} else {
|
||||
log('green', `✓ PQtasksALL 与实际队列长度一致`);
|
||||
}
|
||||
|
||||
const actualCallbackPending = await redis.hLen(QUEUE_NAMES.callbackPending);
|
||||
if (info.CQtasksALL !== actualCallbackPending) {
|
||||
const issue = `CQtasksALL (${info.CQtasksALL}) 与实际回调等待数 (${actualCallbackPending}) 不一致`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({
|
||||
type: 'counter_queue_mismatch',
|
||||
counter: 'CQtasksALL',
|
||||
counterValue: info.CQtasksALL,
|
||||
actualValue: actualCallbackPending
|
||||
});
|
||||
} else {
|
||||
log('green', `✓ CQtasksALL 与实际回调等待数一致`);
|
||||
}
|
||||
|
||||
if (info.platforms) {
|
||||
for (const [key, platform] of Object.entries(info.platforms)) {
|
||||
if (platform.waitQueue) {
|
||||
const actualWaitQueueLength = await redis.lLen(platform.waitQueue);
|
||||
const counterWQtasks = platform.WQtasks ?? 0;
|
||||
|
||||
if (counterWQtasks !== actualWaitQueueLength) {
|
||||
const issue = `平台 [${key}] WQtasks (${counterWQtasks}) 与实际等待队列长度 (${actualWaitQueueLength}) 不一致`;
|
||||
log('yellow', `⚠️ ${issue}`);
|
||||
issues.push({
|
||||
type: 'wait_queue_counter_mismatch',
|
||||
platform: key,
|
||||
counter: 'WQtasks',
|
||||
counterValue: counterWQtasks,
|
||||
actualValue: actualWaitQueueLength
|
||||
});
|
||||
} else {
|
||||
log('green', `✓ 平台 [${key}] WQtasks 与实际等待队列长度一致`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log('red', `✗ 检测一致性失败:`, error.message);
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function generateReport(allIssues) {
|
||||
log('cyan', '\n========================================');
|
||||
log('cyan', ' 检测报告汇总');
|
||||
log('cyan', '========================================');
|
||||
|
||||
const totalIssues = allIssues.flat().length;
|
||||
|
||||
if (totalIssues === 0) {
|
||||
log('green', '\n✓ 没有发现任何问题，队列状态正常！');
|
||||
return;
|
||||
}
|
||||
|
||||
log('yellow', `\n共发现 ${totalIssues} 个问题:\n`);
|
||||
|
||||
const issueTypes = {};
|
||||
for (const issues of allIssues) {
|
||||
for (const issue of issues) {
|
||||
if (!issueTypes[issue.type]) {
|
||||
issueTypes[issue.type] = [];
|
||||
}
|
||||
issueTypes[issue.type].push(issue);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [type, typeIssues] of Object.entries(issueTypes)) {
|
||||
log('yellow', `\n[${type}] - ${typeIssues.length} 个`);
|
||||
for (const issue of typeIssues) {
|
||||
if (issue.queue) {
|
||||
log('yellow', ` - 队列: ${issue.queue}, 数量: ${issue.count}`);
|
||||
} else if (issue.platform) {
|
||||
log('yellow', ` - 平台: ${issue.platform}, 数量: ${issue.count || 'N/A'}`);
|
||||
} else if (issue.counter) {
|
||||
log('yellow', ` - 计数器: ${issue.counter}, 计数器值: ${issue.counterValue}, 实际值: ${issue.actualValue}`);
|
||||
} else {
|
||||
log('yellow', ` - ${JSON.stringify(issue)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log('cyan', '\n========== 建议操作 ==========');
|
||||
|
||||
if (issueTypes.wait_queue_backlog) {
|
||||
log('blue', '1. 等待队列有积压，建议检查消费者是否正常运行');
|
||||
}
|
||||
|
||||
if (issueTypes.process_queue_has_tasks || issueTypes.callback_queue_has_tasks) {
|
||||
log('blue', '2. 处理队列有任务，建议检查任务处理逻辑是否卡住');
|
||||
}
|
||||
|
||||
if (issueTypes.callback_pending_timeout) {
|
||||
log('blue', '3. 有超时的回调等待任务，建议手动清理或重新处理');
|
||||
}
|
||||
|
||||
if (issueTypes.counter_queue_mismatch) {
|
||||
log('blue', '4. 计数器与实际队列不一致，建议重置计数器');
|
||||
}
|
||||
|
||||
if (issueTypes.pending_messages_backlog) {
|
||||
log('blue', '5. 有待处理消息积压，建议检查消息发送逻辑');
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('\n========================================');
|
||||
console.log(' Redis 队列检测脚本');
|
||||
console.log(' 项目前缀:', PREFIX);
|
||||
console.log(' 检测时间:', new Date().toLocaleString('zh-CN'));
|
||||
console.log('========================================\n');
|
||||
|
||||
try {
|
||||
await redis.connect();
|
||||
log('green', 'Redis 连接成功\n');
|
||||
|
||||
const allIssues = [];
|
||||
|
||||
allIssues.push(await checkWaitQueues());
|
||||
allIssues.push(await checkProcessQueues());
|
||||
allIssues.push(await checkResultQueues());
|
||||
allIssues.push(await checkErrorQueues());
|
||||
allIssues.push(await checkCallbackPending());
|
||||
allIssues.push(await checkPendingMessages());
|
||||
allIssues.push(await checkCounters());
|
||||
allIssues.push(await checkQueueCounterConsistency());
|
||||
|
||||
await generateReport(allIssues);
|
||||
|
||||
} catch (error) {
|
||||
log('red', '\n检测过程出错:', error.message);
|
||||
console.error(error);
|
||||
} finally {
|
||||
await redis.disconnect();
|
||||
log('blue', '\nRedis 连接已关闭');
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import redis from './redis/index.js';
|
||||
|
||||
const REDIS_COZE_TOKEN_KEY = 'coze:api:token';
|
||||
const REDIS_COZE_EXPIRE_KEY = 'coze:api:expireTime';
|
||||
|
||||
console.log('=== 清除 Coze Redis Token 缓存 ===\n');
|
||||
|
||||
async function clearCozeTokens() {
|
||||
try {
|
||||
console.log('正在连接 Redis...');
|
||||
|
||||
console.log('清除 token 键:', REDIS_COZE_TOKEN_KEY);
|
||||
const tokenDeleted = await redis.del(REDIS_COZE_TOKEN_KEY);
|
||||
console.log(` token 删除结果: ${tokenDeleted}`);
|
||||
|
||||
console.log('清除过期时间键:', REDIS_COZE_EXPIRE_KEY);
|
||||
const expireDeleted = await redis.del(REDIS_COZE_EXPIRE_KEY);
|
||||
console.log(` 过期时间删除结果: ${expireDeleted}`);
|
||||
|
||||
console.log('\n✓ Coze token 缓存已成功清除');
|
||||
console.log('\n下次请求时会自动重新生成新的 token');
|
||||
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('\n✗ 清除 token 缓存失败:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
clearCozeTokens();
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
|
||||
import dotenv from 'dotenv';
|
||||
import { createClient } from 'redis';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const prefix = process.env.PROJECT_PREFIX || 'default';
|
||||
const initInfoKey = `${prefix}:InitInfo`;
|
||||
|
||||
const redis = createClient({
|
||||
url: process.env.REDIS_URL || 'redis://localhost:6379'
|
||||
});
|
||||
|
||||
async function clearOldPlatforms() {
|
||||
try {
|
||||
console.log('正在连接Redis...');
|
||||
await redis.connect();
|
||||
console.log('Redis连接成功\n');
|
||||
|
||||
const initInfoResult = await redis.json.get(initInfoKey, { path: '$' });
|
||||
|
||||
if (!initInfoResult || !initInfoResult[0]) {
|
||||
console.log('未找到初始化信息');
|
||||
return;
|
||||
}
|
||||
|
||||
const initInfo = initInfoResult[0];
|
||||
const targetAIGC = 'digitalHuman-v3';
|
||||
|
||||
console.log('检查平台数据...');
|
||||
|
||||
const multi = redis.multi();
|
||||
const waitQueuesToDelete = [];
|
||||
const validWaitQueues = [];
|
||||
|
||||
for (const [key, platform] of Object.entries(initInfo.platforms)) {
|
||||
if (platform.AIGC !== targetAIGC) {
|
||||
console.log(` 删除旧平台: ${key}`);
|
||||
multi.json.del(initInfoKey, `$.platforms.${key}`);
|
||||
waitQueuesToDelete.push(platform.waitQueue);
|
||||
} else {
|
||||
validWaitQueues.push(platform.waitQueue);
|
||||
}
|
||||
}
|
||||
|
||||
if (waitQueuesToDelete.length > 0) {
|
||||
console.log(`\n删除旧平台的等待队列 (${waitQueuesToDelete.length} 个):`);
|
||||
for (const queue of waitQueuesToDelete) {
|
||||
console.log(` - ${queue}`);
|
||||
await redis.del(queue);
|
||||
}
|
||||
}
|
||||
|
||||
multi.json.set(initInfoKey, '$.waitQueues', validWaitQueues);
|
||||
|
||||
await multi.exec();
|
||||
|
||||
console.log('\n✅ 旧平台数据已清空！');
|
||||
console.log(` 保留的平台: ${targetAIGC}`);
|
||||
console.log(` 保留的等待队列: ${validWaitQueues.length} 个`);
|
||||
|
||||
} catch (error) {
|
||||
console.error('清空旧平台失败:', error);
|
||||
} finally {
|
||||
if (redis.isOpen) {
|
||||
await redis.disconnect();
|
||||
console.log('\nRedis连接已关闭');
|
||||
}
|
||||
process.exit();
|
||||
}
|
||||
}
|
||||
|
||||
clearOldPlatforms();
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
{
|
||||
"callback": [
|
||||
"comfyui"
|
||||
"comfyui",
|
||||
"runninghub"
|
||||
],
|
||||
"polling": [
|
||||
"coze"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
{
|
||||
"comfyui": 0.0012
|
||||
"comfyui": 0.0012,
|
||||
"runninghub": 0.0012
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
{
|
||||
"digitalHuman-test":{
|
||||
"digitalHuman-v3":{
|
||||
"comfyui":{
|
||||
"apikey":"3c20cd6c85514d1c86d55a5d3bcd53b7",
|
||||
"concurrency":0
|
||||
},
|
||||
"runninghub":{
|
||||
"apikey":"3c20cd6c85514d1c86d55a5d3bcd53b7",
|
||||
"concurrency":13
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
module.exports = {
|
||||
apps: [{
|
||||
name: 'digitalHuman-callbackTask-v2',
|
||||
name: 'digitalHuman-callbackTask-v3',
|
||||
script: './index.js',
|
||||
cwd: './',
|
||||
args: '',
|
||||
|
|
@ -9,7 +9,7 @@ dotenv.config();
|
|||
|
||||
const server = express(); // 接口url
|
||||
const hostname = '0.0.0.0'; // IP地址
|
||||
const port = process.env.CALLBACK_PORT || 8060; // 端口号
|
||||
const port = process.env.CALLBACK_PORT || 8089; // 端口号
|
||||
server.use(cors()); // 允许跨域
|
||||
|
||||
//设置静态资源路径
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
module.exports = {
|
||||
apps: [{
|
||||
name: 'digitalHuman-md-v3',
|
||||
script: './md.js',
|
||||
cwd: './',
|
||||
args: '',
|
||||
interpreter: 'node',
|
||||
interpreter_args: '',
|
||||
|
||||
// 监听文件修改
|
||||
watch: true,
|
||||
ignore_watch: ['config', 'logs', 'node_modules', 'redis', 'school', 'static', 'worker_threads',
|
||||
'.env', 'package.json', 'package-lock.json', 'pnpm-lock.yaml', 'webSocket.js',
|
||||
'outside/outPlatforms', 'outside/generat.js', 'outside/polling.js', 'outside/record.js',
|
||||
'pm2Index.config.cjs','pm2Websocket.config.cjs'],
|
||||
|
||||
// 实例数
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
|
||||
// 自动重启设置
|
||||
autorestart: true,
|
||||
max_restarts: 30,
|
||||
min_uptime: '10s',
|
||||
|
||||
// 内存限制重启
|
||||
// max_memory_restart: '1G',
|
||||
|
||||
// 日志配置
|
||||
out_file: './logs/md/out/out.log',
|
||||
error_file: './logs/md/error/error.log',
|
||||
// log_file: './logs/combined.log',
|
||||
log_type: 'raw', // 或 'json'
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
logrotate: {
|
||||
max_size: '5M', // 日志文件最大大小
|
||||
retain: 30, // 保留最近7天的日志
|
||||
compress: true, // 压缩旧日志
|
||||
date_format: 'YYYY-MM-DD' // 日期格式
|
||||
},
|
||||
|
||||
// 合并日志
|
||||
// combine_logs: true,
|
||||
|
||||
// 监控和重启设置
|
||||
kill_timeout: 1600,
|
||||
restart_delay: 4000,
|
||||
}],
|
||||
};
|
||||
|
|
@ -3,41 +3,34 @@ import redis from '../redis/index.js'
|
|||
import initQueue from '../redis/initQueue.js'
|
||||
const router = Router()
|
||||
|
||||
// 添加API路由来处理文件上传
|
||||
router.post('/all', async (req, res) => {
|
||||
// 立即返回响应，避免平台超时
|
||||
res.status(200).json({ success: true, message: 'Received' });
|
||||
// // console.log('callback',req.body)
|
||||
processCallbackData(req.body).catch(error => {
|
||||
console.error('处理回调数据时出错:', error);
|
||||
console.error('[callback] 处理回调数据出错:', error);
|
||||
});
|
||||
})
|
||||
|
||||
/**
|
||||
* 异步处理回调数据
|
||||
*/
|
||||
async function processCallbackData(body) {
|
||||
let remoteTaskId, eventData
|
||||
remoteTaskId = body.taskId
|
||||
eventData = body.eventData
|
||||
const { taskId: remoteTaskId, eventData } = body;
|
||||
|
||||
if (!remoteTaskId) {
|
||||
console.error('[callback] 回调数据缺少 taskId');
|
||||
return;
|
||||
}
|
||||
|
||||
// 通过remoteTaskId查询对应的taskId
|
||||
const taskId = await redis.get(`${initQueue.callback}:${remoteTaskId}`);
|
||||
|
||||
if (taskId) {
|
||||
// 将eventData存储到redis的数据里，使用taskId作为键
|
||||
const taskKey = `${initQueue.prefix}:task:${taskId}`;
|
||||
await redis.hSet(taskKey, 'resultData', eventData);
|
||||
|
||||
// 增加值到回调结果队列列表
|
||||
await redis.rPush(initQueue.callback, taskId);
|
||||
await initQueue.addCallbackRQtasks(1);
|
||||
console.log('taskKey:', taskKey);
|
||||
console.log('数据已保存到 Redis:', eventData);
|
||||
await initQueue.removeCallbackPendingTask(remoteTaskId);
|
||||
|
||||
console.log(`[callback] 回调处理成功: taskId=${taskId}, externalTaskId=${remoteTaskId}`);
|
||||
} else {
|
||||
console.error('未找到对应的taskId，remoteTaskId:', remoteTaskId);
|
||||
// 可以考虑将未找到的remoteTaskId记录下来，以便后续分析
|
||||
await redis.set(`callback:missing:${remoteTaskId}`, JSON.stringify(body), { EX: 86400 }); // 保存24小时
|
||||
console.error(`[callback] 未找到任务映射: externalTaskId=${remoteTaskId}`);
|
||||
await redis.set(`callback:missing:${remoteTaskId}`, JSON.stringify(body), { EX: 86400 });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,93 +1,49 @@
|
|||
import {modelData} from '../config/Config.js';
|
||||
import outside from './outPlatforms/outside.js'
|
||||
|
||||
// 发送请求
|
||||
export async function externalPostRequest(task, jwtToken = null) {
|
||||
const platform = task.platformName
|
||||
const AIGC = process.env.PROJECT_PREFIX
|
||||
|
||||
// 获取分发标识，默认为 'runninghub'
|
||||
const dispatchType = task.dispatchType || 'runninghub';
|
||||
|
||||
console.log(`[externalPostRequest] 任务分发 - 平台: ${platform}, 分发标识: ${dispatchType}`);
|
||||
let dispatchType = task.dispatchType || 'runninghub';
|
||||
|
||||
const apikey = modelData[AIGC]?.[platform]?.apikey || '';
|
||||
|
||||
let response;
|
||||
let success = false;
|
||||
|
||||
try {
|
||||
// 对于 comfyui 平台，使用分发标识来调用相应的接口
|
||||
const headers = outside[platform].getGenerateHeader(apikey, dispatchType, jwtToken);
|
||||
const headers = await outside[platform].getGenerateHeader(apikey, dispatchType, jwtToken);
|
||||
const url = outside[platform].getGenerateUrl(dispatchType);
|
||||
const body = outside[platform].getGenerateBody({payload: task.taskData, apikey}, dispatchType, jwtToken);
|
||||
|
||||
console.log(`[externalPostRequest] 发送请求到 ${platform} (${dispatchType}): ${url}`);
|
||||
|
||||
response = await fetch(url, { method: 'POST', headers, body: body });
|
||||
|
||||
// 检查响应状态
|
||||
if (!response.ok) {
|
||||
throw new Error(`外部平台返回错误状态: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
success = true;
|
||||
} catch (error) {
|
||||
// 如果是 comfyui 平台且使用了 messageDispatcher 分发，尝试降级到 runninghub
|
||||
if (platform === 'comfyui' && dispatchType === 'messageDispatcher') {
|
||||
console.warn('[externalPostRequest] messageDispatcher 分发失败，降级使用 runninghub:', error.message);
|
||||
|
||||
const fallbackDispatchType = 'runninghub';
|
||||
|
||||
try {
|
||||
const headers = outside[platform].getGenerateHeader(apikey, fallbackDispatchType, jwtToken);
|
||||
const url = outside[platform].getGenerateUrl(fallbackDispatchType);
|
||||
const body = outside[platform].getGenerateBody({payload: task.taskData, apikey}, fallbackDispatchType, jwtToken);
|
||||
|
||||
console.log(`[externalPostRequest] 降级发送请求到 ${platform} (${fallbackDispatchType}): ${url}`);
|
||||
|
||||
response = await fetch(url, { method: 'POST', headers, body: body });
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`降级平台返回错误状态: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
success = true;
|
||||
} catch (fallbackError) {
|
||||
console.error('[externalPostRequest] 降级也失败:', fallbackError.message);
|
||||
return {
|
||||
taskId: task.taskId,
|
||||
remoteTaskId: { type: 2, message: `请求失败: ${fallbackError.message}` },
|
||||
platform,
|
||||
AIGC
|
||||
};
|
||||
}
|
||||
} else {
|
||||
console.error('[externalPostRequest] 外部请求失败:', error);
|
||||
return {
|
||||
taskId: task.taskId,
|
||||
remoteTaskId: { type: 2, message: `外部请求失败: ${error.message}` },
|
||||
platform,
|
||||
AIGC
|
||||
};
|
||||
}
|
||||
console.error(`[externalPostRequest] 请求失败: taskId=${task.taskId}, error=${error.message}`);
|
||||
return {
|
||||
taskId: task.taskId,
|
||||
remoteTaskId: { type: 2, message: '算力维护中' },
|
||||
platform,
|
||||
AIGC
|
||||
};
|
||||
}
|
||||
|
||||
// 处理成功响应
|
||||
try {
|
||||
const successResult = await outside[platform].getSuccessTasks(response, dispatchType);
|
||||
console.log(`[externalPostRequest] ${platform} 响应:`, successResult);
|
||||
|
||||
let remoteTaskId;
|
||||
if (successResult.type === 2) {
|
||||
remoteTaskId = successResult;
|
||||
console.error(`[externalPostRequest] 任务失败: taskId=${task.taskId}, result=`, successResult);
|
||||
} else {
|
||||
remoteTaskId = { type: 1, data: successResult };
|
||||
}
|
||||
|
||||
return { taskId: task.taskId, remoteTaskId, platform, AIGC, workflowId: task.workflowId };
|
||||
} catch (parseError) {
|
||||
console.error('[externalPostRequest] 解析响应失败:', parseError);
|
||||
console.error(`[externalPostRequest] 解析响应失败: taskId=${task.taskId}, error=${parseError.message}`);
|
||||
return {
|
||||
taskId: task.taskId,
|
||||
remoteTaskId: { type: 2, message: `解析响应失败: ${parseError.message}` },
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ export function getGenerateBody(task, dispatchType = DISPATCH_TYPES.RUNNINGHUB,
|
|||
|
||||
if (dispatchType === DISPATCH_TYPES.MESSAGEDISPATCHER) {
|
||||
const payload = { ...taskData, apiKey: jwtToken, webhookUrl: process.env.CALLBACK_URL };
|
||||
console.log('[comfyui - messageDispatcher] 请求体:', payload);
|
||||
return JSON.stringify(payload);
|
||||
}
|
||||
|
||||
|
|
@ -51,23 +50,23 @@ export async function getSuccessTasks(response, dispatchType = DISPATCH_TYPES.RU
|
|||
if (dispatchType === DISPATCH_TYPES.MESSAGEDISPATCHER) {
|
||||
try {
|
||||
const res = await response.json();
|
||||
console.log('[comfyui - messageDispatcher] 响应:\n', res);
|
||||
if (res.success === true && res.data && res.data.requestId) {
|
||||
return { msg: 'success', code: 0, data: { taskId: res.data.requestId } };
|
||||
} else {
|
||||
console.error('[comfyui-messageDispatcher] 返回错误:', res);
|
||||
return { message: res, type: 2 };
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[comfyui - messageDispatcher] 解析响应失败:', error);
|
||||
console.error('[comfyui-messageDispatcher] 解析响应失败:', error);
|
||||
return { message: error.message, type: 2 };
|
||||
}
|
||||
}
|
||||
|
||||
const res = await response.json();
|
||||
console.log('[comfyui - runninghub]:\n', res);
|
||||
if (res.msg === 'success' && res.code === 0) {
|
||||
return res.data.taskId;
|
||||
} else {
|
||||
console.error('[comfyui-runninghub] 返回错误:', res);
|
||||
return { message: res, type: 2 };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ const __dirname = path.dirname(__filename);
|
|||
// Coze配置
|
||||
const COZE_OAUTH_CONFIG = {
|
||||
appId: '1172420148562',
|
||||
kid: 'noU76VVvKw679eiyjwHUZLcU2zDwKtSD6N-rOsPIwe0',
|
||||
kid: 'Ws7hstppWE5xn-UcycITzOLCFyE5gWJSUqfDJQb0rJc',
|
||||
privateKeyPath: path.join(__dirname, 'private_key.pem')
|
||||
};
|
||||
|
||||
|
|
@ -81,27 +81,47 @@ async function getOAuthToken(jwtToken, durationSeconds = 3600) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清除Redis中缓存的无效token
|
||||
*/
|
||||
async function clearInvalidToken() {
|
||||
try {
|
||||
console.log('清除Coze无效token缓存...');
|
||||
await Promise.all([
|
||||
redis.del(REDIS_COZE_TOKEN_KEY),
|
||||
redis.del(REDIS_COZE_EXPIRE_KEY)
|
||||
]);
|
||||
console.log('Coze token缓存已清除');
|
||||
} catch (error) {
|
||||
console.error('清除Coze token缓存失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取有效的API密钥，过期自动刷新
|
||||
* @param {boolean} forceRefresh - 是否强制刷新token
|
||||
* @returns {Promise<string>} 有效的API密钥
|
||||
*/
|
||||
async function getValidApiKey() {
|
||||
async function getValidApiKey(forceRefresh = false) {
|
||||
const now = Date.now();
|
||||
|
||||
try {
|
||||
// 1. 从Redis获取当前token和过期时间
|
||||
const [storedToken, storedExpireTime] = await Promise.all([
|
||||
redis.get(REDIS_COZE_TOKEN_KEY),
|
||||
redis.get(REDIS_COZE_EXPIRE_KEY)
|
||||
]);
|
||||
|
||||
// 2. 如果token存在且未过期（5分钟内），则直接返回
|
||||
if (storedToken && storedExpireTime && now < parseInt(storedExpireTime) - 5 * 60 * 1000) {
|
||||
return storedToken;
|
||||
if (!forceRefresh) {
|
||||
// 1. 从Redis获取当前token和过期时间
|
||||
const [storedToken, storedExpireTime] = await Promise.all([
|
||||
redis.get(REDIS_COZE_TOKEN_KEY),
|
||||
redis.get(REDIS_COZE_EXPIRE_KEY)
|
||||
]);
|
||||
|
||||
// 2. 如果token存在且未过期（5分钟内），则直接返回
|
||||
if (storedToken && storedExpireTime && now < parseInt(storedExpireTime) - 5 * 60 * 1000) {
|
||||
// console.log('使用缓存的Coze API密钥');
|
||||
return storedToken;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 否则重新生成token
|
||||
console.log('Coze API密钥已过期或不存在，重新生成...');
|
||||
console.log('Coze API密钥已过期、不存在或强制刷新，重新生成...');
|
||||
|
||||
const jwtToken = await generateCozeJWT();
|
||||
const tokenResponse = await getOAuthToken(jwtToken);
|
||||
|
|
@ -191,6 +211,35 @@ async function getQueryHeader(apikey = null) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理提交任务的响应，提取任务ID
|
||||
* @param {Response} response - fetch响应对象
|
||||
* @returns {Promise<string|Object>} 任务ID或错误对象
|
||||
*/
|
||||
async function getSuccessTasks(response) {
|
||||
try {
|
||||
const res = await response.json();
|
||||
// console.log('Coze 提交任务响应:', res);
|
||||
|
||||
if (res.code === 0) {
|
||||
const executeId = res.execute_id;
|
||||
if (executeId) {
|
||||
console.log('Coze 任务提交成功，execute_id:', executeId);
|
||||
return executeId;
|
||||
} else {
|
||||
console.error('[coze] 响应中缺少 execute_id:', res);
|
||||
return { message: '响应中缺少 execute_id', type: 2 };
|
||||
}
|
||||
} else {
|
||||
console.error('[coze] 返回错误:', res);
|
||||
return { message: res.msg || res.error_message || 'API调用失败', type: 2 };
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[coze] 解析响应失败:', error);
|
||||
return { message: error.message, type: 2 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理查询响应，判断任务状态
|
||||
* @param {Object} response - 外部平台返回的响应数据
|
||||
|
|
@ -262,5 +311,8 @@ export default {
|
|||
getGenerateBody,
|
||||
getQueryUrl,
|
||||
getQueryHeader,
|
||||
getTaskStatus
|
||||
getTaskStatus,
|
||||
getSuccessTasks,
|
||||
clearInvalidToken,
|
||||
getValidApiKey
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,28 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwjpkxuX8ZQkAJ
|
||||
4rKO2DokUQtS0WZr3HLmJNCZrPRMc4tkXD4TcC1VY3sb03kYq93Beu51x22sxm0n
|
||||
fAbLQ3a2DtbHhN2oXwRCfNGF1NaKUsQVQo6Mc6Al0kFuzEFpaDte8hLMoWVg5bnK
|
||||
ill8utHjRTseq9AuDP36zf+amPG5yk77Wi+hjOmv7un7kRhK9D51qBSovsYClOg5
|
||||
1VtAbizb6vHHaI0b4xCwq9PPqfcK4B0Nj62LF/IFT4U48yozUm8mD7OI8G5JEsRu
|
||||
ISJ9ziGsymtSNu+go33YvI6P9ihC1pBSDNl+7pl4MvCmhTXMZRVPhtfDAhOF4O8o
|
||||
/P2z97FRAgMBAAECggEAJ3LFEdIjbs5ZppvLT5VKcGDXSdrVqpXn6johjaSSNR6/
|
||||
712Y1RkEWAbRM+dtMDD+bEN+UjyL6cWwD9lrXzEkrgrkvFGYgQ0x03U2D1P914wk
|
||||
mad0WDdhefHfgtUKbHXIhi9KOgR5tUu+1l1RH0hSqxgF3JWA/zkR6l7qlG1F3T/S
|
||||
nNjHV2I5z+DzYY1nK+4VXQjECEBJQT0f6D6uJHIXE1kGz4i/NHdbxuJ7d7ey9TkO
|
||||
7PY+p5WEjQKOyXwjTFf84I4TAM5TRD0GxtLzys7ZWP6yOfvWPugbk3/znRyS/Rg3
|
||||
KR4xz7yxRoMCbRcWxJoMoAQnNd4pF4qnDqeO5WQirQKBgQDZXG1v+Jao6ZZTlscY
|
||||
WceVKvr1m+nWLNbWtBsaaU3tDepb2N7YE2O8r/gM6twxqrGs038dZRCAtvmx7ntP
|
||||
MFSOpUlJSIYqK1ANr/uxlGfQF76B1Ht+lMcZ6h72v73jzc0Y2U7OcD+/VrxZMABo
|
||||
1NOyXfatpYqRzWG1ks8dqT2EfwKBgQDP8Ud7G3bM4Tif3E4RwE5CDdEHmv75WpZY
|
||||
VqvSv8zHBrJz1KLekYjsAtSjavIxeROZich4jME2RmVsa+52WuPXXhcSGwx4RmKp
|
||||
zuHKHOj/41VjYvKPim4MWbAxVA3RSVyKT+5vy1AqTZWxzse6JL8eD3ZokCop6Pax
|
||||
DvTxEiyiLwKBgBuTCCceSC6hg3qTNCq4qQMZcsDZyK5s/cw7CP0uwr4B9+sy9gI/
|
||||
Y3W6dSNeYBTE7MlaA1Q9T/ykOcUC1g3TucZm3Yc4dhy/ZeZ2nt2GUC0r9fUOeaQz
|
||||
R5bYBpmS9YoCv7QZTVAPGWcyn65I0qR562lDVlntGEkq3uxj9XZz0+QNAoGAUifs
|
||||
6vm13UqamaZr/d1xze0xigS1+oTM48gSiOiYmoXN2a/ITZFIfJ69rnchi2Rf1wi1
|
||||
+NL7v1re1ZBrHb3ZSQz2poOjUJ3We2qukLENaZRC90pvtUCnLB//We3wq6CFfGwK
|
||||
M4crfBs9KowdIzFDhTfsu3FCB17woJHdOqXIlqcCgYEAv3hYuS4FV+NVr1yUocib
|
||||
aPQmY8/LwaUQsoiAcfWPr5cieghiqmPxTDMScMjN9sM8VeaJczRfvoyoHrXLwLa2
|
||||
U0CXgYACDwpaFjQPIENKEqBjv8E/t/OfggqkpGbWdyAPSFubnsGqYf2XSrzcb2aX
|
||||
I/oPe484hWWwA+NV+sTjQME=
|
||||
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCpVBYP1iFOheni
|
||||
BswkioTmFvb3ImS9eG7ksNf4UZLFz1aWUcpdnLGf2YTxBb5DpMpNUz1b95f/zyKo
|
||||
uXCuqRIe6tZLuMe1ExsBugHHOPshLt2eSnfqES+I0CkF2gsBV0p3DfichLOlLR56
|
||||
FbUkg+ACYUy3Z1iamRcJTrCyWO4y0seM9ntucoojEgGJIHh2NDNB5QE+VWSxY3EG
|
||||
5lW1h4sGn0ktK4SHDvolUnkgXzKIK9L8vuYAciWZpZrftt4Fnz1xCd79jKS6GsZM
|
||||
B5iUPfFa3LD4nRdLXlIOEcgpRe40BOkEfPn0TujmLpI1xqucwtSC5LTNVzR8JAxy
|
||||
1/oslOJ7AgMBAAECggEADIZHMXmtsgCuuYS2Mde7F8fIEaSOHgOb9DduahQijpYU
|
||||
+f5nwP/XlZLBxhAeMCOuwQRwq+5G5Gl+T0uUHZ8s03OcUTKENw0IONr4ybMZxC0S
|
||||
wGQStZH62ZxIBIJJwRojTGYQpwcbwJJzSCV15hMJDTQEOCahQ6IiNZKH+FEioUid
|
||||
5jn7qTDWF7nRBOhNgTDoLAMOfZc2mRTX7WzykCFr820X6p3ZXX5wQLDZtFWTr5nA
|
||||
kBqHoGcOTWW908QfBccGotU1jkoujtAFTbliGJrZvfk8Yf8iM2odu8u+nbBryfRI
|
||||
dhr6B4uKtQRYv9XcUgxAXZVHB68lc6aWehd2fwy3DQKBgQDr+wYLO3J3GDprGweK
|
||||
wouStskXS02YBWXMkY01zf3CWm4RokvD8ssTKPfDntunZe/5PkRZxkI5+HYOs+4s
|
||||
nQckjdhd7EuMvzz3LFDxsrLPYYoYNAfRUxONJL65SfawakfmA3TzTOPrUGvMt0My
|
||||
ifszrpF7f5Ie9FBbCMclkGQPdwKBgQC3sYa3lxOm9kAxVqioVdGWzdmH2Ht4r+2j
|
||||
6oPtpqJ39Z+MAIx4gDUg8E7byLAWmPSgbPXgA9yZxoVGHgDKBhK6cWT5XstVc7F3
|
||||
5N9aHwtdgDqTUO0+3UM8X7os+li/mngLnjnLAMvbyHTPCxcEk+XyOC7AhxYq9L84
|
||||
3UaIzLZuHQKBgHscwD916Tbm5ftg46Np8cU+JVVIzReFoWGDgidS65PM9+WtRVfa
|
||||
QEYjtndRVolT7kmbSa+Idp6l2Hm9N4IA/mv8sKf3kkbAsr7FWQlv0EfPPGt6IaX8
|
||||
cJPPWs1yIAhTumTu4sHYGIR4tXTdG2qvf+WrqmzC1mndzlpgDv2zAfDrAoGAKFhL
|
||||
WGNMI62OJ8f6vw4qPE716go4BCfPr+LAGyAwKty9sAgm0giGordk+oy3cB8kC2Aj
|
||||
GJKAjx21A9NvJO/0iRKCtOqHCjugzM5t5+NNobmaI+TwVpBORiJYR6ysdVi96P9V
|
||||
fiqsm5cJYLf01EKPjIWebxa8Xa7nmNuwtDcSElkCgYBqaDxxvqJgxry612rnfx6q
|
||||
Fj7STW9FDG2lvx7GDyxhiyD2oLp3jorEBTH15Y+pL8+Hod0WRVpBNB60ccpjlEPJ
|
||||
zHYcFtLD8iMgENBu78Vx7PomWz3QElr/dPV6yzxxukLPIg7/Xbl6cEIMuBAawjqb
|
||||
o2JAUPwR3y5xOxOyhAPXag==
|
||||
-----END PRIVATE KEY-----
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import * as comfyui from './comfyui.js';
|
||||
import * as jimuai from './JimuAI.js';
|
||||
import coze from './coze/coze.js';
|
||||
import * as runninghub from './runninghub.js';
|
||||
|
||||
export default { comfyui, jimuai, coze };
|
||||
export default { comfyui, jimuai, coze, runninghub };
|
||||
|
|
@ -1,9 +1,7 @@
|
|||
// 获取生成接口URL
|
||||
export function getGenerateUrl() {
|
||||
return process.env.RunningHub_URL
|
||||
}
|
||||
|
||||
// 获取生成接口请求头
|
||||
export function getGenerateHeader(){
|
||||
return {
|
||||
'Content-Type': 'application/json',
|
||||
|
|
@ -11,45 +9,38 @@ export function getGenerateHeader(){
|
|||
};
|
||||
}
|
||||
|
||||
// 获取生成接口请求体
|
||||
export function getGenerateBody(task) {
|
||||
const taskData = JSON.parse(task.payload)
|
||||
const payload = {...taskData,apiKey:task.apikey,webhookUrl: process.env.CALLBACK_URL}
|
||||
// console.log('getGenerteBody', payload);
|
||||
return JSON.stringify(payload);
|
||||
}
|
||||
|
||||
// 获取回调接口
|
||||
export function getQueryUrl() {
|
||||
return process.env.CALLBACK_URL
|
||||
}
|
||||
|
||||
// 获取任务状态
|
||||
export function getTaskStatus() {
|
||||
if(response.task_status === 'SUCCESS') return true;
|
||||
}
|
||||
|
||||
// 提交任务后对返回的数据处理 从返回的数据里筛选成功的任务数据并返回相应外部平台的任务ID
|
||||
export async function getSuccessTasks(response) {
|
||||
const res = await response.json()
|
||||
console.log('runninghub:\n', res)
|
||||
if(res.msg === 'success' && res.code === 0) {
|
||||
return res.data.taskId
|
||||
} else {
|
||||
console.error('[runninghub] 返回错误:', res);
|
||||
return {message:res, type: 2}
|
||||
}
|
||||
}
|
||||
|
||||
// 获取结果后对返回的数据处理 从返回的数据里筛选成功的任务数据并返回相应外部平台的任务ID
|
||||
export async function getTaskResult(response) {
|
||||
const res = await JSON.parse(response)
|
||||
const files = []
|
||||
if(res.msg === 'success' && res.code === 0) {
|
||||
for(const file of res.data)
|
||||
files.push(file.fileUrl)
|
||||
// console.log('files',files)
|
||||
return {files: files[0], type: 1}
|
||||
} else {
|
||||
return {message:res.msg, type: 2}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
module.exports = {
|
||||
apps: [{
|
||||
name: 'digitalHuman-md-server-v2',
|
||||
script: './md-server.js',
|
||||
cwd: './',
|
||||
args: '',
|
||||
interpreter: 'node',
|
||||
interpreter_args: '',
|
||||
|
||||
watch: true,
|
||||
ignore_watch: ['config', 'logs', 'node_modules', 'redis', 'school', 'static', 'worker_threads',
|
||||
'.env', 'package.json', 'package-lock.json', 'pnpm-lock.yaml',
|
||||
'pm2Index.config.cjs', 'pm2Websocket.config.cjs', 'pm2MdServer.config.cjs'],
|
||||
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
|
||||
autorestart: true,
|
||||
max_restarts: 30,
|
||||
min_uptime: '10s',
|
||||
|
||||
out_file: './logs/md-server/out/out.log',
|
||||
error_file: './logs/md-server/error/error.log',
|
||||
log_type: 'raw',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
|
||||
kill_timeout: 1600,
|
||||
restart_delay: 4000,
|
||||
|
||||
env_production: {
|
||||
NODE_ENV: 'production'
|
||||
}
|
||||
}]
|
||||
};
|
||||
|
|
@ -10,6 +10,10 @@ const logger = {
|
|||
const timestamp = new Date().toISOString();
|
||||
console.log(`[${timestamp}] INFO: ${message}`);
|
||||
},
|
||||
warn: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.warn(`[${timestamp}] WARN: ${message}`);
|
||||
},
|
||||
error: (message, error) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.error(`[${timestamp}] ERROR: ${message}`, error || '');
|
||||
|
|
@ -32,6 +36,8 @@ class InitQueues {
|
|||
this.errorList = `${this.prefix}:error:list`; // 错误列表，存储任务ID
|
||||
this.initInfoKey = `${this.prefix}:InitInfo`; // 初始化信息键名
|
||||
this.pendingMessages = `${this.prefix}:pending:messages`; // 待发送消息队列，存储断连时未发送的消息
|
||||
this.callbackPending = `${this.prefix}:callback:pending`; // 回调等待队列，存储等待回调的任务信息
|
||||
this.CALLBACK_TIMEOUT = parseInt(process.env.CALLBACK_TIMEOUT) || 3600000; // 回调超时时间，默认10分钟
|
||||
}
|
||||
|
||||
// 初始化各个队列
|
||||
|
|
@ -109,6 +115,14 @@ class InitQueues {
|
|||
}
|
||||
}
|
||||
|
||||
// 添加新平台（model.json 中新增但 Redis 中不存在的平台）
|
||||
for (const [key, platform] of Object.entries(platforms)) {
|
||||
if (!existingInfo[0].platforms[key]) {
|
||||
await redis.json.set(this.initInfoKey, `$.platforms.${key}`, platform);
|
||||
logger.info(`已添加新平台 ${key} 配置：MAX_CONCURRENT=${platform.MAX_CONCURRENT}`);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('已更新各平台配置并清零WQtasks计数器');
|
||||
}
|
||||
|
||||
|
|
@ -210,7 +224,8 @@ class InitQueues {
|
|||
async getEQtaskALL() {
|
||||
try {
|
||||
const res = await redis.json.get(this.initInfoKey, { path: '$.EQtaskALL' });
|
||||
return res ? res[0] : 0;
|
||||
const value = res ? res[0] : 0;
|
||||
return (typeof value === 'number' && !isNaN(value)) ? value : 0;
|
||||
} catch (error) {
|
||||
logger.error('获取错误队列任务数失败:', error);
|
||||
return 0;
|
||||
|
|
@ -230,7 +245,7 @@ class InitQueues {
|
|||
if (platformData.polling.includes(platform)) {
|
||||
PQcount++;
|
||||
}
|
||||
logger.debug(`增加相关平台处理队列: AIGC=${aigc}, Platform=${platform}, Count=${count}`);
|
||||
// logger.debug(`增加相关平台处理队列: AIGC=${aigc}, Platform=${platform}, Count=${count}`);
|
||||
}
|
||||
|
||||
multi.json.numIncrBy(this.initInfoKey, '$.PQtasksALL', PQcount);
|
||||
|
|
@ -372,6 +387,51 @@ class InitQueues {
|
|||
}
|
||||
}
|
||||
|
||||
async addCallbackPendingTask(remoteTaskId, taskId, aigc, platform) {
|
||||
try {
|
||||
const taskInfo = JSON.stringify({
|
||||
taskId,
|
||||
aigc,
|
||||
platform,
|
||||
remoteTaskId,
|
||||
createdAt: Date.now()
|
||||
});
|
||||
await redis.hSet(this.callbackPending, remoteTaskId, taskInfo);
|
||||
logger.debug(`添加回调等待任务: remoteTaskId=${remoteTaskId}, taskId=${taskId}, platform=${platform}`);
|
||||
} catch (error) {
|
||||
logger.error('添加回调等待任务失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async removeCallbackPendingTask(remoteTaskId) {
|
||||
try {
|
||||
await redis.hDel(this.callbackPending, remoteTaskId);
|
||||
logger.debug(`移除回调等待任务: remoteTaskId=${remoteTaskId}`);
|
||||
} catch (error) {
|
||||
logger.error('移除回调等待任务失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async getCallbackPendingTasks() {
|
||||
try {
|
||||
const tasks = await redis.hGetAll(this.callbackPending);
|
||||
return tasks;
|
||||
} catch (error) {
|
||||
logger.error('获取回调等待任务失败:', error);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async getCallbackPendingCount() {
|
||||
try {
|
||||
const count = await redis.hLen(this.callbackPending);
|
||||
return count;
|
||||
} catch (error) {
|
||||
logger.error('获取回调等待任务数量失败:', error);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export default new InitQueues();
|
||||
|
|
|
|||
|
|
@ -11,6 +11,73 @@ const redis = createClient({
|
|||
url: process.env.REDIS_URL || 'redis://localhost:6379'
|
||||
});
|
||||
|
||||
async function resetCounters() {
|
||||
try {
|
||||
console.log('正在重置计数器...');
|
||||
|
||||
const initInfoResult = await redis.json.get(initInfoKey, { path: '$' });
|
||||
|
||||
if (!initInfoResult || !initInfoResult[0]) {
|
||||
console.log('未找到初始化信息，跳过计数器重置');
|
||||
return;
|
||||
}
|
||||
|
||||
const initInfo = initInfoResult[0];
|
||||
|
||||
console.log('\n当前计数器状态:');
|
||||
console.log(` PQtasksALL (总处理任务数): ${initInfo.PQtasksALL || 0}`);
|
||||
console.log(` RQtasksALL (总结果任务数): ${initInfo.RQtasksALL || 0}`);
|
||||
console.log(` CQtasksALL (总回调任务数): ${initInfo.CQtasksALL || 0}`);
|
||||
console.log(` EQtaskALL (总错误任务数): ${initInfo.EQtaskALL || 0}`);
|
||||
|
||||
if (initInfo.platforms) {
|
||||
console.log('\n各平台计数器状态:');
|
||||
for (const [key, platform] of Object.entries(initInfo.platforms)) {
|
||||
console.log(` ${key}: WQtasks=${platform.WQtasks || 0}, PQtasks=${platform.PQtasks || 0}`);
|
||||
}
|
||||
}
|
||||
|
||||
const multi = redis.multi();
|
||||
|
||||
multi.json.set(initInfoKey, '$.PQtasksALL', 0);
|
||||
multi.json.set(initInfoKey, '$.RQtasksALL', 0);
|
||||
multi.json.set(initInfoKey, '$.CQtasksALL', 0);
|
||||
multi.json.set(initInfoKey, '$.EQtaskALL', 0);
|
||||
|
||||
if (initInfo.platforms) {
|
||||
for (const key of Object.keys(initInfo.platforms)) {
|
||||
multi.json.set(initInfoKey, `$.platforms.${key}.WQtasks`, 0);
|
||||
multi.json.set(initInfoKey, `$.platforms.${key}.PQtasks`, 0);
|
||||
}
|
||||
}
|
||||
|
||||
await multi.exec();
|
||||
|
||||
console.log('\n✓ 所有计数器已重置为0');
|
||||
|
||||
} catch (error) {
|
||||
console.error('重置计数器失败:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function clearCallbackPendingQueue() {
|
||||
try {
|
||||
const callbackPendingKey = `${prefix}:callback:pending`;
|
||||
const exists = await redis.exists(callbackPendingKey);
|
||||
|
||||
if (exists) {
|
||||
const count = await redis.hLen(callbackPendingKey);
|
||||
if (count > 0) {
|
||||
await redis.del(callbackPendingKey);
|
||||
console.log(`\n✓ 已清除回调等待队列，共 ${count} 个任务`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('清除回调等待队列失败:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function clearAllProjectData() {
|
||||
try {
|
||||
console.log('正在连接Redis...');
|
||||
|
|
@ -21,7 +88,6 @@ async function clearAllProjectData() {
|
|||
|
||||
console.log(`\n开始清除项目 "${prefix}" 的所有Redis数据...`);
|
||||
|
||||
// 1. 获取并显示当前初始化信息
|
||||
try {
|
||||
const initInfoResult = await redis.json.get(initInfoKey, { path: '$' });
|
||||
if (initInfoResult) {
|
||||
|
|
@ -33,13 +99,11 @@ async function clearAllProjectData() {
|
|||
console.log('获取初始化信息失败（可能不存在）:', error.message);
|
||||
}
|
||||
|
||||
// 2. 删除所有相关的Redis键
|
||||
const keysToDelete = [];
|
||||
|
||||
// 使用SCAN模式删除所有相关键
|
||||
const patterns = [
|
||||
`${prefix}:*`, // 所有项目前缀的键
|
||||
`${initQueue?.prefix || prefix}:*`, // 兼容initQueue的前缀
|
||||
`${prefix}:*`,
|
||||
`${initQueue?.prefix || prefix}:*`,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
|
|
@ -60,7 +124,6 @@ async function clearAllProjectData() {
|
|||
console.log(` 游标: ${newCursor}, 找到键数量: ${keys.length}`);
|
||||
|
||||
if (keys.length > 0) {
|
||||
// 过滤有效键
|
||||
const validKeys = keys.filter(key => {
|
||||
return typeof key === 'string' && key.trim() !== '';
|
||||
});
|
||||
|
|
@ -82,7 +145,6 @@ async function clearAllProjectData() {
|
|||
console.log(`模式 "${pattern}" 共删除 ${totalDeleted} 个键`);
|
||||
}
|
||||
|
||||
// 3. 特殊处理：删除等待队列（可能不包含前缀）
|
||||
try {
|
||||
const platforms = await redis.json.get(initInfoKey, { path: '$.platforms' });
|
||||
if (platforms && platforms[0]) {
|
||||
|
|
@ -104,7 +166,6 @@ async function clearAllProjectData() {
|
|||
} catch (error) {
|
||||
console.error('清除Redis数据失败:', error);
|
||||
} finally {
|
||||
// 断开Redis连接
|
||||
if (redis.isOpen) {
|
||||
await redis.disconnect();
|
||||
console.log('\nRedis连接已关闭');
|
||||
|
|
@ -113,4 +174,41 @@ async function clearAllProjectData() {
|
|||
}
|
||||
}
|
||||
|
||||
clearAllProjectData();
|
||||
async function resetQueueCounters() {
|
||||
try {
|
||||
console.log('正在连接Redis...');
|
||||
console.log('REDIS_URL:', process.env.REDIS_URL);
|
||||
|
||||
await redis.connect();
|
||||
console.log('Redis连接成功');
|
||||
|
||||
console.log(`\n========== 重置队列计数器 ==========`);
|
||||
console.log(`项目前缀: ${prefix}`);
|
||||
|
||||
await resetCounters();
|
||||
await clearCallbackPendingQueue();
|
||||
|
||||
console.log('\n========================================');
|
||||
console.log('队列计数器重置完成！');
|
||||
console.log('现在可以正常处理新任务了');
|
||||
console.log('========================================');
|
||||
|
||||
} catch (error) {
|
||||
console.error('重置队列计数器失败:', error);
|
||||
} finally {
|
||||
if (redis.isOpen) {
|
||||
await redis.disconnect();
|
||||
console.log('\nRedis连接已关闭');
|
||||
}
|
||||
process.exit();
|
||||
}
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const mode = args[0] || 'reset';
|
||||
|
||||
if (mode === 'clear') {
|
||||
clearAllProjectData();
|
||||
} else {
|
||||
resetQueueCounters();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,431 @@
|
|||
{
|
||||
"3": {
|
||||
"inputs": {
|
||||
"image": [
|
||||
"4",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "GetImageSizeAndCount",
|
||||
"_meta": {
|
||||
"title": "Get Image Size & Count"
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"inputs": {
|
||||
"width": [
|
||||
"28",
|
||||
0
|
||||
],
|
||||
"height": [
|
||||
"17",
|
||||
0
|
||||
],
|
||||
"upscale_method": "lanczos",
|
||||
"keep_proportion": "crop",
|
||||
"pad_color": "0, 0, 0",
|
||||
"crop_position": "center",
|
||||
"divisible_by": 16,
|
||||
"device": "cpu",
|
||||
"image": [
|
||||
"44",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ImageResizeKJv2",
|
||||
"_meta": {
|
||||
"title": "Resize Image v2"
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"inputs": {
|
||||
"clip_name": "clip_vision_h.safetensors"
|
||||
},
|
||||
"class_type": "CLIPVisionLoader",
|
||||
"_meta": {
|
||||
"title": "Load CLIP Vision"
|
||||
}
|
||||
},
|
||||
"6": {
|
||||
"inputs": {
|
||||
"backend": "inductor",
|
||||
"fullgraph": false,
|
||||
"mode": "default",
|
||||
"dynamic": false,
|
||||
"dynamo_cache_size_limit": 64,
|
||||
"compile_transformer_blocks_only": true,
|
||||
"dynamo_recompile_limit": 128,
|
||||
"force_parameter_static_shapes": true,
|
||||
"allow_unmerged_lora_compile": false
|
||||
},
|
||||
"class_type": "WanVideoTorchCompileSettings",
|
||||
"_meta": {
|
||||
"title": "WanVideo Torch Compile Settings"
|
||||
}
|
||||
},
|
||||
"7": {
|
||||
"inputs": {
|
||||
"model": "wan2.1-i2v-14b-480p-Q4_K_S.gguf",
|
||||
"base_precision": "fp16_fast",
|
||||
"quantization": "disabled",
|
||||
"load_device": "offload_device",
|
||||
"attention_mode": "sdpa",
|
||||
"rms_norm_function": "default",
|
||||
"compile_args": [
|
||||
"6",
|
||||
0
|
||||
],
|
||||
"block_swap_args": [
|
||||
"42",
|
||||
0
|
||||
],
|
||||
"lora": [
|
||||
"40",
|
||||
0
|
||||
],
|
||||
"multitalk_model": [
|
||||
"41",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "WanVideoModelLoader",
|
||||
"_meta": {
|
||||
"title": "WanVideo Model Loader"
|
||||
}
|
||||
},
|
||||
"9": {
|
||||
"inputs": {
|
||||
"model_name": "wan_2.1_vae.safetensors",
|
||||
"precision": "bf16",
|
||||
"use_cpu_cache": false,
|
||||
"verbose": false
|
||||
},
|
||||
"class_type": "WanVideoVAELoader",
|
||||
"_meta": {
|
||||
"title": "WanVideo VAE Loader"
|
||||
}
|
||||
},
|
||||
"11": {
|
||||
"inputs": {
|
||||
"width": [
|
||||
"3",
|
||||
1
|
||||
],
|
||||
"height": [
|
||||
"3",
|
||||
2
|
||||
],
|
||||
"frame_window_size": 81,
|
||||
"motion_frame": 9,
|
||||
"force_offload": false,
|
||||
"colormatch": "disabled",
|
||||
"tiled_vae": false,
|
||||
"mode": "infinitetalk",
|
||||
"output_path": "",
|
||||
"vae": [
|
||||
"9",
|
||||
0
|
||||
],
|
||||
"start_image": [
|
||||
"3",
|
||||
0
|
||||
],
|
||||
"clip_embeds": [
|
||||
"16",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "WanVideoImageToVideoMultiTalk",
|
||||
"_meta": {
|
||||
"title": "WanVideo Long I2V Multi/InfiniteTalk"
|
||||
}
|
||||
},
|
||||
"13": {
|
||||
"inputs": {
|
||||
"model": "TencentGameMate/chinese-wav2vec2-base",
|
||||
"base_precision": "fp16",
|
||||
"load_device": "main_device"
|
||||
},
|
||||
"class_type": "DownloadAndLoadWav2VecModel",
|
||||
"_meta": {
|
||||
"title": "(Down)load Wav2Vec Model"
|
||||
}
|
||||
},
|
||||
"14": {
|
||||
"inputs": {
|
||||
"chunk_fade_shape": "linear",
|
||||
"chunk_length": 10,
|
||||
"chunk_overlap": 0.1,
|
||||
"audio": [
|
||||
"48",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "AudioSeparation",
|
||||
"_meta": {
|
||||
"title": "AudioSeparation"
|
||||
}
|
||||
},
|
||||
"15": {
|
||||
"inputs": {
|
||||
"normalize_loudness": true,
|
||||
"num_frames": [
|
||||
"30",
|
||||
0
|
||||
],
|
||||
"fps": 25,
|
||||
"audio_scale": 1,
|
||||
"audio_cfg_scale": 1,
|
||||
"multi_audio_type": "para",
|
||||
"add_noise_floor": false,
|
||||
"smooth_transients": false,
|
||||
"wav2vec_model": [
|
||||
"13",
|
||||
0
|
||||
],
|
||||
"audio_1": [
|
||||
"14",
|
||||
3
|
||||
]
|
||||
},
|
||||
"class_type": "MultiTalkWav2VecEmbeds",
|
||||
"_meta": {
|
||||
"title": "Multi/InfiniteTalk Wav2vec2 Embeds"
|
||||
}
|
||||
},
|
||||
"16": {
|
||||
"inputs": {
|
||||
"strength_1": 1,
|
||||
"strength_2": 1,
|
||||
"crop": "center",
|
||||
"combine_embeds": "average",
|
||||
"force_offload": true,
|
||||
"tiles": 0,
|
||||
"ratio": 0.5,
|
||||
"clip_vision": [
|
||||
"5",
|
||||
0
|
||||
],
|
||||
"image_1": [
|
||||
"3",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "WanVideoClipVisionEncode",
|
||||
"_meta": {
|
||||
"title": "WanVideo ClipVision Encode"
|
||||
}
|
||||
},
|
||||
"17": {
|
||||
"inputs": {
|
||||
"value": 800
|
||||
},
|
||||
"class_type": "JWInteger",
|
||||
"_meta": {
|
||||
"title": "图片高度"
|
||||
}
|
||||
},
|
||||
"23": {
|
||||
"inputs": {
|
||||
"enable_vae_tiling": false,
|
||||
"tile_x": 272,
|
||||
"tile_y": 272,
|
||||
"tile_stride_x": 144,
|
||||
"tile_stride_y": 128,
|
||||
"normalization": "default",
|
||||
"vae": [
|
||||
"9",
|
||||
0
|
||||
],
|
||||
"samples": [
|
||||
"25",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "WanVideoDecode",
|
||||
"_meta": {
|
||||
"title": "WanVideo Decode"
|
||||
}
|
||||
},
|
||||
"24": {
|
||||
"inputs": {
|
||||
"preview": "",
|
||||
"source": [
|
||||
"15",
|
||||
2
|
||||
]
|
||||
},
|
||||
"class_type": "PreviewAny",
|
||||
"_meta": {
|
||||
"title": "Preview as Text"
|
||||
}
|
||||
},
|
||||
"25": {
|
||||
"inputs": {
|
||||
"steps": 4,
|
||||
"cfg": 1.0000000000000002,
|
||||
"shift": 11.000000000000002,
|
||||
"seed": 2,
|
||||
"force_offload": true,
|
||||
"scheduler": "dpm++_sde",
|
||||
"riflex_freq_index": 0,
|
||||
"denoise_strength": 1,
|
||||
"batched_cfg": false,
|
||||
"rope_function": "comfy",
|
||||
"start_step": 0,
|
||||
"end_step": -1,
|
||||
"add_noise_to_samples": true,
|
||||
"model": [
|
||||
"7",
|
||||
0
|
||||
],
|
||||
"image_embeds": [
|
||||
"11",
|
||||
0
|
||||
],
|
||||
"text_embeds": [
|
||||
"38",
|
||||
0
|
||||
],
|
||||
"multitalk_embeds": [
|
||||
"15",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "WanVideoSampler",
|
||||
"_meta": {
|
||||
"title": "WanVideo Sampler"
|
||||
}
|
||||
},
|
||||
"26": {
|
||||
"inputs": {
|
||||
"frame_rate": 25,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "WanVideo2_1_InfiniteTalk",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": false,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"no_preview": false,
|
||||
"images": [
|
||||
"23",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"48",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoCombine",
|
||||
"_meta": {
|
||||
"title": "Video Combine 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"28": {
|
||||
"inputs": {
|
||||
"value": 450
|
||||
},
|
||||
"class_type": "JWInteger",
|
||||
"_meta": {
|
||||
"title": "图片宽度"
|
||||
}
|
||||
},
|
||||
"30": {
|
||||
"inputs": {
|
||||
"expression": "a*25+10\n",
|
||||
"a": [
|
||||
"51",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "MathExpression|pysssss",
|
||||
"_meta": {
|
||||
"title": "Math Expression 🐍"
|
||||
}
|
||||
},
|
||||
"38": {
|
||||
"inputs": {
|
||||
"model_name": "umt5-xxl-enc-fp8_e4m3fn.safetensors",
|
||||
"precision": "bf16",
|
||||
"positive_prompt": "这个人在说话，手部动作",
|
||||
"negative_prompt": "bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards",
|
||||
"quantization": "disabled",
|
||||
"use_disk_cache": false,
|
||||
"device": "gpu"
|
||||
},
|
||||
"class_type": "WanVideoTextEncodeCached",
|
||||
"_meta": {
|
||||
"title": "WanVideo TextEncode Cached"
|
||||
}
|
||||
},
|
||||
"40": {
|
||||
"inputs": {
|
||||
"lora": "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors",
|
||||
"strength": 1,
|
||||
"low_mem_load": false,
|
||||
"merge_loras": false
|
||||
},
|
||||
"class_type": "WanVideoLoraSelect",
|
||||
"_meta": {
|
||||
"title": "WanVideo Lora Select"
|
||||
}
|
||||
},
|
||||
"41": {
|
||||
"inputs": {
|
||||
"model": "Wan2_1-InfiniTetalk-Single_fp16.safetensors"
|
||||
},
|
||||
"class_type": "MultiTalkModelLoader",
|
||||
"_meta": {
|
||||
"title": "Multi/InfiniteTalk Model Loader"
|
||||
}
|
||||
},
|
||||
"42": {
|
||||
"inputs": {
|
||||
"blocks_to_swap": 20,
|
||||
"offload_img_emb": false,
|
||||
"offload_txt_emb": false,
|
||||
"use_non_blocking": true,
|
||||
"vace_blocks_to_swap": 0,
|
||||
"prefetch_blocks": 1,
|
||||
"block_swap_debug": false
|
||||
},
|
||||
"class_type": "WanVideoBlockSwap",
|
||||
"_meta": {
|
||||
"title": "WanVideo Block Swap"
|
||||
}
|
||||
},
|
||||
"44": {
|
||||
"inputs": {
|
||||
"url": "https://shuziren.xueai.art/v2/api/file/36367795-67ab-4a15-bbc3-337c0776770b/digital-human/5cb1d1d0-37a0-409c-83b5-da5ca6bc6d82/_E7_94_B7_E6_80_A75-_E7_94_B7-40_E5_B2_81.png"
|
||||
},
|
||||
"class_type": "LoadImagesFromURL",
|
||||
"_meta": {
|
||||
"title": "Load Images From URL ♾️Mixlab"
|
||||
}
|
||||
},
|
||||
"48": {
|
||||
"inputs": {
|
||||
"audio_file": "https://shuziren.xueai.art/v2/api/file/36367795-67ab-4a15-bbc3-337c0776770b/voices/ComfyUI_00002_zrptx_1773123259.flac",
|
||||
"seek_seconds": 0
|
||||
},
|
||||
"class_type": "VHS_LoadAudio",
|
||||
"_meta": {
|
||||
"title": "Load Audio (Path)🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"51": {
|
||||
"inputs": {
|
||||
"audio": [
|
||||
"48",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "1hew_AudioDuration",
|
||||
"_meta": {
|
||||
"title": "Audio Duration"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
{
|
||||
"79": {
|
||||
"inputs": {
|
||||
"model": "FlashVSR",
|
||||
"mode": "tiny",
|
||||
"scale": 2,
|
||||
"tiled_vae": true,
|
||||
"tiled_dit": true,
|
||||
"unload_dit": false,
|
||||
"seed": 234015562683294,
|
||||
"frames": [
|
||||
"86",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "FlashVSRNode",
|
||||
"_meta": {
|
||||
"title": "FlashVSR Ultra-Fast"
|
||||
}
|
||||
},
|
||||
"80": {
|
||||
"inputs": {
|
||||
"frame_rate": [
|
||||
"84",
|
||||
0
|
||||
],
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "AnimateDiff",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": false,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"no_preview": false,
|
||||
"images": [
|
||||
"79",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"86",
|
||||
2
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoCombine",
|
||||
"_meta": {
|
||||
"title": "合并为视频"
|
||||
}
|
||||
},
|
||||
"82": {
|
||||
"inputs": {
|
||||
"anything": [
|
||||
"79",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "easy cleanGpuUsed",
|
||||
"_meta": {
|
||||
"title": "清理GPU占用"
|
||||
}
|
||||
},
|
||||
"84": {
|
||||
"inputs": {
|
||||
"video_info": [
|
||||
"86",
|
||||
3
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoInfoSource",
|
||||
"_meta": {
|
||||
"title": "视频信息（初始）"
|
||||
}
|
||||
},
|
||||
"85": {
|
||||
"inputs": {
|
||||
"anything": [
|
||||
"86",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "easy cleanGpuUsed",
|
||||
"_meta": {
|
||||
"title": "清理GPU占用"
|
||||
}
|
||||
},
|
||||
"86": {
|
||||
"inputs": {
|
||||
"video": "https://shuziren.xueai.art/v2/api/file/f58a0bd2-b791-4a4b-abab-3f91fbe4d0d5/videos/AnimateDiff_00001_p82-audio_suulk_1770707164.mp4",
|
||||
"force_rate": 0,
|
||||
"custom_width": 0,
|
||||
"custom_height": 0,
|
||||
"frame_load_cap": 0,
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1,
|
||||
"format": "AnimateDiff"
|
||||
},
|
||||
"class_type": "VHS_LoadVideoPath",
|
||||
"_meta": {
|
||||
"title": "加载视频（路径）"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
{
|
||||
"54": {
|
||||
"inputs": {
|
||||
"seed": 1357,
|
||||
"lips_expression": 1.5,
|
||||
"inference_steps": 25,
|
||||
"images": [
|
||||
"55",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"55",
|
||||
1
|
||||
]
|
||||
},
|
||||
"class_type": "LatentSyncNode",
|
||||
"_meta": {
|
||||
"title": "LatentSync1.6 Node"
|
||||
}
|
||||
},
|
||||
"55": {
|
||||
"inputs": {
|
||||
"mode": "pingpong",
|
||||
"fps": 25,
|
||||
"silent_padding_sec": 0.5,
|
||||
"images": [
|
||||
"129",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"128",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VideoLengthAdjuster",
|
||||
"_meta": {
|
||||
"title": "Video Length Adjuster"
|
||||
}
|
||||
},
|
||||
"81": {
|
||||
"inputs": {
|
||||
"frame_rate": 25,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "AnimateDiff",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": false,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"no_preview": false,
|
||||
"images": [
|
||||
"54",
|
||||
0
|
||||
],
|
||||
"audio": [
|
||||
"54",
|
||||
1
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoCombine",
|
||||
"_meta": {
|
||||
"title": "Video Combine 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"122": {
|
||||
"inputs": {
|
||||
"expression": "a*25+5\n",
|
||||
"a": [
|
||||
"130",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "MathExpression|pysssss",
|
||||
"_meta": {
|
||||
"title": "Math Expression 🐍"
|
||||
}
|
||||
},
|
||||
"126": {
|
||||
"inputs": {
|
||||
"anything": [
|
||||
"55",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "easy cleanGpuUsed",
|
||||
"_meta": {
|
||||
"title": "Clean VRAM Used"
|
||||
}
|
||||
},
|
||||
"127": {
|
||||
"inputs": {
|
||||
"anything": [
|
||||
"55",
|
||||
1
|
||||
]
|
||||
},
|
||||
"class_type": "easy cleanGpuUsed",
|
||||
"_meta": {
|
||||
"title": "Clean VRAM Used"
|
||||
}
|
||||
},
|
||||
"128": {
|
||||
"inputs": {
|
||||
"audio_file": "https://shuziren.xueai.art/v2/api/file/36367795-67ab-4a15-bbc3-337c0776770b/voices/ComfyUI_00002_zrptx_1773123259.flac",
|
||||
"seek_seconds": 0
|
||||
},
|
||||
"class_type": "VHS_LoadAudio",
|
||||
"_meta": {
|
||||
"title": "Load Audio (Path)🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"129": {
|
||||
"inputs": {
|
||||
"video": "https://shuziren.xueai.art/v2/api/file/f58a0bd2-b791-4a4b-abab-3f91fbe4d0d5/videos/AnimateDiff_00001_p82-audio_suulk_1770707164.mp4",
|
||||
"force_rate": 0,
|
||||
"custom_width": [
|
||||
"131",
|
||||
0
|
||||
],
|
||||
"custom_height": [
|
||||
"132",
|
||||
0
|
||||
],
|
||||
"frame_load_cap": [
|
||||
"122",
|
||||
0
|
||||
],
|
||||
"skip_first_frames": 0,
|
||||
"select_every_nth": 1,
|
||||
"format": "AnimateDiff"
|
||||
},
|
||||
"class_type": "VHS_LoadVideoPath",
|
||||
"_meta": {
|
||||
"title": "Load Video (Path) 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"130": {
|
||||
"inputs": {
|
||||
"audio": [
|
||||
"128",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "1hew_AudioDuration",
|
||||
"_meta": {
|
||||
"title": "Audio Duration"
|
||||
}
|
||||
},
|
||||
"131": {
|
||||
"inputs": {
|
||||
"value": 540
|
||||
},
|
||||
"class_type": "PrimitiveInt",
|
||||
"_meta": {
|
||||
"title": "Int"
|
||||
}
|
||||
},
|
||||
"132": {
|
||||
"inputs": {
|
||||
"value": 960
|
||||
},
|
||||
"class_type": "PrimitiveInt",
|
||||
"_meta": {
|
||||
"title": "Int"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,186 @@
|
|||
{
|
||||
"3": {
|
||||
"inputs": {
|
||||
"prompt": "长期高压会削弱免疫力，引发头痛、失眠、胃肠问题。学会主动放松，如深呼吸、正念冥想。当压力无法承受时，寻求专业心理帮助是明智之举。"
|
||||
},
|
||||
"class_type": "CR Prompt Text",
|
||||
"_meta": {
|
||||
"title": "⚙️ CR Prompt Text"
|
||||
}
|
||||
},
|
||||
"10": {
|
||||
"inputs": {
|
||||
"text": "参考内容："
|
||||
},
|
||||
"class_type": "JjkText",
|
||||
"_meta": {
|
||||
"title": "Text"
|
||||
}
|
||||
},
|
||||
"12": {
|
||||
"inputs": {
|
||||
"text": "输入内容："
|
||||
},
|
||||
"class_type": "JjkText",
|
||||
"_meta": {
|
||||
"title": "Text"
|
||||
}
|
||||
},
|
||||
"13": {
|
||||
"inputs": {
|
||||
"text1": [
|
||||
"10",
|
||||
0
|
||||
],
|
||||
"text2": [
|
||||
"3",
|
||||
0
|
||||
],
|
||||
"text3": [
|
||||
"12",
|
||||
0
|
||||
],
|
||||
"text4": [
|
||||
"16",
|
||||
2
|
||||
],
|
||||
"text5": "",
|
||||
"text6": ""
|
||||
},
|
||||
"class_type": "TextCombinerSix",
|
||||
"_meta": {
|
||||
"title": "Text Combiner 6"
|
||||
}
|
||||
},
|
||||
"16": {
|
||||
"inputs": {
|
||||
"模型": "k1nto/Belle-whisper-large-v3-zh-punct-ct2",
|
||||
"每句最大长度": 20,
|
||||
"卸载模型": true,
|
||||
"seed": 750869249246021,
|
||||
"音频": [
|
||||
"24",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ASRMW",
|
||||
"_meta": {
|
||||
"title": "自动语音识别"
|
||||
}
|
||||
},
|
||||
"20": {
|
||||
"inputs": {
|
||||
"start_time": "0:00",
|
||||
"end_time": "0:30",
|
||||
"audio": [
|
||||
"23",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "AudioCrop",
|
||||
"_meta": {
|
||||
"title": "AudioCrop"
|
||||
}
|
||||
},
|
||||
"21": {
|
||||
"inputs": {
|
||||
"prompt": [
|
||||
"13",
|
||||
0
|
||||
],
|
||||
"system_content": "你是一个中文错别字识别大师，请帮我根据参考内容改正输入内容的错别字，要求不改变输入内容的文字格式，只改变错别字。\n注意：最终输出的内容仅为修改了错别字之后的输入内容。",
|
||||
"model": "Qwen/Qwen2.5-7B-Instruct",
|
||||
"seed": 238215280189216,
|
||||
"context_size": 1,
|
||||
"max_tokens": 20000,
|
||||
"api_key": [
|
||||
"22",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "SiliconflowLLM",
|
||||
"_meta": {
|
||||
"title": "LLM Siliconflow ♾️Mixlab"
|
||||
}
|
||||
},
|
||||
"22": {
|
||||
"inputs": {
|
||||
"text": "sk-osqpezzqfkiroexodsqtkjbexbrenbtbryrmeotdsvvsmyia"
|
||||
},
|
||||
"class_type": "JjkText",
|
||||
"_meta": {
|
||||
"title": "Text"
|
||||
}
|
||||
},
|
||||
"23": {
|
||||
"inputs": {
|
||||
"audio_file": "https://shuziren.xueai.art/v2/api/file/36367795-67ab-4a15-bbc3-337c0776770b/voices/ComfyUI_00002_zrptx_1773123259.flac",
|
||||
"seek_seconds": 0,
|
||||
"duration": 0
|
||||
},
|
||||
"class_type": "VHS_LoadAudio",
|
||||
"_meta": {
|
||||
"title": "Load Audio (Path)🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"24": {
|
||||
"inputs": {
|
||||
"text": [
|
||||
"3",
|
||||
0
|
||||
],
|
||||
"mode": "Auto",
|
||||
"do_sample_mode": "on",
|
||||
"temperature": 0.8,
|
||||
"top_p": 0.9,
|
||||
"top_k": 30,
|
||||
"num_beams": 3,
|
||||
"repetition_penalty": 10,
|
||||
"length_penalty": 0,
|
||||
"max_mel_tokens": 1815,
|
||||
"max_tokens_per_sentence": 120,
|
||||
"seed": 2647111708,
|
||||
"reference_audio": [
|
||||
"20",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "IndexTTS2BaseNode",
|
||||
"_meta": {
|
||||
"title": "Index TTS 2 - Base"
|
||||
}
|
||||
},
|
||||
"29": {
|
||||
"inputs": {
|
||||
"save_directory": "",
|
||||
"filename": "",
|
||||
"extension": "txt",
|
||||
"encoding": "utf-8",
|
||||
"mode": "append",
|
||||
"text": [
|
||||
"21",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "KexueSaveText",
|
||||
"_meta": {
|
||||
"title": "可学文本保存"
|
||||
}
|
||||
},
|
||||
"30": {
|
||||
"inputs": {
|
||||
"save_directory": "",
|
||||
"filename": "",
|
||||
"format": "flac",
|
||||
"overwrite": false,
|
||||
"audio": [
|
||||
"24",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "KexueSaveAudio",
|
||||
"_meta": {
|
||||
"title": "可学音频保存"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -160,7 +160,7 @@ class MDWebSocketServer {
|
|||
try {
|
||||
const capacity = await redis.get(REDIS_KEYS.CAPACITY);
|
||||
const result = capacity ? parseInt(capacity, 10) : 0;
|
||||
console.log(`[MDWebSocketServer] 从 Redis 读取算力信息: ${result}`);
|
||||
// console.log(`[MDWebSocketServer] 从 Redis 读取算力信息: ${result}`);
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error('[MDWebSocketServer] 从 Redis 读取算力信息失败:', error);
|
||||
|
|
@ -175,7 +175,7 @@ class MDWebSocketServer {
|
|||
async getJwtTokenFromRedis() {
|
||||
try {
|
||||
const token = await redis.get(REDIS_KEYS.JWT);
|
||||
console.log(`[MDWebSocketServer] 从 Redis 读取 JWT Token: ${token ? '存在' : '不存在'}`);
|
||||
// console.log(`[MDWebSocketServer] 从 Redis 读取 JWT Token: ${token ? '存在' : '不存在'}`);
|
||||
return token;
|
||||
} catch (error) {
|
||||
console.error('[MDWebSocketServer] 从 Redis 读取 JWT Token 失败:', error);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,221 @@
|
|||
import redis from '../redis/index.js';
|
||||
import initQueue from '../redis/initQueue.js';
|
||||
|
||||
const logger = {
|
||||
info: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.log(`[${timestamp}] [QueueRecovery] INFO: ${message}`);
|
||||
},
|
||||
warn: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.warn(`[${timestamp}] [QueueRecovery] WARN: ${message}`);
|
||||
},
|
||||
error: (message, error) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.error(`[${timestamp}] [QueueRecovery] ERROR: ${message}`, error || '');
|
||||
}
|
||||
};
|
||||
|
||||
async function recoverCallbackQueue() {
|
||||
logger.info('开始恢复回调队列...');
|
||||
|
||||
try {
|
||||
const actualLength = await redis.lLen(initQueue.callback);
|
||||
const storedCount = await initQueue.getCQtasksALL();
|
||||
|
||||
logger.info(`回调队列状态: 实际长度=${actualLength}, 存储计数=${storedCount}`);
|
||||
|
||||
if (actualLength !== storedCount) {
|
||||
logger.warn(`检测到不一致，正在修复...`);
|
||||
await redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', actualLength);
|
||||
logger.info(`已修复回调队列计数: CQtasksALL=${actualLength}`);
|
||||
}
|
||||
|
||||
const taskIds = await redis.lRange(initQueue.callback, 0, -1);
|
||||
const orphanTaskIds = [];
|
||||
|
||||
for (const taskId of taskIds) {
|
||||
const taskExists = await redis.exists(`${initQueue.prefix}:task:${taskId}`);
|
||||
if (!taskExists) {
|
||||
orphanTaskIds.push(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
if (orphanTaskIds.length > 0) {
|
||||
logger.warn(`发现${orphanTaskIds.length}个孤立任务，正在移除...`);
|
||||
const multi = redis.multi();
|
||||
for (const orphanId of orphanTaskIds) {
|
||||
multi.lRem(initQueue.callback, 0, orphanId);
|
||||
}
|
||||
await multi.exec();
|
||||
logger.info(`已移除${orphanTaskIds.length}个孤立任务`);
|
||||
}
|
||||
|
||||
logger.info('回调队列恢复完成');
|
||||
return { actualLength, storedCount, orphanCount: orphanTaskIds.length };
|
||||
} catch (error) {
|
||||
logger.error('恢复回调队列失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverCallbackPendingQueue() {
|
||||
logger.info('开始恢复回调等待队列...');
|
||||
|
||||
try {
|
||||
const pendingTasks = await initQueue.getCallbackPendingTasks();
|
||||
const now = Date.now();
|
||||
const staleRemoteTaskIds = [];
|
||||
|
||||
for (const [remoteTaskId, taskInfoStr] of Object.entries(pendingTasks)) {
|
||||
try {
|
||||
const taskInfo = JSON.parse(taskInfoStr);
|
||||
|
||||
const mappingExists = await redis.exists(`${initQueue.callback}:${remoteTaskId}`);
|
||||
const taskExists = await redis.exists(`${initQueue.prefix}:task:${taskInfo.taskId}`);
|
||||
|
||||
if (!mappingExists || !taskExists) {
|
||||
staleRemoteTaskIds.push(remoteTaskId);
|
||||
logger.warn(`发现孤立的回调等待任务: remoteTaskId=${remoteTaskId}, mappingExists=${mappingExists}, taskExists=${taskExists}`);
|
||||
}
|
||||
} catch (parseError) {
|
||||
logger.error(`解析任务信息失败: ${remoteTaskId}`, parseError);
|
||||
staleRemoteTaskIds.push(remoteTaskId);
|
||||
}
|
||||
}
|
||||
|
||||
if (staleRemoteTaskIds.length > 0) {
|
||||
logger.warn(`发现${staleRemoteTaskIds.length}个孤立的回调等待任务，正在移除...`);
|
||||
for (const remoteTaskId of staleRemoteTaskIds) {
|
||||
await initQueue.removeCallbackPendingTask(remoteTaskId);
|
||||
}
|
||||
logger.info(`已移除${staleRemoteTaskIds.length}个孤立的回调等待任务`);
|
||||
}
|
||||
|
||||
logger.info('回调等待队列恢复完成');
|
||||
return { totalCount: Object.keys(pendingTasks).length, staleCount: staleRemoteTaskIds.length };
|
||||
} catch (error) {
|
||||
logger.error('恢复回调等待队列失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverPlatformCounts() {
|
||||
logger.info('开始恢复平台计数...');
|
||||
|
||||
try {
|
||||
const platforms = await initQueue.getPlatforms();
|
||||
|
||||
for (const [platformKey, platformInfo] of Object.entries(platforms)) {
|
||||
const waitQueueLength = await redis.lLen(platformInfo.waitQueue);
|
||||
|
||||
const pollingKeys = await redis.keys(`${initQueue.prefix}:processPolling:${platformInfo.AIGC}:${platformInfo.platformName}`);
|
||||
let pollingCount = 0;
|
||||
for (const key of pollingKeys) {
|
||||
pollingCount += await redis.hLen(key);
|
||||
}
|
||||
|
||||
const pendingCount = await initQueue.getCallbackPendingCount();
|
||||
|
||||
logger.info(`平台 ${platformKey}: 等待队列=${waitQueueLength}, 轮询队列=${pollingCount}`);
|
||||
}
|
||||
|
||||
logger.info('平台计数恢复完成');
|
||||
} catch (error) {
|
||||
logger.error('恢复平台计数失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupExpiredMappings() {
|
||||
logger.info('开始清理过期的回调映射...');
|
||||
|
||||
try {
|
||||
const keys = await redis.keys(`${initQueue.callback}:*`);
|
||||
let cleanedCount = 0;
|
||||
|
||||
for (const key of keys) {
|
||||
const remoteTaskId = key.replace(`${initQueue.callback}:`, '');
|
||||
const taskId = await redis.get(key);
|
||||
|
||||
if (taskId) {
|
||||
const taskExists = await redis.exists(`${initQueue.prefix}:task:${taskId}`);
|
||||
|
||||
if (!taskExists) {
|
||||
await redis.del(key);
|
||||
await initQueue.removeCallbackPendingTask(remoteTaskId);
|
||||
cleanedCount++;
|
||||
logger.debug(`清理孤立映射: ${key} -> ${taskId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`清理完成，共清理${cleanedCount}个过期映射`);
|
||||
return cleanedCount;
|
||||
} catch (error) {
|
||||
logger.error('清理过期映射失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function fullRecovery() {
|
||||
logger.info('========== 开始完整队列恢复 ==========');
|
||||
|
||||
const results = {
|
||||
callbackQueue: null,
|
||||
callbackPending: null,
|
||||
platformCounts: null,
|
||||
expiredMappings: null
|
||||
};
|
||||
|
||||
try {
|
||||
results.callbackQueue = await recoverCallbackQueue();
|
||||
results.callbackPending = await recoverCallbackPendingQueue();
|
||||
results.platformCounts = await recoverPlatformCounts();
|
||||
results.expiredMappings = await cleanupExpiredMappings();
|
||||
|
||||
logger.info('========== 完整队列恢复完成 ==========');
|
||||
logger.info('恢复结果:', JSON.stringify(results, null, 2));
|
||||
|
||||
return results;
|
||||
} catch (error) {
|
||||
logger.error('完整队列恢复失败:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const command = args[0] || 'full';
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
if (!redis.isOpen) {
|
||||
await redis.connect();
|
||||
logger.info('Redis 连接成功');
|
||||
}
|
||||
|
||||
switch (command) {
|
||||
case 'callback':
|
||||
await recoverCallbackQueue();
|
||||
break;
|
||||
case 'pending':
|
||||
await recoverCallbackPendingQueue();
|
||||
break;
|
||||
case 'platforms':
|
||||
await recoverPlatformCounts();
|
||||
break;
|
||||
case 'cleanup':
|
||||
await cleanupExpiredMappings();
|
||||
break;
|
||||
case 'full':
|
||||
default:
|
||||
await fullRecovery();
|
||||
break;
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
logger.error('恢复操作失败:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
|
|
@ -85,6 +85,7 @@ const wait = createWorker('./worker_threads/wait/waiting.js');
|
|||
const polling = createWorker('./worker_threads/process/process.js');
|
||||
const result = createWorker('./worker_threads/result/result.js');
|
||||
const callback_result = createWorker('./worker_threads/callback_result/result.js');
|
||||
const callback_timeout = createWorker('./worker_threads/callback_timeout/callbackTimeout.js');
|
||||
const error = createWorker('./worker_threads/error/error.js');
|
||||
|
||||
// 发送消息给客户端的工具函数
|
||||
|
|
@ -99,8 +100,8 @@ async function sendMessageToClient(id, message, close = false, closeCode = 1000,
|
|||
if (socket && socket.readyState === WebSocket.OPEN && message) {
|
||||
try {
|
||||
socket.send(message);
|
||||
const messagePreview = typeof message === 'string' ? message.slice(0, 50) : JSON.stringify(message).slice(0, 50);
|
||||
logger.debug(`成功发送消息到客户端，id: ${id}, 消息: ${messagePreview}...`);
|
||||
// const messagePreview = typeof message === 'string' ? message.slice(0, 50) : JSON.stringify(message).slice(0, 50);
|
||||
// logger.debug(`成功发送消息到客户端，id: ${id}, 消息: ${messagePreview}...`);
|
||||
if (close) {
|
||||
socket.close(closeCode, closeReason);
|
||||
}
|
||||
|
|
@ -199,13 +200,52 @@ function createWebSocketServer() {
|
|||
// 处理收到的消息
|
||||
socket.on('message', (message) => {
|
||||
const messageStr = typeof message === 'string' ? message : message.toString();
|
||||
|
||||
// 首先检查是否为心跳消息
|
||||
if (messageStr === 'ping') {
|
||||
socket.send('pong'); // 回复心跳
|
||||
return;
|
||||
}
|
||||
|
||||
if (messageStr === 'pong') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (messageStr === 'please give me tasks') {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const msg = JSON.parse(messageStr);
|
||||
|
||||
if (msg.type === 'JWT_UPDATE') {
|
||||
logger.info(`收到 JWT_UPDATE 消息`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'CAPACITY_UPDATE') {
|
||||
logger.debug(`收到算力状态更新: 可用容量 = ${msg.data?.summary?.availableCapacity || 0}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'INSTANCE_ONLINE') {
|
||||
logger.debug(`收到实例上线: ${msg.data?.instanceId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'INSTANCE_OFFLINE') {
|
||||
logger.debug(`收到实例下线: ${msg.data?.instanceId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === 'HEARTBEAT') {
|
||||
socket.send(JSON.stringify({
|
||||
type: 'HEARTBEAT_ACK',
|
||||
data: { timestamp: new Date().toISOString() }
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
// 只检查前面100个字符是否包含 `"type": "generate"`，提高大消息处理性能
|
||||
const prefix = messageStr.slice(0, 50);
|
||||
if (prefix.includes('"type":"generate"') || prefix.includes("'type':'generate'")) {
|
||||
|
|
@ -232,7 +272,7 @@ function createWebSocketServer() {
|
|||
const heartbeatInterval = setInterval(() => {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
socket.send('ping');
|
||||
logger.debug(`向 ${id} 号后端发送心跳`);
|
||||
// logger.debug(`向 ${id} 号后端发送心跳`);
|
||||
}
|
||||
}, 30000); // 每30秒发送一次心跳
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
module.exports = {
|
||||
apps: [{
|
||||
name: 'digitalHuman-websocketTask-v2',
|
||||
name: 'digitalHuman-websocketTask-v3',
|
||||
script: './webSocket.js',
|
||||
cwd: './',
|
||||
args: '',
|
||||
|
|
@ -34,11 +34,5 @@ module.exports = {
|
|||
// 监控和重启设置
|
||||
kill_timeout: 1600,
|
||||
restart_delay: 4000,
|
||||
|
||||
// 环境变量
|
||||
env_production: {
|
||||
NODE_ENV: 'production',
|
||||
PORT: 8080
|
||||
}
|
||||
}]
|
||||
};
|
||||
|
|
@ -2,146 +2,181 @@ import { parentPort } from 'worker_threads'
|
|||
import redis from '../../redis/index.js'
|
||||
import initQueue from '../../redis/initQueue.js'
|
||||
|
||||
// 日志工具函数
|
||||
const logger = {
|
||||
info: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.log(`[${timestamp}] INFO: ${message}`);
|
||||
console.log(`[${timestamp}] [CallbackResult] INFO: ${message}`);
|
||||
},
|
||||
warn: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.warn(`[${timestamp}] [CallbackResult] WARN: ${message}`);
|
||||
},
|
||||
error: (message, error) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.error(`[${timestamp}] ERROR: ${message}`, error || '');
|
||||
console.error(`[${timestamp}] [CallbackResult] ERROR: ${message}`, error || '');
|
||||
},
|
||||
debug: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.debug(`[${timestamp}] DEBUG: ${message}`);
|
||||
console.debug(`[${timestamp}] [CallbackResult] DEBUG: ${message}`);
|
||||
}
|
||||
};
|
||||
|
||||
// 批量获取处理任务的信息
|
||||
const MAX_RETRIES = 3;
|
||||
const STUCK_TASK_THRESHOLD = 300000;
|
||||
|
||||
async function getTasks() {
|
||||
try {
|
||||
// 从回调结果队列列表获取最多50个数据
|
||||
const taskIds = await redis.lRange(initQueue.callback, 0, 49);
|
||||
const taskCountMap = new Map()
|
||||
|
||||
const taskCountMap = new Map();
|
||||
const orphanTaskIds = [];
|
||||
const processedTaskIds = [];
|
||||
const seenTaskIds = new Set();
|
||||
|
||||
if (taskIds.length === 0) {
|
||||
// 如果没有任务ID，强制将回调队列任务数设置为0
|
||||
redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', 0);
|
||||
await redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', 0);
|
||||
logger.debug('回调结果队列为空，已重置任务数为0');
|
||||
return [];
|
||||
}
|
||||
|
||||
logger.debug('回调结果队列任务ID:', taskIds);
|
||||
// logger.debug(`回调结果队列任务ID: ${taskIds.slice(0, 5).join(', ')}${taskIds.length > 5 ? '...' : ''}`);
|
||||
|
||||
// 批量获取任务的backendId和resultData字段
|
||||
const multi = redis.multi();
|
||||
for (const taskId of taskIds) {
|
||||
// 只获取需要的字段
|
||||
multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'backendId');
|
||||
multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'resultData');
|
||||
multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'AIGC');
|
||||
multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'platform');
|
||||
multi.hGetAll(`${initQueue.prefix}:task:${taskId}`);
|
||||
}
|
||||
|
||||
const results = await multi.exec();
|
||||
|
||||
// 发送结果给客户端
|
||||
const processedTaskIds = [];
|
||||
|
||||
for (let i = 0; i < taskIds.length; i++) {
|
||||
// 从结果数组中提取对应字段（每4个结果对应一个任务）
|
||||
const backendId = results[i * 4] || '';
|
||||
const resultData = results[i * 4 + 1] || '';
|
||||
const aigc = results[i * 4 + 2] || 'default';
|
||||
const platform = results[i * 4 + 3] || 'default';
|
||||
const taskId = taskIds[i];
|
||||
|
||||
if (seenTaskIds.has(taskId)) {
|
||||
continue;
|
||||
}
|
||||
seenTaskIds.add(taskId);
|
||||
|
||||
const taskInfo = results[i];
|
||||
|
||||
if (backendId) {
|
||||
try {
|
||||
// 直接打包结果和任务ID，发送给主线程
|
||||
const resultWithTaskId = {
|
||||
taskId: taskId,
|
||||
result: resultData
|
||||
};
|
||||
if (!taskInfo || Object.keys(taskInfo).length === 0) {
|
||||
logger.warn(`任务数据不存在或为空: ${taskId}，标记为孤立任务`);
|
||||
orphanTaskIds.push(taskId);
|
||||
continue;
|
||||
}
|
||||
|
||||
const backendId = taskInfo.backendId;
|
||||
const resultData = taskInfo.resultData;
|
||||
const aigc = taskInfo.AIGC || 'default';
|
||||
const platform = taskInfo.platform || 'default';
|
||||
|
||||
if (!backendId) {
|
||||
logger.warn(`任务缺少backendId: ${taskId}，标记为孤立任务`);
|
||||
orphanTaskIds.push(taskId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const resultWithTaskId = {
|
||||
taskId: taskId,
|
||||
result: resultData
|
||||
};
|
||||
|
||||
parentPort.postMessage({
|
||||
type: 'success',
|
||||
backendId: backendId,
|
||||
message: JSON.stringify(resultWithTaskId)
|
||||
});
|
||||
parentPort.postMessage({
|
||||
type: 'success',
|
||||
backendId: backendId,
|
||||
message: JSON.stringify(resultWithTaskId)
|
||||
});
|
||||
|
||||
logger.debug(`成功发送结果给客户端，taskId: ${taskId}`);
|
||||
logger.debug(`成功发送结果给客户端，taskId: ${taskId}`)
|
||||
|
||||
// 统计需要减少的平台任务数
|
||||
const key = `${aigc}:${platform}`;
|
||||
console.log('key:', key);
|
||||
if(taskCountMap.has(key)){
|
||||
taskCountMap.set(key, taskCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskCountMap.set(key, 1);
|
||||
}
|
||||
processedTaskIds.push(taskId);
|
||||
} catch (parseError) {
|
||||
logger.error(`发送结果给客户端失败: ${taskId}`, parseError);
|
||||
const key = `${aigc}:${platform}`;
|
||||
if(taskCountMap.has(key)){
|
||||
taskCountMap.set(key, taskCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskCountMap.set(key, 1);
|
||||
}
|
||||
processedTaskIds.push(taskId);
|
||||
} catch (sendError) {
|
||||
logger.error(`发送结果给客户端失败: ${taskId}`, sendError);
|
||||
processedTaskIds.push(taskId);
|
||||
}
|
||||
}
|
||||
|
||||
// 只有在成功发送结果后才执行后续操作
|
||||
if (processedTaskIds.length > 0) {
|
||||
// 使用原子操作执行多项任务
|
||||
const multi = redis.multi();
|
||||
if (orphanTaskIds.length > 0) {
|
||||
logger.warn(`发现${orphanTaskIds.length}个孤立任务，将从队列中移除`);
|
||||
const multiOrphan = redis.multi();
|
||||
for (const orphanId of orphanTaskIds) {
|
||||
multiOrphan.lRem(initQueue.callback, 0, orphanId);
|
||||
}
|
||||
await multiOrphan.exec();
|
||||
}
|
||||
|
||||
if (processedTaskIds.length > 0 || orphanTaskIds.length > 0) {
|
||||
const multiFinal = redis.multi();
|
||||
|
||||
// 1. 移除已获取的任务ID
|
||||
multi.lTrim(initQueue.callback, processedTaskIds.length, -1);
|
||||
if (processedTaskIds.length > 0) {
|
||||
multiFinal.lTrim(initQueue.callback, processedTaskIds.length, -1);
|
||||
}
|
||||
|
||||
// 2. 执行所有Redis操作
|
||||
await multi.exec();
|
||||
await multiFinal.exec();
|
||||
|
||||
// 3. 更新平台任务数（如果有需要更新的任务）
|
||||
if (taskCountMap.size > 0) {
|
||||
await initQueue.reducePlatformsProcess(taskCountMap);
|
||||
}
|
||||
|
||||
// 4. 更新回调队列任务数
|
||||
await initQueue.reduceCQtasksALL(processedTaskIds.length);
|
||||
const totalProcessed = processedTaskIds.length + orphanTaskIds.length;
|
||||
await initQueue.reduceCQtasksALL(totalProcessed);
|
||||
|
||||
logger.debug(`已处理${processedTaskIds.length}个回调结果任务，发送结果后结束`);
|
||||
logger.info(`已处理${processedTaskIds.length}个回调结果任务，移除${orphanTaskIds.length}个孤立任务`);
|
||||
}
|
||||
|
||||
return taskIds;
|
||||
} catch (error) {
|
||||
logger.error('处理回调结果任务失败:', error);
|
||||
// 出错时，强制将回调队列任务数设置为0，避免死循环
|
||||
await redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', 0);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// 持续执行批量处理
|
||||
async function syncQueueState() {
|
||||
try {
|
||||
const actualLength = await redis.lLen(initQueue.callback);
|
||||
const storedCount = await initQueue.getCQtasksALL();
|
||||
|
||||
if (actualLength !== storedCount) {
|
||||
logger.warn(`检测到队列状态不一致: 实际长度=${actualLength}, 存储计数=${storedCount}`);
|
||||
await redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', actualLength);
|
||||
logger.info(`已同步队列状态: CQtasksALL=${actualLength}`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('同步队列状态失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
logger.info('回调结果处理线程启动');
|
||||
|
||||
let checkCount = 0;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
// 判断结果队列是否有可发送的任务
|
||||
const rqTasksAll = await initQueue.getCQtasksALL();
|
||||
checkCount++;
|
||||
|
||||
const rqTasksAll = await initQueue.getCQtasksALL();
|
||||
|
||||
if (rqTasksAll !== 0) {
|
||||
logger.info('回调结果队列有任务可处理，数量:', rqTasksAll);
|
||||
// logger.debug('回调结果队列任务数量:', rqTasksAll);
|
||||
// 先处理任务，再减少队列任务数
|
||||
await getTasks(); // 处理结果队列的任务
|
||||
logger.info(`回调结果队列有任务可处理，数量: ${rqTasksAll}`);
|
||||
await getTasks();
|
||||
|
||||
// 添加延迟，避免高频率执行
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
} else {
|
||||
// 回调结果队列无任务可处理，等待10秒后重试
|
||||
if (checkCount % 10 === 0) {
|
||||
await syncQueueState();
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 10000));
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error('持续处理回调结果任务失败:', error);
|
||||
// 出错后等待5秒再重试
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,183 @@
|
|||
import { parentPort } from 'worker_threads'
|
||||
import redis from '../../redis/index.js'
|
||||
import initQueue from '../../redis/initQueue.js'
|
||||
|
||||
const logger = {
|
||||
info: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.log(`[${timestamp}] [CallbackTimeout] INFO: ${message}`);
|
||||
},
|
||||
warn: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.warn(`[${timestamp}] [CallbackTimeout] WARN: ${message}`);
|
||||
},
|
||||
error: (message, error) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.error(`[${timestamp}] [CallbackTimeout] ERROR: ${message}`, error || '');
|
||||
},
|
||||
debug: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.debug(`[${timestamp}] [CallbackTimeout] DEBUG: ${message}`);
|
||||
}
|
||||
};
|
||||
|
||||
async function checkTimeoutTasks() {
|
||||
try {
|
||||
const pendingTasks = await initQueue.getCallbackPendingTasks();
|
||||
const now = Date.now();
|
||||
const timeoutTasks = [];
|
||||
|
||||
for (const [remoteTaskId, taskInfoStr] of Object.entries(pendingTasks)) {
|
||||
try {
|
||||
const taskInfo = JSON.parse(taskInfoStr);
|
||||
const elapsed = now - taskInfo.createdAt;
|
||||
|
||||
if (elapsed > initQueue.CALLBACK_TIMEOUT) {
|
||||
logger.warn(`检测到超时任务: remoteTaskId=${remoteTaskId}, taskId=${taskInfo.taskId}, 已等待${Math.round(elapsed/1000)}秒`);
|
||||
timeoutTasks.push({
|
||||
remoteTaskId,
|
||||
taskId: taskInfo.taskId,
|
||||
aigc: taskInfo.aigc,
|
||||
platform: taskInfo.platform,
|
||||
elapsed
|
||||
});
|
||||
}
|
||||
} catch (parseError) {
|
||||
logger.error(`解析任务信息失败: ${remoteTaskId}`, parseError);
|
||||
await initQueue.removeCallbackPendingTask(remoteTaskId);
|
||||
}
|
||||
}
|
||||
|
||||
return timeoutTasks;
|
||||
} catch (error) {
|
||||
logger.error('检查超时任务失败:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function processTimeoutTasks(timeoutTasks) {
|
||||
if (timeoutTasks.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const multi = redis.multi();
|
||||
const taskCountMap = new Map();
|
||||
|
||||
for (const task of timeoutTasks) {
|
||||
const taskKey = `${initQueue.prefix}:task:${task.taskId}`;
|
||||
|
||||
const errorMessage = JSON.stringify({
|
||||
error: 'callback_timeout',
|
||||
message: `回调超时，等待时间超过${Math.round(initQueue.CALLBACK_TIMEOUT/1000)}秒`,
|
||||
elapsed: task.elapsed
|
||||
});
|
||||
|
||||
multi.hSet(taskKey, 'resultData', errorMessage);
|
||||
multi.hSet(taskKey, 'status', 'failed');
|
||||
multi.lPush(initQueue.errorList, task.taskId);
|
||||
multi.del(`${initQueue.callback}:${task.remoteTaskId}`);
|
||||
|
||||
const key = `${task.aigc}:${task.platform}`;
|
||||
if (taskCountMap.has(key)) {
|
||||
taskCountMap.set(key, taskCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskCountMap.set(key, 1);
|
||||
}
|
||||
|
||||
await initQueue.removeCallbackPendingTask(task.remoteTaskId);
|
||||
}
|
||||
|
||||
await multi.exec();
|
||||
|
||||
if (taskCountMap.size > 0) {
|
||||
await initQueue.addEQtaskALL(timeoutTasks.length);
|
||||
logger.info(`已处理${timeoutTasks.length}个超时任务，已推入错误队列`);
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupStaleMappings() {
|
||||
try {
|
||||
const keys = await redis.keys(`${initQueue.callback}:*`);
|
||||
let cleanedCount = 0;
|
||||
|
||||
for (const key of keys) {
|
||||
if (key === initQueue.callbackPending) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const remoteTaskId = key.replace(`${initQueue.callback}:`, '');
|
||||
const taskId = await redis.get(key);
|
||||
|
||||
if (taskId) {
|
||||
const taskExists = await redis.exists(`${initQueue.prefix}:task:${taskId}`);
|
||||
|
||||
if (!taskExists) {
|
||||
await redis.del(key);
|
||||
await initQueue.removeCallbackPendingTask(remoteTaskId);
|
||||
cleanedCount++;
|
||||
logger.debug(`清理孤立映射: ${key} -> ${taskId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (cleanedCount > 0) {
|
||||
logger.info(`清理了${cleanedCount}个孤立的回调映射`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('清理孤立映射失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function syncCounters() {
|
||||
try {
|
||||
const actualQueueLength = await redis.lLen(initQueue.callback);
|
||||
const storedCount = await initQueue.getCQtasksALL();
|
||||
|
||||
if (actualQueueLength !== storedCount) {
|
||||
logger.warn(`检测到计数器不一致: 实际队列长度=${actualQueueLength}, 存储计数=${storedCount}`);
|
||||
await redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', actualQueueLength);
|
||||
logger.info(`已同步计数器: CQtasksALL=${actualQueueLength}`);
|
||||
}
|
||||
|
||||
|
||||
} catch (error) {
|
||||
logger.error('同步计数器失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
logger.info('回调超时检测线程启动');
|
||||
logger.info(`回调超时时间: ${initQueue.CALLBACK_TIMEOUT/1000}秒`);
|
||||
|
||||
let checkCount = 0;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
checkCount++;
|
||||
|
||||
const timeoutTasks = await checkTimeoutTasks();
|
||||
|
||||
if (timeoutTasks.length > 0) {
|
||||
logger.warn(`发现${timeoutTasks.length}个超时任务`);
|
||||
await processTimeoutTasks(timeoutTasks);
|
||||
}
|
||||
|
||||
if (checkCount % 10 === 0) {
|
||||
await cleanupStaleMappings();
|
||||
await syncCounters();
|
||||
checkCount = 0;
|
||||
}
|
||||
|
||||
const pendingCount = await initQueue.getCallbackPendingCount();
|
||||
if (pendingCount > 0) {
|
||||
logger.debug(`当前等待回调的任务数: ${pendingCount}`);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 30000));
|
||||
|
||||
} catch (error) {
|
||||
logger.error('回调超时检测循环出错:', error);
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
|
@ -4,17 +4,21 @@ import initQueue from '../../redis/initQueue.js'
|
|||
|
||||
// 日志工具函数
|
||||
const logger = {
|
||||
info: (message) => {
|
||||
info: (...args) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.log(`[${timestamp}] INFO: ${message}`);
|
||||
console.log(`[${timestamp}] INFO:`, ...args);
|
||||
},
|
||||
error: (message, error) => {
|
||||
error: (...args) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.error(`[${timestamp}] ERROR: ${message}`, error || '');
|
||||
console.error(`[${timestamp}] ERROR:`, ...args);
|
||||
},
|
||||
debug: (message) => {
|
||||
debug: (...args) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.debug(`[${timestamp}] DEBUG: ${message}`);
|
||||
console.debug(`[${timestamp}] DEBUG:`, ...args);
|
||||
},
|
||||
warn: (...args) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.warn(`[${timestamp}] WARN:`, ...args);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -24,12 +28,16 @@ async function getTasks() {
|
|||
const taskIds = await redis.lRange(initQueue.errorList, 0, -1);
|
||||
|
||||
if (taskIds.length === 0) {
|
||||
const currentCount = await initQueue.getEQtaskALL();
|
||||
if (currentCount > 0) {
|
||||
await initQueue.reduceEQtaskALL(currentCount);
|
||||
logger.info(`错误队列计数器修正: ${currentCount} -> 0`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
logger.debug('错误队列任务ID:', taskIds);
|
||||
|
||||
// 批量获取错误任务信息
|
||||
const multi = redis.multi();
|
||||
for (const taskId of taskIds) {
|
||||
multi.hGetAll(`${initQueue.prefix}:task:${taskId}`);
|
||||
|
|
@ -39,8 +47,8 @@ async function getTasks() {
|
|||
|
||||
let processedCount = 0;
|
||||
const taskCountMap = new Map();
|
||||
const validTaskIds = [];
|
||||
|
||||
// 处理结果
|
||||
for (let i = 0; i < taskIds.length; i++) {
|
||||
const taskId = taskIds[i];
|
||||
const taskInfo = results[i];
|
||||
|
|
@ -49,7 +57,6 @@ async function getTasks() {
|
|||
try {
|
||||
logger.debug('错误队列任务数据:', taskInfo);
|
||||
|
||||
// 直接打包错误信息和任务ID，发送给主线程
|
||||
const resultWithTaskId = {
|
||||
taskId: taskInfo.taskId,
|
||||
result: taskInfo.resultData
|
||||
|
|
@ -61,8 +68,8 @@ async function getTasks() {
|
|||
message: JSON.stringify(resultWithTaskId)
|
||||
});
|
||||
processedCount++;
|
||||
validTaskIds.push(taskId);
|
||||
|
||||
// 统计需要减少计数的任务
|
||||
const key = `${taskInfo.AIGC}:${taskInfo.platform}`;
|
||||
if(taskCountMap.has(key)){
|
||||
taskCountMap.set(key, taskCountMap.get(key) + 1);
|
||||
|
|
@ -71,24 +78,25 @@ async function getTasks() {
|
|||
}
|
||||
} catch (parseError) {
|
||||
logger.error(`解析错误任务数据失败: ${taskInfo.resultData}`, parseError);
|
||||
validTaskIds.push(taskId);
|
||||
}
|
||||
} else {
|
||||
validTaskIds.push(taskId);
|
||||
logger.warn(`错误任务信息不存在或无效，将清理: taskId=${taskId}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 删除已处理的错误任务
|
||||
if (processedCount > 0) {
|
||||
if (validTaskIds.length > 0) {
|
||||
const deleteMulti = redis.multi();
|
||||
for (const taskId of taskIds) {
|
||||
for (const taskId of validTaskIds) {
|
||||
deleteMulti.lRem(initQueue.errorList, 1, taskId);
|
||||
// 同时删除tasks中的任务信息（从哈希存储中删除），使用项目前缀
|
||||
deleteMulti.del(`${initQueue.prefix}:task:${taskId}`);
|
||||
}
|
||||
await deleteMulti.exec();
|
||||
|
||||
await initQueue.reduceEQtaskALL(processedCount);
|
||||
logger.info(`处理了 ${processedCount} 个错误任务`);
|
||||
await initQueue.reduceEQtaskALL(validTaskIds.length);
|
||||
logger.info(`处理了 ${processedCount} 个错误任务，清理了 ${validTaskIds.length} 个任务记录`);
|
||||
|
||||
// 减少等待队列的计数（错误任务来自等待队列）
|
||||
if (taskCountMap.size > 0) {
|
||||
await initQueue.reducePlatformsWait(taskCountMap);
|
||||
}
|
||||
|
|
@ -114,7 +122,7 @@ async function getTasks() {
|
|||
} else {
|
||||
// 没有可处理的错误任务，等待15秒后重试
|
||||
await new Promise(resolve => setTimeout(resolve, 15000));
|
||||
logger.debug('错误队列无任务可处理');
|
||||
// logger.debug('错误队列无任务可处理');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('持续处理错误任务失败:', error);
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ async function storeSuccessTasks(SuccessTasks) {
|
|||
const multi = redis.multi();
|
||||
|
||||
for (const task of SuccessTasks) {
|
||||
const taskId = task.taskid || task.taskId;
|
||||
const taskId = task.taskId;
|
||||
const remoteTaskId = task.remoteTaskId;
|
||||
const aigc = task.aigc;
|
||||
const platform = task.platform;
|
||||
|
|
@ -79,14 +79,17 @@ async function storeSuccessTasks(SuccessTasks) {
|
|||
await multi.exec();
|
||||
console.log(`[pollingTask] 已完成Redis批量操作，删除了轮询任务并存储到结果队列`);
|
||||
|
||||
// 更新平台计数（使用原子操作）
|
||||
// 无论成功还是失败任务，都需要减少PQtasks计数器
|
||||
if (taskCountMap.size > 0) {
|
||||
await initQueue.reducePlatformsProcess(taskCountMap);
|
||||
console.log(`[pollingTask] 已更新平台计数: ${JSON.stringify(Array.from(taskCountMap.entries()))}`);
|
||||
console.log(`[pollingTask] 已更新成功任务平台计数: ${JSON.stringify(Array.from(taskCountMap.entries()))}`);
|
||||
}
|
||||
|
||||
// 更新错误队列计数
|
||||
// 失败任务也需要减少PQtasks计数器，因为它们不再处于处理状态
|
||||
if (taskErrorCountMap.size > 0) {
|
||||
await initQueue.reducePlatformsProcess(taskErrorCountMap);
|
||||
console.log(`[pollingTask] 已更新失败任务平台计数: ${JSON.stringify(Array.from(taskErrorCountMap.entries()))}`);
|
||||
|
||||
const totalErrorCount = Object.values(taskErrorCountMap).reduce((a, b) => a + b, 0);
|
||||
initQueue.addEQtaskALL(totalErrorCount);
|
||||
console.log(`[pollingTask] 已更新错误队列计数: ${JSON.stringify(Array.from(taskErrorCountMap.entries()))}`);
|
||||
|
|
|
|||
|
|
@ -4,12 +4,10 @@ import initQueue from '../../redis/initQueue.js';
|
|||
import { externalPostRequest } from '../../outside/generat.js';
|
||||
import { platformData } from '../../config/Config.js';
|
||||
|
||||
// Redis key 定义
|
||||
const REDIS_KEYS = {
|
||||
JWT: `${process.env.PROJECT_PREFIX}:md:jwt`
|
||||
};
|
||||
|
||||
// 从 Redis 获取 JWT Token
|
||||
async function getJwtTokenFromRedis() {
|
||||
try {
|
||||
const token = await redis.get(REDIS_KEYS.JWT);
|
||||
|
|
@ -20,15 +18,11 @@ async function getJwtTokenFromRedis() {
|
|||
}
|
||||
}
|
||||
|
||||
// 批量转发待处理任务到各外部平台
|
||||
async function generatTask(tasksData) {
|
||||
// 预先获取 JWT Token，避免每个任务都读取一次 Redis
|
||||
const jwtToken = await getJwtTokenFromRedis();
|
||||
console.log(`[generatTask] 预获取 JWT Token: ${jwtToken ? '存在' : '不存在'}`);
|
||||
|
||||
const generatTasks = []
|
||||
for (const task of tasksData) {
|
||||
// 将 jwtToken 传递给 externalPostRequest
|
||||
const generatTaskPromise = externalPostRequest(task, jwtToken)
|
||||
generatTasks.push(generatTaskPromise)
|
||||
}
|
||||
|
|
@ -37,14 +31,12 @@ async function generatTask(tasksData) {
|
|||
const responseTasks = await Promise.all(generatTasks)
|
||||
return responseTasks
|
||||
} catch (error) {
|
||||
console.error('Error:', error);
|
||||
return []; // 确保总是返回数组
|
||||
console.error('[generatTask] 批量请求出错:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// 批量储存外部平台返回的任务数据到处理队列
|
||||
async function storeGeneratTasks(tasks) {
|
||||
// 确保tasks是数组
|
||||
if (!tasks || !Array.isArray(tasks)) {
|
||||
console.error('storeGeneratTasks函数接收到无效的tasks参数:', tasks);
|
||||
return;
|
||||
|
|
@ -54,163 +46,144 @@ async function storeGeneratTasks(tasks) {
|
|||
let errorCount = 0;
|
||||
const taskErrorCountMap = new Map();
|
||||
const taskCountMap = new Map();
|
||||
|
||||
for (const task of tasks) {
|
||||
//错误任务
|
||||
if(task.remoteTaskId?.type === 2){
|
||||
console.log('储存在错误队列', task);
|
||||
console.error(`[generatTask] 任务失败: taskId=${task.taskId}, 错误:`, task.remoteTaskId.message);
|
||||
const aigc = task.AIGC || task.aigc;
|
||||
const platform = task.platform || task.platformName;
|
||||
|
||||
// 存储错误信息到任务数据中
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', JSON.stringify(task.remoteTaskId.message));
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed');
|
||||
|
||||
// 推送任务ID到错误列表
|
||||
multi.lPush(initQueue.errorList, task.taskId);
|
||||
|
||||
errorCount++;
|
||||
// 存储相关平台信息
|
||||
const key = `${aigc}:${platform}`;
|
||||
if(taskErrorCountMap.has(key)){
|
||||
taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskErrorCountMap.set(key, 1);
|
||||
}
|
||||
continue // 跳过错误任务
|
||||
continue;
|
||||
}
|
||||
|
||||
// 处理成功的任务
|
||||
let externalTaskId;
|
||||
if (task.remoteTaskId?.type === 1 && task.remoteTaskId?.data) {
|
||||
// 使用解析后的响应数据提取外部平台任务ID
|
||||
try {
|
||||
const responseData = task.remoteTaskId.data;
|
||||
|
||||
// 直接处理响应数据，提取任务ID
|
||||
const platform = task.platform || task.platformName;
|
||||
if ((responseData.msg === 'success' || platform === 'coze') && responseData.code === 0) {
|
||||
// Coze平台返回的是execute_id，其他平台返回的是data.taskId
|
||||
if (platform === 'coze') {
|
||||
externalTaskId = responseData.execute_id;
|
||||
} else {
|
||||
externalTaskId = responseData.data?.taskId;
|
||||
}
|
||||
|
||||
if (externalTaskId) {
|
||||
console.log('成功提取外部平台任务ID:', externalTaskId);
|
||||
} else {
|
||||
console.error('无法从响应中提取外部平台任务ID:', responseData);
|
||||
// 视为错误任务
|
||||
const errorMessage = JSON.stringify({ message: '无法从响应中提取外部平台任务ID', response: responseData });
|
||||
const responseData = task.remoteTaskId.data;
|
||||
|
||||
if (typeof responseData === 'string') {
|
||||
externalTaskId = responseData;
|
||||
} else if (typeof responseData === 'object' && responseData !== null) {
|
||||
try {
|
||||
const platform = task.platform || task.platformName;
|
||||
if ((responseData.msg === 'success' || platform === 'coze') && responseData.code === 0) {
|
||||
if (platform === 'coze') {
|
||||
externalTaskId = responseData.execute_id;
|
||||
} else {
|
||||
externalTaskId = responseData.data?.taskId;
|
||||
}
|
||||
|
||||
// 存储错误信息到任务数据中
|
||||
if (!externalTaskId) {
|
||||
console.error(`[generatTask] 无法提取任务ID: taskId=${task.taskId}`);
|
||||
const errorMessage = JSON.stringify({ message: '无法从响应中提取外部平台任务ID', response: responseData });
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage);
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed');
|
||||
multi.lPush(initQueue.errorList, task.taskId);
|
||||
errorCount++;
|
||||
const key = `${task.AIGC}:${task.platform}`;
|
||||
if(taskErrorCountMap.has(key)){
|
||||
taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskErrorCountMap.set(key, 1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
console.error(`[generatTask] 平台返回错误: taskId=${task.taskId}, 响应:`, responseData);
|
||||
const aigc = task.AIGC || task.aigc;
|
||||
const platform = task.platform || task.platformName;
|
||||
const errorMessage = JSON.stringify(responseData);
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage);
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed');
|
||||
|
||||
// 推送任务ID到错误列表
|
||||
multi.lPush(initQueue.errorList, task.taskId);
|
||||
|
||||
errorCount++;
|
||||
// 存储相关平台信息
|
||||
const key = `${task.AIGC}:${task.platform}`;
|
||||
const key = `${aigc}:${platform}`;
|
||||
if(taskErrorCountMap.has(key)){
|
||||
taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskErrorCountMap.set(key, 1);
|
||||
}
|
||||
continue; // 跳过错误任务
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
console.error('外部平台返回错误:', responseData);
|
||||
// 视为错误任务
|
||||
const aigc = task.AIGC || task.aigc;
|
||||
const platform = task.platform || task.platformName;
|
||||
const errorMessage = JSON.stringify(responseData);
|
||||
|
||||
// 存储错误信息到任务数据中
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage);
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed');
|
||||
|
||||
// 推送任务ID到错误列表
|
||||
multi.lPush(initQueue.errorList, task.taskId);
|
||||
|
||||
errorCount++;
|
||||
// 存储相关平台信息
|
||||
const key = `${aigc}:${platform}`;
|
||||
if(taskErrorCountMap.has(key)){
|
||||
taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1);
|
||||
} catch (extractError) {
|
||||
console.error(`[generatTask] 提取任务ID失败: taskId=${task.taskId}`, extractError);
|
||||
const aigc = task.AIGC || task.aigc;
|
||||
const platform = task.platform || task.platformName;
|
||||
const errorMessage = JSON.stringify({ message: '提取外部平台任务ID失败', error: extractError.message });
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage);
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed');
|
||||
multi.lPush(initQueue.errorList, task.taskId);
|
||||
errorCount++;
|
||||
const key = `${aigc}:${platform}`;
|
||||
if(taskErrorCountMap.has(key)){
|
||||
taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskErrorCountMap.set(key, 1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
taskErrorCountMap.set(key, 1);
|
||||
}
|
||||
continue; // 跳过错误任务
|
||||
}
|
||||
} catch (extractError) {
|
||||
console.error('提取外部平台任务ID失败:', extractError);
|
||||
// 视为错误任务
|
||||
console.error(`[generatTask] remoteTaskId.data 类型异常: taskId=${task.taskId}, type=${typeof responseData}`);
|
||||
const aigc = task.AIGC || task.aigc;
|
||||
const platform = task.platform || task.platformName;
|
||||
const errorMessage = JSON.stringify({ message: '提取外部平台任务ID失败', error: extractError.message });
|
||||
|
||||
// 存储错误信息到任务数据中
|
||||
const errorMessage = JSON.stringify({ message: 'remoteTaskId.data 类型异常', type: typeof responseData });
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage);
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed');
|
||||
|
||||
// 推送任务ID到错误列表
|
||||
multi.lPush(initQueue.errorList, task.taskId);
|
||||
|
||||
errorCount++;
|
||||
// 存储相关平台信息
|
||||
const key = `${aigc}:${platform}`;
|
||||
if(taskErrorCountMap.has(key)){
|
||||
taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1);
|
||||
} else {
|
||||
taskErrorCountMap.set(key, 1);
|
||||
}
|
||||
continue; // 跳过错误任务
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
// 直接使用remoteTaskId作为外部平台任务ID
|
||||
externalTaskId = task.remoteTaskId;
|
||||
}
|
||||
|
||||
//回调任务
|
||||
const aigc = task.AIGC || task.aigc;
|
||||
const platform = task.platform || task.platformName;
|
||||
|
||||
// console.log(`[generatTask] 任务映射: taskId=${task.taskId}, externalTaskId=${externalTaskId}, platform=${platform}`);
|
||||
|
||||
if(platformData.callback.includes(platform)) {
|
||||
console.log('储存在回调队列', externalTaskId, task.taskId);
|
||||
multi.set(`${initQueue.callback}:${externalTaskId}`, task.taskId)
|
||||
} else { // 轮询任务
|
||||
// 按平台+AIGC类型存储轮询任务
|
||||
multi.set(`${initQueue.callback}:${externalTaskId}`, task.taskId, { EX: 7200 });
|
||||
await initQueue.addCallbackPendingTask(externalTaskId, task.taskId, aigc, platform);
|
||||
} else {
|
||||
const pollingKey = `${initQueue.prefix}:processPolling:${aigc}:${platform}`;
|
||||
// 从task中提取workflow_id，优先使用task.workflowId
|
||||
let workflowId = task.workflowId || '';
|
||||
try {
|
||||
if (!workflowId && task.taskData) {
|
||||
// taskData 已经是字符串，直接解析
|
||||
const taskDataParsed = JSON.parse(task.taskData);
|
||||
workflowId = taskDataParsed.workflow_id || '';
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[generatTask] 解析taskData获取workflow_id失败:', e);
|
||||
}
|
||||
console.log(`[generatTask] 提取到的workflowId: ${workflowId}`);
|
||||
// 确保workflowId被传递到轮询任务中
|
||||
const pollingData = {
|
||||
taskId: task.taskId,
|
||||
platform: platform,
|
||||
AIGC: aigc,
|
||||
workflowId: workflowId // 包含workflowId，为空则使用空字符串
|
||||
workflowId: workflowId
|
||||
};
|
||||
console.log(`[generatTask] 添加轮询任务: pollingKey=${pollingKey}, externalTaskId=${externalTaskId}, pollingData=${JSON.stringify(pollingData)}`);
|
||||
multi.hSet(pollingKey, externalTaskId, JSON.stringify(pollingData))
|
||||
multi.hSet(pollingKey, externalTaskId, JSON.stringify(pollingData));
|
||||
}
|
||||
|
||||
// 更新任务信息，添加 remoteTaskId 字段
|
||||
multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'remoteTaskId', externalTaskId);
|
||||
// 确保任务有2小时的过期时间
|
||||
multi.expire(`${initQueue.prefix}:task:${task.taskId}`, 7200);
|
||||
|
||||
// 记录相关队列处理的任务数
|
||||
const key = `${aigc}:${platform}`;
|
||||
if(taskCountMap.has(key)){
|
||||
taskCountMap.set(key, taskCountMap.get(key) + 1);
|
||||
|
|
@ -219,11 +192,12 @@ async function storeGeneratTasks(tasks) {
|
|||
}
|
||||
}
|
||||
|
||||
// 更新平台信息
|
||||
if(errorCount > 0){
|
||||
initQueue.addEQtaskALL(errorCount) // 添加错误队列任务数量
|
||||
initQueue.addEQtaskALL(errorCount);
|
||||
}
|
||||
await multi.exec();
|
||||
|
||||
console.log(`[generatTask] 批次处理完成: 成功=${taskCountMap.size} 个平台, 错误=${errorCount} 个`);
|
||||
}
|
||||
|
||||
parentPort.on('message', async (tasksData) => {
|
||||
|
|
|
|||
|
|
@ -2,13 +2,11 @@ import { parentPort, Worker } from 'worker_threads';
|
|||
import redis from '../../redis/index.js';
|
||||
import initQueue from '../../redis/initQueue.js';
|
||||
|
||||
// Redis key 定义（与 mdWebSocketServer 保持一致）
|
||||
const REDIS_KEYS = {
|
||||
CAPACITY: `${process.env.PROJECT_PREFIX}:md:capacity`,
|
||||
JWT: `${process.env.PROJECT_PREFIX}:md:jwt`
|
||||
};
|
||||
|
||||
// 日志工具函数
|
||||
const logger = {
|
||||
info: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
|
|
@ -18,20 +16,17 @@ const logger = {
|
|||
const timestamp = new Date().toISOString();
|
||||
console.error(`[${timestamp}] ERROR: ${message}`, error || '');
|
||||
},
|
||||
debug: (message) => {
|
||||
warn: (message) => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.debug(`[${timestamp}] DEBUG: ${message}`);
|
||||
console.warn(`[${timestamp}] WARN: ${message}`);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 从 Redis 读取内部算力信息
|
||||
* @returns {Promise<number>} 内部可用算力
|
||||
*/
|
||||
let lastCapacityState = { capacity: null, hasJwt: null };
|
||||
|
||||
async function getInternalCapacityFromRedis() {
|
||||
try {
|
||||
const capacity = await redis.get(REDIS_KEYS.CAPACITY);
|
||||
console.log(`[MDWebSocketServer] 从 Redis 读取算力信息: ${capacity}`);
|
||||
return capacity ? parseInt(capacity, 10) : 0;
|
||||
} catch (error) {
|
||||
logger.error('从 Redis 读取算力信息失败:', error);
|
||||
|
|
@ -39,14 +34,9 @@ async function getInternalCapacityFromRedis() {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 Redis 读取 JWT Token
|
||||
* @returns {Promise<string|null>} JWT Token
|
||||
*/
|
||||
async function getJwtTokenFromRedis() {
|
||||
try {
|
||||
const token = await redis.get(REDIS_KEYS.JWT);
|
||||
console.log(`[MDWebSocketServer] 从 Redis 读取 JWT Token: ${token ? '存在' : '不存在'}`);
|
||||
return token;
|
||||
} catch (error) {
|
||||
logger.error('从 Redis 读取 JWT Token 失败:', error);
|
||||
|
|
@ -54,10 +44,6 @@ async function getJwtTokenFromRedis() {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 分发状态管理器
|
||||
* 用于跟踪 comfyui 任务在内部算力和外部算力之间的分配
|
||||
*/
|
||||
class DispatchStateManager {
|
||||
constructor() {
|
||||
this.internalCapacity = 0;
|
||||
|
|
@ -70,111 +56,114 @@ class DispatchStateManager {
|
|||
const jwtToken = await getJwtTokenFromRedis();
|
||||
this.hasJwtToken = !!jwtToken;
|
||||
this.assignedToInternal = 0;
|
||||
logger.info(`[DispatchStateManager] 初始化: 内部可用算力=${this.internalCapacity}, JWT=${this.hasJwtToken ? '存在' : '不存在'}`);
|
||||
}
|
||||
|
||||
getDispatchType(platformName) {
|
||||
if (platformName !== 'comfyui') {
|
||||
return null;
|
||||
async getDispatchType(platformName) {
|
||||
if (platformName === 'comfyui') {
|
||||
const internalCapacity = await getInternalCapacityFromRedis();
|
||||
const jwtToken = await getJwtTokenFromRedis();
|
||||
const hasJwtToken = !!jwtToken;
|
||||
|
||||
if (hasJwtToken && this.assignedToInternal < internalCapacity) {
|
||||
this.assignedToInternal++;
|
||||
return 'messageDispatcher';
|
||||
}
|
||||
|
||||
if (!hasJwtToken) {
|
||||
return 'error_no_jwt';
|
||||
}
|
||||
|
||||
return 'error_no_capacity';
|
||||
}
|
||||
|
||||
if (this.hasJwtToken && this.assignedToInternal < this.internalCapacity) {
|
||||
this.assignedToInternal++;
|
||||
logger.debug(`[DispatchStateManager] 分配到 messageDispatcher (已分配 ${this.assignedToInternal}/${this.internalCapacity})`);
|
||||
return 'messageDispatcher';
|
||||
if (platformName === 'runninghub' || platformName === 'coze') {
|
||||
return 'external';
|
||||
}
|
||||
|
||||
logger.debug(`[DispatchStateManager] 分配到 runninghub (内部已满或无JWT)`);
|
||||
return 'runninghub';
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const dispatchStateManager = new DispatchStateManager();
|
||||
|
||||
// 创建专门的线程池管理 Worker
|
||||
const generateWorker = new Worker(new URL('./GenerateWorkerManager.js', import.meta.url));
|
||||
|
||||
// 判断并发数，获取可进行任务处理的等待队列
|
||||
async function judgConcurrency() {
|
||||
async function julgConcurrency() {
|
||||
try {
|
||||
// 获取平台相关信息包括等待队列名与并发数，当前任务数等
|
||||
const platforms = await initQueue.getPlatforms();
|
||||
// 储存可进行任务处理的等待队列
|
||||
const wDeficiency = [];
|
||||
|
||||
logger.debug('获取到的平台信息：', platforms);
|
||||
|
||||
// 获取内部可用算力（用于 comfyui 平台）
|
||||
const internalCapacity = await getInternalCapacityFromRedis();
|
||||
const jwtToken = await getJwtTokenFromRedis();
|
||||
const hasInternalCapacity = internalCapacity > 0 && !!jwtToken;
|
||||
|
||||
if (hasInternalCapacity) {
|
||||
logger.info(`[judgConcurrency] 内部可用算力: ${internalCapacity}, JWT: 存在`);
|
||||
const hasJwtToken = !!jwtToken;
|
||||
const hasInternalCapacity = internalCapacity > 0 && hasJwtToken;
|
||||
|
||||
const currentState = { capacity: internalCapacity, hasJwt: hasJwtToken };
|
||||
if (currentState.capacity !== lastCapacityState.capacity ||
|
||||
currentState.hasJwt !== lastCapacityState.hasJwt) {
|
||||
logger.info(`[waiting] 内部算力状态变更: 容量=${internalCapacity}, JWT=${jwtToken ? '存在' : '不存在'}`);
|
||||
lastCapacityState = currentState;
|
||||
}
|
||||
|
||||
// 检查每个平台的实际队列长度
|
||||
for(const [aigcPfName, info] of Object.entries(platforms)) {
|
||||
try {
|
||||
// 直接检查 Redis 队列的实际长度
|
||||
const actualQueueLength = await redis.lLen(info.waitQueue);
|
||||
// logger.debug(`平台 ${aigcPfName} 信息：PQtasks=${info.PQtasks}, MAX_CONCURRENT=${info.MAX_CONCURRENT}, 实际队列长度=${actualQueueLength}`);
|
||||
|
||||
// 计算总并发能力
|
||||
let totalCapacity = info.MAX_CONCURRENT;
|
||||
|
||||
// 对于 comfyui 平台，如果有内部算力，增加可处理任务数
|
||||
if (info.platformName === 'comfyui' && hasInternalCapacity) {
|
||||
totalCapacity = info.MAX_CONCURRENT + internalCapacity;
|
||||
logger.debug(`[judgConcurrency] comfyui 平台总并发: ${totalCapacity} (外部${info.MAX_CONCURRENT} + 内部${internalCapacity})`);
|
||||
}
|
||||
|
||||
// 判断是否可以处理任务：总并发未满且队列中有任务
|
||||
if (info.PQtasks < totalCapacity && actualQueueLength > 0) {
|
||||
let count = totalCapacity - info.PQtasks;
|
||||
|
||||
// 可处理的任务数不能大于队列实际长度
|
||||
if(count > actualQueueLength) {
|
||||
count = actualQueueLength;
|
||||
|
||||
if (info.platformName === 'comfyui') {
|
||||
if (!hasJwtToken) {
|
||||
if (actualQueueLength > 0) {
|
||||
const count = Math.min(50, actualQueueLength);
|
||||
logger.warn(`[waiting] messageDispatcher 未连接，限制取出 ${count} 个任务丢入 error`);
|
||||
wDeficiency.push({ aigcPfName, info, count });
|
||||
}
|
||||
} else if (hasInternalCapacity) {
|
||||
let totalCapacity = info.MAX_CONCURRENT + internalCapacity;
|
||||
|
||||
if (info.PQtasks < totalCapacity && actualQueueLength > 0) {
|
||||
let count = totalCapacity - info.PQtasks;
|
||||
|
||||
if(count > actualQueueLength) {
|
||||
count = actualQueueLength;
|
||||
}
|
||||
wDeficiency.push({ aigcPfName, info, count });
|
||||
}
|
||||
}
|
||||
wDeficiency.push({ aigcPfName, info, count }); // 储存可进行任务处理的等待队列
|
||||
logger.debug(`平台 ${aigcPfName} 满足处理条件，可处理 ${count} 个任务`);
|
||||
} else {
|
||||
// logger.debug(`平台 ${aigcPfName} 不满足处理条件：PQtasks < MAX_CONCURRENT = ${info.PQtasks < info.MAX_CONCURRENT}, 队列长度 > 0 = ${actualQueueLength > 0}`);
|
||||
if (info.PQtasks < info.MAX_CONCURRENT && actualQueueLength > 0) {
|
||||
let count = info.MAX_CONCURRENT - info.PQtasks;
|
||||
|
||||
if(count > actualQueueLength) {
|
||||
count = actualQueueLength;
|
||||
}
|
||||
wDeficiency.push({ aigcPfName, info, count });
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`检查平台 ${aigcPfName} 队列长度失败:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return wDeficiency; // 返回可处理的队列列表
|
||||
|
||||
return wDeficiency;
|
||||
} catch (error) {
|
||||
logger.error('判断并发数失败:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// 批量获取等待队列任务的任务ID（仅获取，不移除）
|
||||
async function getBatchWaitTasksID(platforms) {
|
||||
try {
|
||||
const multi = redis.multi();
|
||||
// 从等待队列批量获取任务ID，但不立即移除
|
||||
for(const platform of platforms) {
|
||||
multi.lRange(platform.info.waitQueue, 0, platform.count - 1);
|
||||
}
|
||||
|
||||
// 执行所有命令
|
||||
const results = await multi.exec();
|
||||
console.log('批量获取等待队列任务ID结果:', results);
|
||||
|
||||
// 将任务ID与处理队列关联
|
||||
|
||||
for(let i = 0; i < results.length; i++) {
|
||||
const taskIDs = results[i] || [];
|
||||
const platform = platforms[i];
|
||||
platform.waitTaskID = taskIDs;
|
||||
}
|
||||
|
||||
logger.debug('批量获取等待队列任务ID：', platforms);
|
||||
|
||||
return platforms;
|
||||
} catch (error) {
|
||||
logger.error('批量获取等待队列任务ID失败:', error);
|
||||
|
|
@ -182,18 +171,15 @@ async function getBatchWaitTasksID(platforms) {
|
|||
}
|
||||
}
|
||||
|
||||
// 批量获取多个等待队列中的任务数据
|
||||
async function getBatchWaitTasks(aigcPfTasks) {
|
||||
const tasksData = [];
|
||||
|
||||
|
||||
try {
|
||||
// 在处理任务前，初始化分发状态管理器（获取最新的内部算力信息）
|
||||
await dispatchStateManager.init();
|
||||
|
||||
// 收集所有需要获取的任务ID
|
||||
|
||||
const allTaskIds = [];
|
||||
const taskIdMap = new Map(); // 用于映射任务ID到平台信息
|
||||
|
||||
const taskIdMap = new Map();
|
||||
|
||||
for(const aigcPfTask of aigcPfTasks) {
|
||||
for(const taskId of aigcPfTask.waitTaskID) {
|
||||
if (taskId) {
|
||||
|
|
@ -206,49 +192,60 @@ async function getBatchWaitTasks(aigcPfTasks) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (allTaskIds.length === 0) {
|
||||
return tasksData;
|
||||
}
|
||||
|
||||
// 批量获取任务数据
|
||||
|
||||
const multi = redis.multi();
|
||||
for(const taskId of allTaskIds) {
|
||||
multi.hGetAll(`${initQueue.prefix}:task:${taskId}`);
|
||||
}
|
||||
|
||||
|
||||
const results = await multi.exec();
|
||||
|
||||
// 处理结果
|
||||
|
||||
for(let i = 0; i < results.length; i++) {
|
||||
const taskInfo = results[i];
|
||||
const taskId = allTaskIds[i];
|
||||
const platformInfo = taskIdMap.get(taskId);
|
||||
|
||||
|
||||
if (taskInfo) {
|
||||
try {
|
||||
// 使用分发状态管理器获取分发类型（前 N 个用 messageDispatcher，剩余用 runninghub）
|
||||
const dispatchType = dispatchStateManager.getDispatchType(platformInfo.platformName);
|
||||
tasksData.push({
|
||||
backendId: taskInfo.backendId,
|
||||
taskId: taskInfo.taskId, // 单个任务ID
|
||||
platformName: platformInfo.platformName,
|
||||
aigc: platformInfo.aigc,
|
||||
aigcPfName: platformInfo.aigcPfName,
|
||||
taskData: taskInfo.payload,
|
||||
workflowId: taskInfo.workflowId || '',
|
||||
dispatchType: dispatchType,
|
||||
});
|
||||
// logger.debug(`已获取任务 ${taskId} 数据：platform=${platformInfo.platformName}, aigc=${platformInfo.aigc}`);
|
||||
const dispatchType = await dispatchStateManager.getDispatchType(platformInfo.platformName);
|
||||
|
||||
if (dispatchType === 'error_no_jwt') {
|
||||
logger.warn(`[waiting] messageDispatcher 未连接，任务 ${taskId} 标记为待处理`);
|
||||
tasksData.push({
|
||||
backendId: taskInfo.backendId,
|
||||
taskId: taskInfo.taskId,
|
||||
platformName: platformInfo.platformName,
|
||||
aigc: platformInfo.aigc,
|
||||
aigcPfName: platformInfo.aigcPfName,
|
||||
taskData: taskInfo.payload,
|
||||
workflowId: taskInfo.workflowId || '',
|
||||
dispatchType: dispatchType,
|
||||
errorType: 'messageDispatcher 未连接'
|
||||
});
|
||||
} else {
|
||||
tasksData.push({
|
||||
backendId: taskInfo.backendId,
|
||||
taskId: taskInfo.taskId,
|
||||
platformName: platformInfo.platformName,
|
||||
aigc: platformInfo.aigc,
|
||||
aigcPfName: platformInfo.aigcPfName,
|
||||
taskData: taskInfo.payload,
|
||||
workflowId: taskInfo.workflowId || '',
|
||||
dispatchType: dispatchType,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`解析任务${taskId}数据失败:`, error);
|
||||
}
|
||||
} else {
|
||||
logger.warn(`任务 ${taskId} 数据不存在`);
|
||||
logger.error(`任务 ${taskId} 数据不存在`);
|
||||
}
|
||||
}
|
||||
|
||||
// logger.debug('批量获取多个等待队列中的任务数据：', tasksData);
|
||||
|
||||
return tasksData;
|
||||
} catch (error) {
|
||||
logger.error('批量获取任务数据失败:', error);
|
||||
|
|
@ -256,23 +253,19 @@ async function getBatchWaitTasks(aigcPfTasks) {
|
|||
}
|
||||
}
|
||||
|
||||
// 批量移除任务ID、更新等待队列计数并增加处理队列任务数
|
||||
async function updateTaskCounts(wDeficiency) {
|
||||
try {
|
||||
const taskCountMap = new Map();
|
||||
const multi = redis.multi();
|
||||
|
||||
// 1. 准备批量移除任务ID和更新计数
|
||||
|
||||
for(const aigcPfTask of wDeficiency) {
|
||||
const { waitTaskID, info } = aigcPfTask;
|
||||
const key = aigcPfTask.aigcPfName;
|
||||
const count = waitTaskID.length;
|
||||
|
||||
|
||||
if (count > 0) {
|
||||
// 移除已处理的任务ID
|
||||
multi.lTrim(info.waitQueue, count, -1);
|
||||
|
||||
// 更新计数映射
|
||||
|
||||
if(taskCountMap.has(key)) {
|
||||
taskCountMap.set(key, taskCountMap.get(key) + count);
|
||||
} else {
|
||||
|
|
@ -280,72 +273,92 @@ async function updateTaskCounts(wDeficiency) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 执行Redis命令，移除任务ID
|
||||
|
||||
await multi.exec();
|
||||
|
||||
// 3. 更新平台计数
|
||||
|
||||
if (taskCountMap.size > 0) {
|
||||
// 减少平台等待队列的待处理任务数
|
||||
await initQueue.reducePlatformsWait(taskCountMap);
|
||||
// 增加平台处理队列的正在处理任务数
|
||||
await initQueue.addPlatformsProcess(taskCountMap);
|
||||
logger.debug('更新任务计数完成');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('更新任务计数失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// 持续执行批量处理
|
||||
(async () => {
|
||||
while(true) {
|
||||
try {
|
||||
// 判断并发数，获取可进行任务处理的等待队列
|
||||
const wDeficiency = await judgConcurrency();
|
||||
const wDeficiency = await julgConcurrency();
|
||||
|
||||
// 判断是否有可处理的队列
|
||||
if(wDeficiency.length > 0) {
|
||||
// logger.debug('可进行任务处理的等待队列：', wDeficiency);
|
||||
logger.info('有可进行处理的队列，数量: ' + wDeficiency.length);
|
||||
console.log(wDeficiency);
|
||||
|
||||
// 批量获取多个等待队列中的任务ID
|
||||
const tasksWithIds = await getBatchWaitTasksID(wDeficiency);
|
||||
|
||||
// 通过任务ID批量获取多个等待队列中的任务数据
|
||||
const tasksData = await getBatchWaitTasks(tasksWithIds);
|
||||
|
||||
// 更新任务计数 - 无论是否获取到任务数据，都需要更新计数，因为任务ID已经从Redis队列中移除
|
||||
|
||||
const tasksToProcess = [];
|
||||
const errorTasks = [];
|
||||
|
||||
for (const task of tasksData) {
|
||||
if (task.dispatchType === 'error_no_jwt') {
|
||||
errorTasks.push(task);
|
||||
} else if (task.dispatchType === 'messageDispatcher' || task.dispatchType === 'external') {
|
||||
tasksToProcess.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
if (errorTasks.length > 0) {
|
||||
const errorTasksToProcess = errorTasks.slice(0, 50);
|
||||
const remainingTasks = errorTasks.slice(50);
|
||||
|
||||
if (errorTasksToProcess.length > 0) {
|
||||
logger.warn(`[waiting] messageDispatcher 未连接，将 ${errorTasksToProcess.length} 个任务直接丢入 error 队列`);
|
||||
|
||||
for (const task of errorTasksToProcess) {
|
||||
tasksToProcess.push({
|
||||
...task,
|
||||
dispatchType: 'messageDispatcher'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (remainingTasks.length > 0) {
|
||||
const multi = redis.multi();
|
||||
for (const task of remainingTasks) {
|
||||
multi.rPush(task.aigcPfName.replace(/:.*/, `:${task.platformName}:wait`), task.taskId);
|
||||
}
|
||||
await multi.exec();
|
||||
logger.warn(`[waiting] 超过50个任务限制，${remainingTasks.length} 个任务重新放回等待队列`);
|
||||
}
|
||||
}
|
||||
|
||||
if (tasksToProcess.length === 0) {
|
||||
await updateTaskCounts(tasksWithIds);
|
||||
await new Promise(resolve => setTimeout(resolve, 15000));
|
||||
continue;
|
||||
}
|
||||
|
||||
await updateTaskCounts(tasksWithIds);
|
||||
|
||||
// 将任务发送给生成 Worker 处理
|
||||
if (tasksData.length > 0) {
|
||||
logger.info('发送任务给生成Worker处理，数量: ' + tasksData.length);
|
||||
generateWorker.postMessage(tasksData);
|
||||
|
||||
if (tasksToProcess.length > 0) {
|
||||
logger.info(`[waiting] 开始处理 ${tasksToProcess.length} 个任务`);
|
||||
generateWorker.postMessage(tasksToProcess);
|
||||
}
|
||||
} else {
|
||||
// 没有可处理的队列，等待10秒后重试
|
||||
await new Promise(resolve => setTimeout(resolve, 15000));
|
||||
logger.debug('没有可处理的队列');
|
||||
}
|
||||
|
||||
|
||||
} catch (error) {
|
||||
logger.error('批量处理任务失败:', error);
|
||||
// 出错后等待5秒再重试
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
// 监听 generateWorker 的消息
|
||||
generateWorker.on('message', (message) => {
|
||||
if (message.status === 'completed') {
|
||||
logger.debug('等待任务处理完成');
|
||||
logger.info('[waiting] 任务批次处理完成');
|
||||
}
|
||||
});
|
||||
|
||||
generateWorker.on('error', (error) => {
|
||||
logger.error('生成 Worker 错误:', error);
|
||||
// 可以考虑重启Worker
|
||||
});
|
||||
});
|
||||
|
|
@ -1,639 +0,0 @@
|
|||
# 任务队列后端 - 业务流程分析报告
|
||||
|
||||
## 一、项目概述
|
||||
|
||||
**项目名称**: ComfyUI 任务队列后端
|
||||
|
||||
**核心功能**: 一个分布式任务处理系统，用于管理和分发 AI 图像生成任务到多个外部平台（如 comfyui、coze 等），支持任务队列、并发控制、状态跟踪和结果返回。
|
||||
|
||||
**技术栈**:
|
||||
|
||||
- Node.js + Express
|
||||
- Redis（任务队列和数据存储）
|
||||
- WebSocket（实时通信）
|
||||
- Worker Threads（多线程处理）
|
||||
|
||||
***
|
||||
|
||||
## 二、系统架构
|
||||
|
||||
### 2.1 核心组件
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 前端 / 后端客户端 │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│ WebSocket (8087)
|
||||
│ HTTP (8089 - 回调)
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ index.js (主入口) │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ WebSocket │ │ HTTP Server │ │ Worker │ │
|
||||
│ │ Server │ │ │ │ Manager │ │
|
||||
│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │
|
||||
└─────────┼───────────────────┼───────────────────┼─────────────────┘
|
||||
│ │ │
|
||||
└───────────────────┴───────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Redis │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ 等待队列 │ │ 处理队列 │ │ 结果队列 │ │
|
||||
│ │ (Wait) │ │ (Process) │ │ (Result) │ │
|
||||
│ └──────────────┘ └──────────────┘ └──────────────┘ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ 错误队列 │ │ 任务数据 │ │ 初始化配置 │ │
|
||||
│ │ (Error) │ │ (Hash) │ │ (InitInfo) │ │
|
||||
│ └──────────────┘ └──────────────┘ └──────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.2 Worker Threads 架构
|
||||
|
||||
系统使用 6 个独立的 Worker Threads 处理不同阶段的任务：
|
||||
|
||||
| Worker 名称 | 功能描述 | 文件路径 |
|
||||
| ---------------- | ------------- | ------------------------------------------ |
|
||||
| assessment | 任务预处理、参数校验、入队 | `worker_threads/assessment/assessment.js` |
|
||||
| wait | 等待队列监控、任务分发 | `worker_threads/wait/waiting.js` |
|
||||
| polling | 轮询外部平台获取任务结果 | `worker_threads/process/process.js` |
|
||||
| result | 结果队列处理、返回给客户端 | `worker_threads/result/result.js` |
|
||||
| callback\_result | 回调结果处理 | `worker_threads/callback_result/result.js` |
|
||||
| error | 错误队列处理 | `worker_threads/error/error.js` |
|
||||
|
||||
***
|
||||
|
||||
## 三、完整业务流程
|
||||
|
||||
### 3.1 流程总览图
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Start([系统启动]) --> Init[初始化 Redis 队列]
|
||||
Init --> WS[WebSocket 服务启动]
|
||||
Init --> HTTP[HTTP 服务启动]
|
||||
WS --> ReceiveTask[接收任务: type=generate]
|
||||
|
||||
ReceiveTask --> Assessment[Assessment Worker 处理]
|
||||
Assessment --> Validate{参数校验}
|
||||
Validate -->|失败| ErrorReturn[返回错误给客户端]
|
||||
Validate -->|成功| StoreTask[存储任务到 Redis Hash]
|
||||
StoreTask --> PushWait[推入等待队列]
|
||||
PushWait --> NotifySuccess[通知客户端: 任务提交成功]
|
||||
|
||||
PushWait --> WaitWorker[Wait Worker 监控]
|
||||
WaitWorker --> CheckConcurrency{检查并发数}
|
||||
CheckConcurrency -->|并发未满| GetTasks[批量获取任务]
|
||||
GetTasks --> GenerateWorker[Generate Worker 分发]
|
||||
GenerateWorker --> ExternalPost[提交任务到外部平台]
|
||||
|
||||
ExternalPost --> PostResult{提交结果}
|
||||
PostResult -->|失败| PushError[推入错误队列]
|
||||
PostResult -->|成功| CheckPlatform{平台类型}
|
||||
|
||||
CheckPlatform -->|回调型| StoreCallback[存储 remoteTaskId 映射]
|
||||
CheckPlatform -->|轮询型| PushPolling[推入轮询队列]
|
||||
|
||||
StoreCallback --> WaitCallback[等待外部回调]
|
||||
PushPolling --> PollingWorker[Polling Worker 轮询]
|
||||
|
||||
WaitCallback --> CallbackReceived[收到回调]
|
||||
PollingWorker --> PollingResult{轮询结果}
|
||||
|
||||
CallbackReceived --> ProcessCallback[处理回调数据]
|
||||
PollingResult -->|完成| StoreResult[存储结果]
|
||||
PollingResult -->|未完成| ContinuePolling[继续轮询]
|
||||
|
||||
ProcessCallback --> StoreResult
|
||||
StoreResult --> PushResultQueue[推入结果队列]
|
||||
|
||||
PushResultQueue --> ResultWorker[Result Worker 处理]
|
||||
ResultWorker --> SendResult[发送结果给客户端]
|
||||
|
||||
PushError --> ErrorWorker[Error Worker 处理]
|
||||
ErrorWorker --> SendError[发送错误给客户端]
|
||||
|
||||
ContinuePolling --> PollingWorker
|
||||
```
|
||||
|
||||
### 3.2 详细流程步骤
|
||||
|
||||
#### 阶段 1: 系统启动与初始化
|
||||
|
||||
**步骤 1.1: 启动主程序**
|
||||
|
||||
- 入口文件: [index.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\index.js)
|
||||
- 加载环境变量 (.env)
|
||||
- 创建 6 个 Worker Threads
|
||||
|
||||
**步骤 1.2: Redis 初始化**
|
||||
|
||||
- 连接 Redis
|
||||
- 初始化队列配置 ([initQueue.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\redis\initQueue.js))
|
||||
- 创建 `InitInfo` 配置对象，包含：
|
||||
- 等待队列列表
|
||||
- 各平台并发配置
|
||||
- 任务计数器
|
||||
|
||||
**步骤 1.3: 启动服务**
|
||||
|
||||
- HTTP 服务器 (端口: 8087)
|
||||
- WebSocket 服务器 (同端口)
|
||||
- 回调服务器 (端口: 8089, [app.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\app.js))
|
||||
|
||||
***
|
||||
|
||||
#### 阶段 2: 任务接收与预处理
|
||||
|
||||
**步骤 2.1: 接收 WebSocket 任务**
|
||||
|
||||
- 客户端通过 WebSocket 连接，携带 token 和 id
|
||||
- 验证 token (TOKEN\_SECRET)
|
||||
- 接收 `type: "generate"` 消息
|
||||
|
||||
**步骤 2.2: Assessment Worker 处理**
|
||||
|
||||
- 文件: [assessment.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\assessment\assessment.js)
|
||||
- 内部线程池: 3 个 [PreproTask.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\assessment\PreproTask.js)
|
||||
|
||||
**步骤 2.3: 参数校验**
|
||||
|
||||
- 校验必填字段: `taskId`, `platform`, `payload`
|
||||
- 失败返回: `JSONError`, `OpcodeError`
|
||||
|
||||
**步骤 2.4: 存储任务**
|
||||
|
||||
- 创建任务 Hash: `{prefix}:task:{taskId}`
|
||||
- 字段: `taskId`, `payload`, `backendId`, `AIGC`, `platform`, `status`, `workflowId`
|
||||
- 设置过期时间: 2 小时 (7200秒)
|
||||
|
||||
**步骤 2.5: 推入等待队列**
|
||||
|
||||
- 队列命名: `{AIGC}:{platform}:wait`
|
||||
- 例如: `digitalHuman-test:comfyui:wait`
|
||||
- 更新 `InitInfo.platforms.{key}.WQtasks` +1
|
||||
|
||||
**步骤 2.6: 通知客户端**
|
||||
|
||||
- 返回成功消息: `"任务提交成功，正在排队中..."`
|
||||
|
||||
***
|
||||
|
||||
#### 阶段 3: 等待队列监控与任务分发
|
||||
|
||||
**步骤 3.1: Wait Worker 监控**
|
||||
|
||||
- 文件: [waiting.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\wait\waiting.js)
|
||||
- 循环检查各平台等待队列 (间隔: 15秒)
|
||||
|
||||
**步骤 3.2: 判断并发容量**
|
||||
|
||||
- 获取 `InitInfo.platforms.{key}.PQtasks` (正在处理数)
|
||||
- 获取 `InitInfo.platforms.{key}.MAX_CONCURRENT` (最大并发数)
|
||||
- 对于 comfyui 平台，还会检查内部算力 (messageDispatcher)
|
||||
|
||||
**步骤 3.3: 批量获取任务**
|
||||
|
||||
- 从等待队列获取任务 ID (不超过剩余并发数)
|
||||
- 批量获取任务数据 (Hash)
|
||||
|
||||
**步骤 3.4: 更新计数器**
|
||||
|
||||
- `WQtasks` -N (减少等待数)
|
||||
- `PQtasks` +N (增加处理数)
|
||||
- `PQtasksALL` +N (总处理数)
|
||||
|
||||
**步骤 3.5: 分发到 Generate Worker**
|
||||
|
||||
- 文件: [GenerateWorkerManager.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\wait\GenerateWorkerManager.js)
|
||||
- 内部线程池: [GenerateThreadPool.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\wait\GenerateThreadPool.js)
|
||||
|
||||
***
|
||||
|
||||
#### 阶段 4: 提交任务到外部平台
|
||||
|
||||
**步骤 4.1: Generate Worker 处理**
|
||||
|
||||
- 文件: [generatTask.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\wait\generatTask.js)
|
||||
- 批量调用 `externalPostRequest()`
|
||||
|
||||
**步骤 4.2: 外部平台接口调用**
|
||||
|
||||
- 文件: [generat.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\outside\generat.js)
|
||||
- 平台适配器: [outside.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\outside\outPlatforms\outside.js)
|
||||
|
||||
**支持的平台**:
|
||||
|
||||
| 平台 | 类型 | 适配器文件 |
|
||||
| ------- | --- | -------------------------------------------------------------------------------------------------------------- |
|
||||
| comfyui | 回调型 | [comfyui.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\outside\outPlatforms\comfyui.js) |
|
||||
| coze | 轮询型 | [coze/coze.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\outside\outPlatforms\coze\coze.js) |
|
||||
| jimuai | - | [JimuAI.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\outside\outPlatforms\JimuAI.js) |
|
||||
|
||||
**comfyui 分发策略**:
|
||||
|
||||
- 优先使用 `messageDispatcher` (内部算力)
|
||||
- 失败降级到 `runninghub` (外部平台)
|
||||
|
||||
**步骤 4.3: 处理提交结果**
|
||||
|
||||
**情况 A: 提交失败**
|
||||
|
||||
- 存储错误信息到任务 Hash
|
||||
- 推入错误队列: `{prefix}:error:list`
|
||||
- `EQtaskALL` +1
|
||||
|
||||
**情况 B: 提交成功 (回调型平台)**
|
||||
|
||||
- 存储映射: `{prefix}:callback:{remoteTaskId}` → `taskId`
|
||||
- 更新任务 Hash: `remoteTaskId`
|
||||
|
||||
**情况 C: 提交成功 (轮询型平台)**
|
||||
|
||||
- 推入轮询队列: `{prefix}:processPolling:{AIGC}:{platform}`
|
||||
- Hash 结构: `{remoteTaskId}` → `{taskId, platform, AIGC, workflowId}`
|
||||
- 更新任务 Hash: `remoteTaskId`
|
||||
|
||||
***
|
||||
|
||||
#### 阶段 5: 任务结果获取
|
||||
|
||||
**分支 A: 回调型平台 (comfyui)**
|
||||
|
||||
**步骤 5A.1: 接收回调**
|
||||
|
||||
- 回调接口: `POST /callback/all` ([callback.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\outside\callback.js))
|
||||
- 立即返回 200 响应
|
||||
|
||||
**步骤 5A.2: 处理回调数据**
|
||||
|
||||
- 通过 `remoteTaskId` 查询 `taskId`
|
||||
- 存储 `eventData` 到任务 Hash 的 `resultData`
|
||||
|
||||
* 推入回调队列: `{prefix}:callback`
|
||||
* `CQtasksALL` +1
|
||||
|
||||
**步骤 5A.3: Callback Result Worker 处理**
|
||||
|
||||
- 文件: [callback\_result/result.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\callback_result\result.js)
|
||||
- 发送结果给客户端
|
||||
|
||||
***
|
||||
|
||||
**分支 B: 轮询型平台 (coze)**
|
||||
|
||||
**步骤 5B.1: Polling Worker 监控**
|
||||
|
||||
- 文件: [process.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\process\process.js)
|
||||
- 为每个平台启动独立轮询循环
|
||||
- 动态轮询间隔: 5-30秒 (根据任务数调整)
|
||||
|
||||
**步骤 5B.2: 查询外部平台**
|
||||
|
||||
- 文件: [polling.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\outside\polling.js)
|
||||
- 调用 `externalGetRequest()`
|
||||
- 批量查询 (每批最多 100 个任务)
|
||||
|
||||
**步骤 5B.3: 判断任务状态**
|
||||
|
||||
**情况 1: 任务未完成**
|
||||
|
||||
- 继续轮询 (不做任何操作)
|
||||
|
||||
**情况 2: 任务成功**
|
||||
|
||||
- 存储结果到任务 Hash 的 `resultData`
|
||||
- 更新状态: `status = "success"`
|
||||
- 推入结果队列: `{prefix}:result:list`
|
||||
- 从轮询队列删除
|
||||
- `PQtasks` -1, `RQtasksALL` +1
|
||||
|
||||
**情况 3: 任务失败**
|
||||
|
||||
- 存储错误信息到 `resultData`
|
||||
- 更新状态: `status = "failed"`
|
||||
- 推入错误队列: `{prefix}:error:list`
|
||||
- 从轮询队列删除
|
||||
- `PQtasks` -1, `EQtaskALL` +1
|
||||
|
||||
***
|
||||
|
||||
#### 阶段 6: 结果返回给客户端
|
||||
|
||||
**分支 A: 结果队列处理**
|
||||
|
||||
- 文件: [result/result.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\result\result.js)
|
||||
- 从结果队列获取任务
|
||||
- 通过 WebSocket 发送给对应 `backendId` 的客户端
|
||||
- 从结果队列删除
|
||||
- `RQtasksALL` -1
|
||||
|
||||
**分支 B: 错误队列处理**
|
||||
|
||||
- 文件: [error/error.js](file:///d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge\任务队列后端\worker_threads\error\error.js)
|
||||
- 从错误队列获取任务
|
||||
- 通过 WebSocket 发送错误给客户端
|
||||
- 从错误队列和任务 Hash 删除
|
||||
- `EQtaskALL` -1, `WQtasks` -1
|
||||
|
||||
***
|
||||
|
||||
## 四、任务状态转换图
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> pending: 任务创建
|
||||
pending --> processing: 从等待队列取出
|
||||
processing --> success: 任务成功完成
|
||||
processing --> failed: 任务失败
|
||||
success --> [*]: 结果已发送
|
||||
failed --> [*]: 错误已发送
|
||||
|
||||
note right of pending
|
||||
存储在等待队列
|
||||
WQtasks +1
|
||||
end note
|
||||
|
||||
note right of processing
|
||||
存储在处理队列/轮询队列
|
||||
PQtasks +1
|
||||
end note
|
||||
|
||||
note right of success
|
||||
存储在结果队列
|
||||
RQtasksALL +1
|
||||
end note
|
||||
|
||||
note right of failed
|
||||
存储在错误队列
|
||||
EQtaskALL +1
|
||||
end note
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## 五、Redis 数据结构
|
||||
|
||||
### 5.1 队列列表
|
||||
|
||||
| Key 模式 | 类型 | 描述 |
|
||||
| ------------------------------------------- | ---- | ------------------------ |
|
||||
| `{prefix}:{AIGC}:{platform}:wait` | List | 等待队列，存储 taskId |
|
||||
| `{prefix}:process:Polling` | - | (已弃用，见下方) |
|
||||
| `{prefix}:processPolling:{AIGC}:{platform}` | Hash | 轮询队列，remoteTaskId → 任务信息 |
|
||||
| `{prefix}:result:queue` | - | (已弃用) |
|
||||
| `{prefix}:result:list` | List | 结果队列，存储 taskId |
|
||||
| `{prefix}:callback` | List | 回调队列，存储 taskId |
|
||||
| `{prefix}:error:queue` | - | (已弃用) |
|
||||
| `{prefix}:error:list` | List | 错误队列，存储 taskId |
|
||||
|
||||
### 5.2 任务数据 (Hash)
|
||||
|
||||
| Key | 类型 | 字段 |
|
||||
| ------------------------ | ------ | ---------------------------------------------------------------------------------------------------------- |
|
||||
| `{prefix}:task:{taskId}` | Hash | `taskId`, `payload`, `backendId`, `AIGC`, `platform`, `status`, `resultData`, `remoteTaskId`, `workflowId` |
|
||||
| <br /> | <br /> | **过期时间**: 7200秒 (2小时) |
|
||||
|
||||
### 5.3 映射关系
|
||||
|
||||
| Key | 类型 | 描述 |
|
||||
| --------------------------------------------------- | ------ | --------------------- |
|
||||
| `{prefix}:callback:{remoteTaskId}` | String | remoteTaskId → taskId |
|
||||
| `{prefix}:pending:messages` | List | 待发送消息键列表 |
|
||||
| `{prefix}:pending:messages:{backendId}:{timestamp}` | Hash | 待发送消息数据 |
|
||||
|
||||
### 5.4 初始化配置 (JSON)
|
||||
|
||||
| Key | 路径 | 描述 |
|
||||
| ------------------- | ---------------------------------- | -------- |
|
||||
| `{prefix}:InitInfo` | `$.waitQueues` | 等待队列名称数组 |
|
||||
| <br /> | `$.processPolling` | 轮询队列名称 |
|
||||
| <br /> | `$.processCallback` | 回调队列名称 |
|
||||
| <br /> | `$.resultName` | 结果队列名称 |
|
||||
| <br /> | `$.PQtasksALL` | 总处理任务数 |
|
||||
| <br /> | `$.RQtasksALL` | 总结果任务数 |
|
||||
| <br /> | `$.CQtasksALL` | 总回调任务数 |
|
||||
| <br /> | `$.EQtaskALL` | 总错误任务数 |
|
||||
| <br /> | `$.platforms.{key}.WQtasks` | 平台等待任务数 |
|
||||
| <br /> | `$.platforms.{key}.PQtasks` | 平台处理任务数 |
|
||||
| <br /> | `$.platforms.{key}.MAX_CONCURRENT` | 平台最大并发数 |
|
||||
| <br /> | `$.platforms.{key}.waitQueue` | 平台等待队列名 |
|
||||
|
||||
***
|
||||
|
||||
## 六、任务唯一标识与类型
|
||||
|
||||
### 6.1 任务唯一标识
|
||||
|
||||
| 标识 | 生成位置 | 用途 |
|
||||
| -------------- | ------ | -------- |
|
||||
| `taskId` | 前端/客户端 | 内部任务唯一标识 |
|
||||
| `remoteTaskId` | 外部平台 | 外部平台任务标识 |
|
||||
| `backendId` | 客户端连接时 | 客户端连接标识 |
|
||||
|
||||
### 6.2 任务类型
|
||||
|
||||
| 类型 | 说明 | 处理方式 |
|
||||
| -------------- | ------- | ---------- |
|
||||
| 回调型 (callback) | comfyui | 等待外部回调通知 |
|
||||
| 轮询型 (polling) | coze | 主动轮询外部平台状态 |
|
||||
|
||||
***
|
||||
|
||||
## 七、关键配置文件
|
||||
|
||||
### 7.1 model.json
|
||||
|
||||
```json
|
||||
{
|
||||
"digitalHuman-test": {
|
||||
"comfyui": {
|
||||
"apikey": "...",
|
||||
"concurrency": 13
|
||||
},
|
||||
"coze": {
|
||||
"apikey": "...",
|
||||
"concurrency": 20
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 7.2 Platform.json
|
||||
|
||||
```json
|
||||
{
|
||||
"callback": ["comfyui"],
|
||||
"polling": ["coze"]
|
||||
}
|
||||
```
|
||||
|
||||
### 7.3 .env 环境变量
|
||||
|
||||
```env
|
||||
PROJECT_PREFIX='digitalHuman-test'
|
||||
TOKEN_SECRET='...'
|
||||
WS_PORT=8087
|
||||
CALLBACK_PORT=8089
|
||||
RunningHub_URL='...'
|
||||
CALLBACK_URL='...'
|
||||
REDIS_URL='...'
|
||||
MESSAGE_DISPATCHER_URL='...'
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## 八、错误码与消息
|
||||
|
||||
### 8.1 code.json
|
||||
|
||||
```json
|
||||
{
|
||||
"ERROR": {
|
||||
"JSONError": "消息格式错误，请联系服务商。",
|
||||
"OpcodeError": "错误提交，请稍后再试。",
|
||||
"BalanceError": "余额不足，请充值后继续使用。",
|
||||
"AssessmentError": "任务提交失败，请稍后再试。"
|
||||
},
|
||||
"SUCCESS": {
|
||||
"AssessmentSuccess": "任务提交成功，正在排队中..."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## 九、消息持久化机制
|
||||
|
||||
当客户端断开连接时，待发送消息会被保存到 Redis，待客户端重连后重试发送：
|
||||
|
||||
1. **保存待发送消息**: `messagePersistence.savePendingMessage()`
|
||||
2. **客户端重连时获取**: `messagePersistence.getPendingMessages()`
|
||||
3. **发送成功后删除**: `messagePersistence.removePendingMessage()`
|
||||
4. **定期清理过期消息**: 超过 2 天的消息自动清理
|
||||
|
||||
***
|
||||
|
||||
## 十、流程图
|
||||
|
||||
### 10.1 完整数据流时序图
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client as 客户端
|
||||
participant WS as WebSocket Server
|
||||
participant A as Assessment Worker
|
||||
participant Redis as Redis
|
||||
participant W as Wait Worker
|
||||
participant G as Generate Worker
|
||||
participant Ext as 外部平台
|
||||
participant P as Polling Worker
|
||||
participant R as Result Worker
|
||||
|
||||
Client->>WS: WebSocket 连接 (token, id)
|
||||
WS-->>Client: 连接成功
|
||||
|
||||
Client->>WS: {type: "generate", taskId, platform, payload}
|
||||
WS->>A: 转发任务
|
||||
A->>A: 参数校验
|
||||
alt 校验失败
|
||||
A-->>WS: 返回错误
|
||||
WS-->>Client: 错误消息
|
||||
else 校验成功
|
||||
A->>Redis: HSET {prefix}:task:{taskId}
|
||||
A->>Redis: RPUSH {AIGC}:{platform}:wait
|
||||
A->>Redis: INCR WQtasks
|
||||
A-->>WS: 成功
|
||||
WS-->>Client: "任务提交成功，正在排队中..."
|
||||
end
|
||||
|
||||
loop 每15秒检查
|
||||
W->>Redis: GET InitInfo.platforms
|
||||
W->>Redis: LLEN wait queue
|
||||
alt 有可处理任务
|
||||
W->>Redis: LRANGE wait queue (批量获取)
|
||||
W->>Redis: HGETALL task data
|
||||
W->>Redis: LTRIM wait queue
|
||||
W->>Redis: DECR WQtasks, INCR PQtasks
|
||||
W->>G: 发送任务数据
|
||||
G->>Ext: POST /task/create
|
||||
Ext-->>G: {remoteTaskId}
|
||||
|
||||
alt 提交失败
|
||||
G->>Redis: HSET resultData (error)
|
||||
G->>Redis: LPUSH error:list
|
||||
G->>Redis: INCR EQtaskALL
|
||||
else 回调型平台
|
||||
G->>Redis: SET callback:{remoteTaskId} = taskId
|
||||
G->>Redis: HSET remoteTaskId
|
||||
else 轮询型平台
|
||||
G->>Redis: HSET processPolling:{remoteTaskId}
|
||||
G->>Redis: HSET remoteTaskId
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
alt 回调型平台
|
||||
Ext->>WS: POST /callback/all
|
||||
WS->>Redis: GET callback:{remoteTaskId}
|
||||
WS->>Redis: HSET resultData
|
||||
WS->>Redis: LPUSH callback
|
||||
WS->>Redis: INCR CQtasksALL
|
||||
else 轮询型平台
|
||||
loop 每5-30秒轮询
|
||||
P->>Redis: HGETALL processPolling
|
||||
P->>Ext: GET /task/status
|
||||
alt 任务未完成
|
||||
P->>P: 继续轮询
|
||||
else 任务成功
|
||||
P->>Redis: HSET resultData
|
||||
P->>Redis: LPUSH result:list
|
||||
P->>Redis: HDEL processPolling
|
||||
P->>Redis: DECR PQtasks, INCR RQtasksALL
|
||||
else 任务失败
|
||||
P->>Redis: HSET resultData (error)
|
||||
P->>Redis: LPUSH error:list
|
||||
P->>Redis: HDEL processPolling
|
||||
P->>Redis: DECR PQtasks, INCR EQtaskALL
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
loop 每15秒检查
|
||||
R->>Redis: LLEN result:list / error:list
|
||||
alt 有结果
|
||||
R->>Redis: LRANGE result:list
|
||||
R->>Redis: HGETALL task data
|
||||
R->>WS: 发送结果
|
||||
WS->>Client: {taskId, result}
|
||||
R->>Redis: LREM result:list
|
||||
R->>Redis: DECR RQtasksALL
|
||||
else 有错误
|
||||
R->>Redis: LRANGE error:list
|
||||
R->>Redis: HGETALL task data
|
||||
R->>WS: 发送错误
|
||||
WS->>Client: {taskId, error}
|
||||
R->>Redis: LREM error:list
|
||||
R->>Redis: DEL task:{taskId}
|
||||
R->>Redis: DECR EQtaskALL, DECR WQtasks
|
||||
end
|
||||
end
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## 十一、总结
|
||||
|
||||
该任务队列后端系统是一个设计完善的分布式任务处理系统，具有以下特点：
|
||||
|
||||
1. **多线程架构**: 使用 Worker Threads 实现各阶段解耦，提高并发处理能力
|
||||
2. **Redis 作为核心**: 利用 Redis 的 List、Hash、JSON 等数据结构实现任务队列和状态管理
|
||||
3. **支持多平台**: 可灵活接入不同类型的外部平台（回调型、轮询型）
|
||||
4. **任务状态追踪**: 完整的任务生命周期管理，从接收、处理到完成/失败
|
||||
5. **消息持久化**: 支持客户端断线重连后的消息补发
|
||||
6. **并发控制**: 各平台独立的并发数配置，防止过载
|
||||
7. **优雅降级**: comfyui 平台支持内部算力和外部平台的自动切换
|
||||
|
||||
系统的核心流程清晰，各组件职责明确，是一个生产级别的任务队列管理系统。
|
||||
Loading…
Reference in New Issue