diff --git a/AI实现指导提示词.md b/AI实现指导提示词.md new file mode 100644 index 0000000..599716d --- /dev/null +++ b/AI实现指导提示词.md @@ -0,0 +1,1083 @@ +# AI 实现指导提示词 + +## 任务概述 + +你需要实现将任务队列后端的 runninghub 任务优先分发到内部 message-dispatcher 系统的功能。请按照以下详细指导进行修改。 + +--- + +## 一、任务队列后端项目修改指导 + +### 项目路径 +`d:\Ke_xue_web\独立项目\comfyui桥接器\任务队列后端\` + +--- + +### 修改点 0：新增 WebSocket 通信模块 + +**文件：** `utils/mdWebSocketClient.js`（新建） + +**通信架构说明：** +- message-dispatcher 主动通过 WebSocket 连接任务队列后端 +- WebSocket 用于： + - JWT Token 接收与定期更新 + - 算力状态上报 + - 实例状态变化同步 + - 健康检查 +- **仅任务提交通过 HTTP 接口完成** +- 任务提交接口 URL 通过环境变量获取，与 runninghub 保持一致 +- **所有其他通信（包括 Token 获取）均通过 WebSocket** + +**功能要求：** +1. 作为 WebSocket 服务端，等待 message-dispatcher 连接 +2. 接收 JWT Token 消息（JWT_UPDATE） +3. 接收算力状态更新消息（CAPACITY_UPDATE） +4. 接收实例状态变化消息（INSTANCE_ONLINE/INSTANCE_OFFLINE 等） +5. 发送心跳响应 +6. 提供 Token、算力状态查询接口 + +**核心方法：** +```javascript +class MDWebSocketServer { + constructor() { + this.wss = null; + this.connectedClients = new Map(); + this.currentJwtToken = null; + this.currentCapacity = { internal: 0, external: 0 }; + this.instances = new Map(); + } + + // 初始化并启动 WebSocket 服务 + async init() + + // 获取当前 JWT Token + getJwtToken() + + // 获取当前内部算力可用数 + getInternalCapacity() + + // 获取当前外部容量 + getExternalCapacity() + + // 获取所有实例状态 + getInstances() + + // 检查是否有连接的客户端 + hasConnectedClients() +} +``` + +**WebSocket 消息处理：** +```javascript +// JWT Token 更新 +handleJwtUpdate(data) { + this.currentJwtToken = data.token; + console.log('[MDWebSocketServer] JWT Token 已更新'); +} + +// 算力状态更新 +handleCapacityUpdate(data) { + this.currentCapacity.internal = data.summary.onlineInstances - data.summary.busyInstances; +} + +// 实例上线 +handleInstanceOnline(data) { + this.instances.set(data.instanceId, { ...data, status: 'online' }); +} + +// 实例下线 +handleInstanceOffline(data) { + this.instances.set(data.instanceId, { ...data, status: 'offline' }); +} + +// 心跳响应 +handleHeartbeat(data, ws) { + ws.send(JSON.stringify({ + type: 'HEARTBEAT_ACK', + data: { timestamp: new Date().toISOString() } + })); +} +``` + +**WebSocket 消息格式：** +```javascript +// message-dispatcher 发送的 JWT 更新消息 +{ + type: 'JWT_UPDATE', + data: { + token: 'eyJhbGciOiJIUzI1NiIs...', + expiresAt: '2024-01-02T00:00:00.000Z', + timestamp: '2024-01-01T00:00:00.000Z' + } +} + +// message-dispatcher 发送的心跳 +{ + type: 'HEARTBEAT', + data: { + timestamp: '2024-01-01T00:00:00.000Z' + } +} +``` + +--- + +### 修改点 1：新增 messageDispatcher 平台适配器 + +**文件：** `outside/outPlatforms/messageDispatcher.js`（新建） + +**核心原则：** +- 与 runninghub.js 保持完全一致的接口方法签名 +- 仅修改请求地址和 apiKey 字段的值 +- 请求体所有字段名称、类型、格式与 runninghub 完全一致 +- 任务提交通过 HTTP 接口，URL 通过环境变量获取 + +**要求：** +- 参考 `runninghub.js` 的接口设计，保持方法签名完全一致 +- 实现以下 6 个核心方法： + +```javascript +getGenerateUrl() // 返回 message-dispatcher 的任务提交接口（从环境变量获取） +getGenerateHeader(apikey) // 返回请求头（包含 JWT Token 作为 apiKey） +getGenerateBody(task) // 构造请求体（与 runninghub 完全一致） +getSuccessTasks(response) // 处理成功响应，转换为 runninghub 兼容格式 +getTaskResult(response) // 处理结果回调 +getQueryUrl() // 返回回调地址（与 runninghub 保持一致） +``` + +**请求体标准化规则：** + +| 字段名称 | 类型 | 是否修改 | 说明 | +|---------|------|---------|------| +| `workflow_id` | String | 否 | 原样保留 | +| `node_info_list` | Array | 否 | 原样保留 | +| `apiKey` | String | 是 | 值改为 JWT Token | +| `webhookUrl` | String | 否 | 原样保留 | +| 其他所有字段 | - | 否 | 原样保留 | + +**请求体示例对比：** + +```javascript +// runninghub 请求体（原样） +{ + "workflow_id": "123", + "node_info_list": [...], + "apiKey": "runninghub-api-key-xxx", + "webhookUrl": "http://callback-url" +} + +// message-dispatcher 请求体（仅修改 apiKey 值） +{ + "workflow_id": "123", + "node_info_list": [...], + "apiKey": "eyJhbGciOiJIUzI1NiIs...", // JWT Token + "webhookUrl": "http://callback-url" +} +``` + +**响应转换规则：** +```javascript +// message-dispatcher 响应 +{ success: true, data: { requestId: "xxx" } } + +// 转换为 +{ msg: "success", code: 0, data: { taskId: "xxx" } } +``` + +--- + +### 修改点 2：简化 JWT Token 获取（不再需要独立模块） + +**重要说明：** +- 不再需要独立的 JWTManager 模块 +- JWT Token 由 message-dispatcher 通过 WebSocket 主动推送 +- Token 定期更新也通过 WebSocket 推送 +- 从 MDWebSocketServer 获取 Token 即可 + +**Token 获取方式：** +```javascript +// 从 WebSocket 服务端获取当前 Token +const jwtToken = mdWebSocketServer.getJwtToken(); +``` + +--- + +### 修改点 3：新增任务分流模块 + +**文件：** `utils/taskDistributor.js`（新建） + +**功能要求：** +1. 移除高低优先级区分 +2. 实现统一的任务分流逻辑 +3. 从 WebSocket 服务端获取实时容量信息 +4. 根据容量进行任务分配 + +**核心计算公式：** +``` +内部算力可用数 = 从 MDWebSocketClient 获取 +外部容量上限 = 从配置/环境变量获取（如 10） +总可分发任务上限 = 内部算力可用数 + 外部容量上限 +``` + +**分流策略：** + +| 待分发任务数 | 内部容量 | 外部容量 | 分配结果 | +|------------|---------|---------|---------| +| ≤ 内部 | 实时 | 10 | 全部走内部 | +| > 内部 ≤ 总 | 实时 | 10 | 前N内部，超出部分走外部 | +| > 总 | 实时 | 10 | 内部 + 外部，剩余等待 | + +**具体示例：** +``` +示例 1： +内部 = 30（实时）, 外部 = 10, 待分发 = 25 +结果：全部 25 个走内部 + +示例 2： +内部 = 30（实时）, 外部 = 10, 待分发 = 35 +结果：30 个内部，5 个外部 + +示例 3： +内部 = 30（实时）, 外部 = 10, 待分发 = 45 +结果：30 个内部，10 个外部，5 个等待 +``` + +**核心实现：** +```javascript +async function distributeTasks(tasks, mdWebSocketServer) { + const internalCapacity = mdWebSocketServer.getInternalCapacity(); + const externalCapacity = await getExternalCapacityFromConfig(); + + const internalTasks = tasks.slice(0, internalCapacity); + const externalTasks = tasks.slice(internalCapacity, internalCapacity + externalCapacity); + const remainingTasks = tasks.slice(internalCapacity + externalCapacity); + + return { internalTasks, externalTasks, remainingTasks }; +} +``` + +--- + +### 修改点 4：修改任务分发逻辑（批量） + +**文件：** `worker_threads/wait/waiting.js` + +**修改位置：** 任务批量获取和分发逻辑 + +**要求：** +- 引入 MDWebSocketServer +- 在获取任务后，先调用 taskDistributor 进行分流（传入 WebSocket 服务端） +- 根据分流结果分别分发到内部/外部 +- 剩余任务返回队列 + +--- + +### 修改点 5：修改任务分发逻辑（单个） + +**文件：** `outside/generat.js` + +**修改位置：** `externalPostRequest()` 函数 + +**决策逻辑：** +```javascript +if (platform === 'runninghub') { + if (使用内部算力) { + 尝试使用 messageDispatcher 平台发送任务 + if (成功) { + 返回内部结果 + } else { + 记录降级日志 + 降级使用 runninghub + } + } else { + 使用 runninghub + } +} else { + 使用原平台 +} +``` + +**关键实现：** +- 引入 MDWebSocketServer（从这里获取 JWT Token） +- 引入 messageDispatcher 平台适配器 +- 增加降级日志记录 +- 保持原有错误处理逻辑不变 + +--- + +### 修改点 6：更新平台管理 + +**文件：** `outside/outPlatforms/outside.js` + +**修改内容：** +- 导入 messageDispatcher 模块 +- 将其添加到导出对象中 + +```javascript +import * as runninghub from './runninghub.js'; +import * as jimuai from './JimuAI.js'; +import coze from './coze/coze.js'; +import * as messageDispatcher from './messageDispatcher.js'; // 新增 + +export default { runninghub, jimuai, coze, messageDispatcher }; // 新增 +``` + +--- + +### 修改点 7：更新环境变量配置 + +**文件：** `.env` + +**新增配置：** +```env +# Message Dispatcher 配置 +MESSAGE_DISPATCHER_URL=http://localhost:4000 +MESSAGE_DISPATCHER_WS_PORT=8087 +MESSAGE_DISPATCHER_ENABLED=true +MESSAGE_DISPATCHER_TIMEOUT=30000 + +# 外部容量配置 +EXTERNAL_CAPACITY_MAX=10 +``` + +**说明：** +- `MESSAGE_DISPATCHER_URL`：HTTP 任务提交接口 URL +- `MESSAGE_DISPATCHER_WS_PORT`：任务队列后端 WebSocket 服务端口 +- 不再需要 MD_USERNAME 和 MD_PASSWORD（Token 通过 WebSocket 推送） + +--- + +### 修改点 9：新增配置文件 + +**文件：** `config/messageDispatcher.json`（新建） + +```json +{ + "enabled": true, + "priority": true, + "task": { + "timeout": 30000, + "retryCount": 1 + }, + "capacity": { + "external": 10 + }, + "websocket": { + "port": 8087 + } +} +``` + +--- + +### 修改点 10：算力更新逻辑检查与修复 + +**文件：** `worker_threads/wait/waiting.js`、`worker_threads/callback_result/result.js`、`redis/initQueue.js` 等相关文件 + +**核心需求：** +- 详细检查任务状态管理与算力更新相关代码 +- 验证 30 个任务未全部完成，收到算力更新通知的场景 +- 实现未用算力数计算边界检查，防止负值 +- 添加防御性编程，避免空转或无限循环 +- 实现完整单元测试 + +#### 9.1 检查清单 + +| 检查项 | 检查文件 | 风险级别 | +|---------|---------|---------| +| 算力计数更新原子性 | `redis/initQueue.js` | 高 | +| 任务完成回调时算力计数 | `worker_threads/callback_result/result.js` | 高 | +| 算力减少时的任务处理 | `worker_threads/wait/waiting.js` | 高 | +| 负数检查与防御 | 所有相关文件 | 高 | +| 并发安全 | 所有相关文件 | 中 | + +#### 9.2 核心修复方案 + +**修复 1：添加算力计数边界检查** + +**文件：** `redis/initQueue.js` + +**问题：** 任务完成后，PQtasks（处理中任务数）可能减为负数 + +**修复代码：** +```javascript +async function reducePlatformsProcess(platformKey) { + const key = `${prefix}:platforms:${platformKey}`; + + try { + const current = await redis.hGet(key, 'PQtasks'); + let newValue = parseInt(current) - 1; + + // 边界检查：确保不小于 0 + if (newValue < 0) { + console.warn(`[CapacityManager] 检测到负值: ${platformKey} PQtasks = ${newValue}, 已修正为 0`); + newValue = 0; + } + + await redis.hSet(key, 'PQtasks', newValue.toString()); + console.log(`[CapacityManager] ${platformKey} PQtasks: ${current} -> ${newValue}`); + + return newValue; + } catch (error) { + console.error(`[CapacityManager] 更新 PQtasks 失败:`, error); + throw error; + } +} +``` + +**修复 2：添加算力更新状态锁** + +**文件：** `utils/capacityGuard.js`（新建） + +**功能：** 防止算力更新期间的并发问题 + +```javascript +class CapacityGuard { + constructor() { + this.updateLock = false; + this.pendingUpdates = []; + } + + async acquireLock() { + while (this.updateLock) { + await new Promise(resolve => setTimeout(resolve, 10)); + } + this.updateLock = true; + } + + releaseLock() { + this.updateLock = false; + + // 处理排队的更新 + if (this.pendingUpdates.length > 0) { + const nextUpdate = this.pendingUpdates.shift(); + nextUpdate(); + } + } + + async executeWithLock(fn) { + await this.acquireLock(); + try { + return await fn(); + } finally { + this.releaseLock(); + } + } +} + +export default new CapacityGuard(); +``` + +**修复 3：处理算力突然降低场景** + +**文件：** `worker_threads/wait/waiting.js` + +**场景：** 已发送 30 个任务，算力降低至 20，任务未全部完成 + +**修复代码：** +```javascript +async function handleCapacityReductionFromMD(newInternalCapacity) { + console.log(`[Waiting] 收到算力更新: 内部容量 -> ${newInternalCapacity}`); + + await capacityGuard.executeWithLock(async () => { + // 获取当前正在处理的任务数 + const currentProcessing = await getCurrentProcessingCount(); + + if (currentProcessing <= newInternalCapacity) { + console.log(`[Waiting] 当前处理数 ${currentProcessing} ≤ 新容量 ${newInternalCapacity}, 无需调整`); + return; + } + + const excess = currentProcessing - newInternalCapacity; + console.warn(`[Waiting] 检测到算力降低: 当前处理 ${currentProcessing} > 新容量 ${newInternalCapacity}, 超出 ${excess} 个任务`); + + // 记录超出情况，但不主动取消任务 + // 让任务自然完成，通过回调正确更新计数 + console.log(`[Waiting] 将等待任务自然完成，确保计数正确`); + }); +} +``` + +**修复 4：添加空转防御** + +**文件：** `worker_threads/wait/waiting.js` + +**问题：** 无任务时可能无限循环 + +**修复代码：** +```javascript +// 主循环 +(async () => { + let idleCount = 0; + const MAX_IDLE_COUNT = 10; // 最大连续空转次数 + const IDLE_SLEEP_MS = 10000; // 空转时的睡眠时间 + + while (true) { + try { + const wDeficiency = await judgConcurrency(); + + if (wDeficiency.length > 0) { + idleCount = 0; // 重置空转计数 + logger.info('有可进行处理的队列，数量: ' + wDeficiency.length); + + // ... 原有处理逻辑 ... + } else { + idleCount++; + + if (idleCount >= MAX_IDLE_COUNT) { + logger.debug(`连续空转 ${idleCount} 次，进入长睡眠`); + await new Promise(resolve => setTimeout(resolve, IDLE_SLEEP_MS)); + idleCount = 0; + } else { + logger.debug('没有可处理的队列'); + await new Promise(resolve => setTimeout(resolve, 10000)); + } + } + + } catch (error) { + logger.error('批量处理任务失败:', error); + await new Promise(resolve => setTimeout(resolve, 5000)); + } + } +})(); +``` + +#### 9.3 单元测试方案 + +**测试文件：** `test/capacity.test.js`（新建） + +**测试用例：** + +```javascript +describe('Capacity Management Tests', () => { + + test('正常情况: 任务完成后算力正确增加', async () => { + await initQueue.addPlatformsProcess({ 'digitalHuman:runninghub': 1 }); + const result = await initQueue.reducePlatformsProcess('digitalHuman:runninghub'); + assert.strictEqual(result, 0); + }); + + test('边界情况: 算力为0时尝试减少', async () => { + await initQueue.addPlatformsProcess({ 'digitalHuman:runninghub': 0 }); + const result = await initQueue.reducePlatformsProcess('digitalHuman:runninghub'); + assert.strictEqual(result, 0, '应该保持为0'); + }); + + test('边界情况: 算力从30降低到20时的处理', async () => { + // 模拟发送30个任务 + for (let i = 0; i < 30; i++) { + await initQueue.addPlatformsProcess({ 'digitalHuman:runninghub': 1 }); + } + + // 模拟收到算力降低通知 + await handleCapacityReductionFromMD(20); + + // 验证不会导致负数 + const currentPQtasks = await getCurrentPQtasks(); + assert(currentPQtasks >= 0, 'PQtasks 不能为负数'); + }); + + test('防御性测试: 并发更新', async () => { + const promises = []; + for (let i = 0; i < 100; i++) { + promises.push(initQueue.addPlatformsProcess({ 'digitalHuman:runninghub': 1 })); + promises.push(initQueue.reducePlatformsProcess('digitalHuman:runninghub')); + } + await Promise.all(promises); + const finalCount = await getCurrentPQtasks(); + assert(finalCount >= 0, '并发更新后不能为负数'); + }); + + test('防御性测试: 空转检测', async () => { + const startTime = Date.now(); + // 模拟无任务场景 + await simulateIdleLoop(); + const duration = Date.now() - startTime; + assert(duration < 60000, '不应该无限循环'); + }); +}); +``` + +#### 9.4 验证检查清单 + +| 验证项 | 验证方法 | 预期结果 | +|--------|---------|---------| +| 算力不出现负值 | 检查日志中是否有负值警告 | 如有警告，确认已自动修正为 0 | +| 任务完成后计数正确 | 发送 10 个任务，全部完成 | 最终 PQtasks = 0 | +| 算力降低时不崩溃 | 发送 30 个任务，降低算力到 20 | 系统稳定运行，无错误 | +| 无任务时不空转 | 监控无任务时的 CPU | 进入睡眠，不占用 CPU | +| 并发更新安全 | 100 次并发增减 | 最终计数正确，无负值 | + +--- + +## 二、message-dispatcher 项目修改指导 + +### 项目路径 +`d:\Ke_xue_web\独立项目\comfyui桥接器\message-dispatcher\` + +--- + +### 修改点 1：新增 WebSocket 客户端模块 + +**文件：** `src/md-websocket-client/index.js`（新建） + +**通信架构说明：** +- message-dispatcher 作为 WebSocket 客户端，主动连接任务队列后端 +- WebSocket 用于： + - JWT Token 主动推送与定期更新 + - 算力状态上报 + - 实例状态变化同步 + - 心跳保活 +- **仅任务提交通过 HTTP 接口完成** +- **所有其他通信（包括 Token 推送）均通过 WebSocket** + +**功能要求：** +1. 建立与任务队列后端的 WebSocket 连接 +2. 连接成功后立即推送当前 JWT Token +3. 定期推送 JWT Token 更新（如每 20 小时） +4. 定期推送算力状态（CAPACITY_UPDATE） +5. 实例状态变化时推送（INSTANCE_ONLINE/INSTANCE_OFFLINE 等） +6. 发送心跳保持连接 +7. 自动重连机制（指数退避） + +**核心方法：** +```javascript +class MDWebSocketClient { + constructor() { + this.ws = null; + this.connected = false; + this.reconnectAttempts = 0; + this.tokenPushInterval = null; + this.capacityPushInterval = null; + } + + // 初始化并连接 + async init() + + // 连接到任务队列后端 + async connect() + + // 断开连接 + disconnect() + + // 推送 JWT Token + pushJwtToken() + + // 推送算力状态 + pushCapacityState() + + // 推送实例上线 + pushInstanceOnline(instanceId) + + // 推送实例下线 + pushInstanceOffline(instanceId) + + // 发送消息 + send(message) + + // 处理接收到的消息 + handleMessage(message) +} +``` + +**WebSocket 消息格式（message-dispatcher 发送）：** +```javascript +// JWT Token 更新推送 +{ + type: 'JWT_UPDATE', + data: { + token: 'eyJhbGciOiJIUzI1NiIs...', + expiresAt: '2024-01-02T00:00:00.000Z', + timestamp: '2024-01-01T00:00:00.000Z' + } +} + +// 算力状态更新推送 +{ + type: 'CAPACITY_UPDATE', + data: { + timestamp: '2024-01-01T00:00:00.000Z', + bridges: [...], + summary: { + totalBridges: 2, + totalInstances: 8, + onlineInstances: 6, + busyInstances: 2, + availableCapacity: 4 + } + } +} + +// 心跳 +{ + type: 'HEARTBEAT', + data: { + timestamp: '2024-01-01T00:00:00.000Z' + } +} +``` + +--- + +### 修改点 2：修改启动流程，集成 WebSocket 客户端 + +**文件：** `src/index.js` + +**修改内容：** +- 导入并初始化 MDWebSocketClient +- 在服务启动后启动 WebSocket 客户端 +- 在服务关闭时断开 WebSocket 连接 + +--- + +### 修改点 3：新增 runninghub 兼容接口（可选） + +**目标：** 确保接口兼容性，新增 runninghub 兼容的任务提交接口 + +**文件：** `src/api/index.js` + +**新增接口（可选）：** +```javascript +// 兼容 runninghub 格式的任务提交接口 +router.post('/task/runninghub', authMiddleware, async (req, res) => { + // 请求体已经是 runninghub 格式，直接使用 + // 调用现有 /api/task 逻辑 +}); +``` + +--- + +### 修改点 4：确保回调兼容 + +**说明：** message-dispatcher 已支持 `webhookUrl` 参数，任务完成后会调用该回调。需要确保回调格式与 runninghub 保持一致。 + +**检查点：** +- 确认 `TASK_END` 消息处理中正确调用 webhookUrl +- 确保回调数据格式与 runninghub 兼容 + +--- + +### 修改点 5：新增任务处理与算力动态调整机制 + +**文件：** `src/task-scheduler/index.js`（新建） + +**核心需求：** +- 当任务队列后端发送 30 个任务，而系统算力突然降低至 20 时，实现任务保留机制 +- 将超出当前算力的任务存入缓存队列 +- 实时监控算力变化，有空闲算力时按 FIFO 取出任务处理 +- 管理缓存任务状态（等待中、处理中、已完成、失败重试等） + +**数据结构设计：** + +| 数据结构 | 类型 | 说明 | +|---------|------|------| +| `pendingTaskQueue` | Array | 待执行任务缓存队列（FIFO） | +| `processingTasks` | Map | 执行中任务 | +| `completedTasks` | List | 已完成任务（最近 1000 条） | +| `failedTasks` | List | 失败任务（最近 100 条） | + +**任务状态定义：** +```javascript +const TASK_STATES = { + PENDING: 'pending', // 等待中 + PROCESSING: 'processing', // 处理中 + COMPLETED: 'completed', // 已完成 + FAILED: 'failed', // 失败 + RETRYING: 'retrying' // 重试中 +}; +``` + +**核心类设计：** +```javascript +class TaskScheduler { + constructor() { + this.pendingTaskQueue = []; // FIFO 队列 + this.processingTasks = new Map(); // taskId -> taskInfo + this.currentCapacity = 0; // 当前可用算力 + this.maxCapacity = 0; // 最大算力 + this.schedulerLoop = null; + } + + // 初始化调度器 + async init() + + // 设置当前可用算力 + setCurrentCapacity(capacity) + + // 添加任务到缓存队列 + addTaskToPending(task) + + // 从缓存队列取出任务 + getTaskFromPending() + + // 将任务标记为处理中 + markTaskAsProcessing(taskId, instanceId) + + // 将任务标记为已完成 + markTaskAsCompleted(taskId, result) + + // 将任务标记为失败 + markTaskAsFailed(taskId, error) + + // 主调度循环 + async schedulerLoop() + + // 检查是否有空闲算力 + hasAvailableCapacity() + + // 获取可用任务数 + getAvailableSlots() + + // 处理算力降低 + handleCapacityReduction(newCapacity) + + // 处理算力增加 + handleCapacityIncrease(newCapacity) +} +``` + +**场景处理示例：** + +**场景 1：算力从 30 降低至 20** +```javascript +async handleCapacityReduction(newCapacity) { + const currentProcessingCount = this.processingTasks.size; + + // 如果处理中的任务数超过新容量 + if (currentProcessingCount > newCapacity) { + const excessCount = currentProcessingCount - newCapacity; + + // 获取最早开始的 excessCount 个任务 + const tasksToMoveBack = Array.from(this.processingTasks.values()) + .sort((a, b) => a.startTime - b.startTime) + .slice(0, excessCount); + + // 将任务移回 pending 队列头部（优先级高） + for (const task of tasksToMoveBack.reverse()) { + this.processingTasks.delete(task.taskId); + this.pendingTaskQueue.unshift({ + ...task, + state: TASK_STATES.PENDING, + movedBackAt: new Date().toISOString() + }); + } + + console.log(`[TaskScheduler] 算力降低: ${this.currentCapacity} -> ${newCapacity}, 已将 ${excessCount} 个任务移回缓存队列`); + } + + this.currentCapacity = newCapacity; +} +``` + +**场景 2：有空闲算力时调度任务** +```javascript +async schedulePendingTasks() { + const availableSlots = this.getAvailableSlots(); + + if (availableSlots <= 0 || this.pendingTaskQueue.length === 0) { + return; + } + + const tasksToSchedule = this.pendingTaskQueue.splice(0, availableSlots); + + for (const task of tasksToSchedule) { + // 分配任务到可用实例 + const instanceId = await this.selectAvailableInstance(); + + this.markTaskAsProcessing(task.taskId, instanceId); + + // 发送任务到实例 + await this.sendTaskToInstance(task, instanceId); + } + + console.log(`[TaskScheduler] 已调度 ${tasksToSchedule.length} 个任务`); +} +``` + +**算力更新监听：** +```javascript +// 监听来自任务队列后端的算力更新（通过 WebSocket） +handleCapacityUpdateFromBackend(data) { + const newCapacity = data.summary.availableCapacity; + + if (newCapacity < this.currentCapacity) { + this.handleCapacityReduction(newCapacity); + } else if (newCapacity > this.currentCapacity) { + this.handleCapacityIncrease(newCapacity); + } else { + this.currentCapacity = newCapacity; + } +} +``` + +--- + +### 修改点 6：集成任务调度器到主流程 + +**文件：** `src/index.js` + +**修改内容：** +- 导入并初始化 TaskScheduler +- 在 WebSocket 客户端收到算力更新时通知调度器 +- 在任务开始/完成时通知调度器更新状态 +- 在服务关闭时优雅关闭调度器 + +--- + +## 三、技术要求 + +### 3.1 代码规范 +- 遵循现有代码风格（ES Module, async/await） +- 保持与 runninghub.js 相同的接口签名 +- 请求体字段名称、类型、格式必须与 runninghub 完全一致 +- 仅修改 apiKey 字段的值为 JWT Token +- 添加充分的日志记录（使用 console.log/console.error） + +### 3.2 错误处理 +- 健康检查失败不应导致主进程崩溃 +- 降级机制必须可靠 +- 超时处理完善 +- JWT Token 更新失败不应中断服务 + +### 3.3 性能要求 +- 健康检查间隔不小于 10 秒 +- 决策时间不超过 100ms +- 不影响现有系统吞吐量 + +--- + +## 四、验收标准 + +### 4.1 功能验收 +- [ ] 任务分流逻辑正确：≤内部容量全部走内部，超出部分走外部 +- [ ] message-dispatcher 不可用时自动降级至 runninghub +- [ ] 请求体与 runninghub 完全一致（仅 apiKey 值不同） +- [ ] JWT Token 自动更新机制正常工作 +- [ ] 任务成功执行并返回正确结果 +- [ ] 回调接口正常工作 +- [ ] 现有功能不受影响（jimuai、coze 等平台正常工作） + +### 4.2 接口兼容性验收 +- [ ] messageDispatcher.js 接口方法签名与 runninghub.js 完全一致 +- [ ] 请求体字段名称、类型、格式与 runninghub 完全一致 +- [ ] 响应格式与 runninghub 兼容 +- [ ] 不修改 runninghub.js 的任何代码 + +### 4.3 容量边界验收 +- [ ] WebSocket 服务正常启动，message-dispatcher 成功连接 +- [ ] JWT Token 成功通过 WebSocket 接收 +- [ ] JWT Token 定期更新通过 WebSocket 接收 +- [ ] 算力状态实时同步（通过 WebSocket） +- [ ] 实例状态变化实时同步（通过 WebSocket） +- [ ] 场景 1：25个任务 → 全部25个走内部（实时容量） +- [ ] 场景 2：30个任务 → 全部30个走内部（实时容量） +- [ ] 场景 3：35个任务 → 30个内部，5个外部（实时容量） +- [ ] 场景 4：40个任务 → 30个内部，10个外部（实时容量） +- [ ] 场景 5：45个任务 → 30个内部，10个外部，5个等待（实时容量） + +### 4.4 日志验收 +- [ ] 记录每次分发决策（使用内部/外部） +- [ ] 记录容量使用统计 +- [ ] 记录降级事件及原因 +- [ ] 记录 JWT Token 接收/更新日志（通过 WebSocket） +- [ ] 记录 WebSocket 连接/断开日志 + +### 4.5 可靠性验收 +- [ ] message-dispatcher 重启后自动恢复 +- [ ] 网络波动不影响降级逻辑 +- [ ] JWT Token 自动更新不中断服务 +- [ ] 连续运行 24 小时无崩溃 + +### 4.6 算力管理验收 +- [ ] 算力计数从不出现负值 +- [ ] 任务完成后算力计数正确增加 +- [ ] 算力从 30 降低到 20 时系统稳定 +- [ ] 无任务时进入睡眠，不空转 +- [ ] 100 次并发更新后计数正确 +- [ ] 单元测试覆盖正常、边界、异常情况 + +--- + +## 五、关键文件参考 + +### 任务队列后端 +- `outside/outPlatforms/runninghub.js` - 参考接口设计 +- `outside/generat.js` - 修改任务分发逻辑 +- `outside/outPlatforms/outside.js` - 平台注册 +- `worker_threads/wait/waiting.js` - 批量任务分流 +- `worker_threads/wait/generatTask.js` - 任务处理流程 + +### message-dispatcher +- `src/api/index.js` - API 接口定义 +- `src/bridge-manager/index.js` - 桥接器管理 +- `src/websocket-server/index.js` - 任务发送逻辑 + +--- + +## 六、实现顺序建议 + +### 任务队列后端实现顺序 +1. **第一步：** 创建 `mdWebSocketServer.js` WebSocket 服务模块 +2. **第二步：** 创建 `taskDistributor.js` 任务分流模块 +3. **第三步：** 创建 `messageDispatcher.js` 平台适配器 +4. **第四步：** 修改 `waiting.js` 实现批量任务分流 +5. **第五步：** 修改 `generat.js` 实现单任务分发决策 +6. **第六步：** 更新 `outside.js` 和配置文件 +7. **第七步：** 检查并修复 `redis/initQueue.js` 算力计数（添加边界检查） +8. **第八步：** 创建 `capacityGuard.js` 算力更新锁 +9. **第九步：** 修改 `waiting.js` 添加空转防御 +10. **第十步：** 创建单元测试 `test/capacity.test.js` +11. **第十一步：** 测试验证功能 + +### message-dispatcher 实现顺序 +1. **第一步：** 创建 `md-websocket-client/index.js` WebSocket 客户端模块 +2. **第二步：** 修改 `src/index.js` 集成 WebSocket 客户端 +3. **第三步：** 创建 `task-scheduler/index.js` 任务调度器 +4. **第四步：** 修改 `src/index.js` 集成任务调度器 +5. **第五步：** 添加 runninghub 兼容接口（可选） +6. **第六步：** 确保回调兼容 +7. **第七步：** 测试验证功能 + +--- + +## 七、注意事项 + +⚠️ **重要提醒：** +1. **不要修改** `runninghub.js` 的现有代码 +2. **保持** `externalPostRequest()` 的返回值格式不变 +3. **确保** 请求体与 runninghub 完全一致（仅 apiKey 值不同） +4. **确保** 回调接口格式与 runninghub 完全一致 +5. **不要** 破坏现有其他平台（jimuai、coze）的功能 +6. **移除** 所有高低优先级任务的区分逻辑 +7. **新增** 日志时使用清晰的前缀，如 `[MessageDispatcher]`、`[JWTManager]`、`[TaskDistributor]` + +--- + +## 八、验证方法 + +### 8.1 接口兼容性验证 + +**验证步骤：** +1. 对比 messageDispatcher.js 与 runninghub.js 的方法签名 +2. 验证 getGenerateBody() 返回的请求体字段名称 +3. 验证 getGenerateBody() 返回的请求体字段类型 +4. 确认仅 apiKey 字段的值被修改 + +**验证代码：** +```javascript +// 对比两个平台适配器的接口 +const runninghubMethods = Object.keys(runninghub); +const messageDispatcherMethods = Object.keys(messageDispatcher); +assert.deepEqual(runninghubMethods, messageDispatcherMethods, '接口方法必须一致'); +``` + +### 8.2 任务分流验证 + +**验证场景：** +1. 构造不同数量的待分发任务 +2. 检查内部/外部任务分配比例 +3. 验证不超过各自容量上限 + +--- + +现在，请按照以上指导开始实现！ diff --git a/README.md b/README.md deleted file mode 100644 index c627dc4..0000000 --- a/README.md +++ /dev/null @@ -1,183 +0,0 @@ -# ComfyUI 实例集群通信中间层 - -多服务器 ComfyUI 实例集群的专用通信桥梁，实现统一状态管理、任务转发与结果回传。 - -## 项目特点 - -- 多服务器分布式部署支持 -- 实时实例状态监控 -- WebSocket 双向通信 -- 任务精准转发与结果回传 -- 专业的可视化管理界面 -- 热更新配置支持 - -## 技术栈 - -| 层级 | 技术 | -|------|------| -| 后端 | Node.js + Express | -| 前端 | Vue 3 + Element Plus + Vite | -| 数据存储 | Redis | -| 通信 | WebSocket (ws) + Axios | - -## 快速开始 - -### 环境要求 - -- Node.js >= 16 -- Redis >= 6 -- 可访问的 ComfyUI 实例 - -### 1. 安装依赖 - -```bash -# 根目录 -cd d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge - -# 安装后端依赖 -cd backend -npm install - -# 安装前端依赖 -cd ../frontend -npm install -``` - -### 2. 配置 - -#### 2.1 环境变量 - -复制并编辑 `.env` 文件： - -```bash -cd backend -cp .env.example .env -# 编辑 .env 文件，配置 Redis、JWT 密钥等 -``` - -#### 2.2 服务器配置 - -编辑 `backend/config/servers.json` 文件，配置 ComfyUI 实例： - -```json -{ - "servers": [ - { - "id": "server-1", - "name": "主服务器", - "ip": "127.0.0.1", - "enabled": true, - "instances": [ - { "port": 8001, "enabled": true }, - { "port": 8002, "enabled": true } - ] - } - ] -} -``` - -### 3. 启动 Redis - -```bash -# Windows -redis-server - -# 或使用 Docker -docker run -d -p 6379:6379 redis:latest -``` - -### 4. 启动服务 - -```bash -# 开发模式（同时启动后端和前端） -cd d:\WebUI\Kexue\comfyui\comfyui-cluster-bridge -npm run dev - -# 或分别启动 -# 后端 (端口 3000) -cd backend -npm run dev - -# 前端 (端口 5173) -cd frontend -npm run dev -``` - -### 5. 访问管理界面 - -打开浏览器访问：http://localhost:5173 - -默认登录账号： -- 用户名：`admin` -- 密码：`admin123` - -## 目录结构 - -``` -comfyui-cluster-bridge/ -├── backend/ # 后端服务 -│ ├── src/ -│ │ ├── index.js # 主入口 -│ │ ├── config/ # 配置模块 -│ │ ├── logger/ # 日志模块 -│ │ ├── cluster-manager/# 集群管理 -│ │ ├── websocket-client/# WebSocket 通信 -│ │ ├── task-forwarder/ # 任务转发 -│ │ ├── file-uploader/ # 文件上传 -│ │ └── admin-api/ # 管理 API -│ ├── config/ # 配置文件 -│ ├── logs/ # 日志目录 -│ └── uploads/ # 上传文件目录 -├── frontend/ # 前端管理界面 -│ ├── src/ -│ │ ├── views/ # 页面组件 -│ │ ├── stores/ # Pinia 状态 -│ │ ├── router/ # 路由配置 -│ │ └── api/ # API 封装 -│ └── index.html -└── README.md -``` - -## API 文档 - -详细的接口文档请参考 [API.md](./API.md)。 - -## 配置说明 - -### 服务器配置 (servers.json) - -| 字段 | 类型 | 说明 | -|------|------|------| -| servers | array | 服务器列表 | -| servers[].id | string | 服务器唯一标识 | -| servers[].name | string | 服务器名称 | -| servers[].ip | string | 服务器 IP 地址 | -| servers[].enabled | boolean | 是否启用 | -| servers[].instances | array | ComfyUI 实例列表 | -| servers[].instances[].port | number | 实例端口 | -| servers[].instances[].enabled | boolean | 是否启用 | -| healthCheck.interval | number | 健康检查间隔 (毫秒) | -| healthCheck.timeout | number | 健康检查超时 (毫秒) | -| taskQueue.websocketUrl | string | 任务队列 WebSocket 地址 | -| upload.url | string | 文件上传接口地址 | - -## 常见问题 - -### Redis 连接失败 - -确保 Redis 服务已启动，检查 `.env` 文件中的 `REDIS_HOST` 和 `REDIS_PORT` 配置。 - -### ComfyUI 实例离线 - -检查： -1. ComfyUI 服务是否正常运行 -2. 端口是否可访问 -3. 防火墙配置 - -### 任务队列 WebSocket 连接失败 - -检查 `servers.json` 中的 `taskQueue.websocketUrl` 配置是否正确。 - -## 许可证 - -MIT diff --git a/backend/docs/任务队列集成提示词.md b/backend/docs/任务队列集成提示词.md deleted file mode 100644 index 877a2fc..0000000 --- a/backend/docs/任务队列集成提示词.md +++ /dev/null @@ -1,306 +0,0 @@ -# 任务队列后端改造提示词 - -## 概述 - -你需要改造现有的任务队列后端，使其能够与 ComfyUI Cluster Bridge（桥接器）建立并维持可靠的双向通信。桥接器位于任务队列和 ComfyUI 后端之间，负责任务的转发和结果的回调。 - -## 通信架构 - -``` -客户端 → 任务队列后端 ←(WebSocket)→ 桥接器 → ComfyUI后端 - ↑_______________________________(Webhook回调)_____| -``` - -## 核心要求 - -### 1. WebSocket 服务器 - -任务队列后端需要实现一个 WebSocket 服务器，桥接器会主动连接到该服务器。 - -#### 连接地址配置 -- WebSocket 路径建议：`/ws` -- 桥接器可通过后台管理界面修改连接地址 -- 默认配置：`ws://localhost:8080/ws` - -#### 连接生命周期 -- 桥接器启动时会主动连接 -- 连接断开后，桥接器每分钟尝试重连一次，直至成功 -- 任务队列后端应支持多个桥接器同时连接 - -### 2. 消息协议 - -#### 消息通用格式 -所有消息采用 JSON 格式，结构如下： -```json -{ - "type": "消息类型", - "data": { ... } -} -``` - -#### 桥接器 → 任务队列（桥接器发送的消息） - -##### REGISTER - 注册消息 -桥接器连接成功后立即发送： -```json -{ - "type": "REGISTER", - "data": { - "bridgeId": "uuid字符串", - "instanceCount": 8, - "availableInstanceCount": 5, - "timestamp": "2024-01-01T00:00:00.000Z" - } -} -``` - -##### HEARTBEAT - 心跳消息 -桥接器定期发送（建议每30秒一次）： -```json -{ - "type": "HEARTBEAT", - "data": { - "instanceCount": 8, - "availableInstanceCount": 5, - "busyInstanceCount": 3, - "timestamp": "2024-01-01T00:00:00.000Z" - } -} -``` - -##### TASK_ACK - 任务确认消息 -桥接器收到任务后立即发送： -```json -{ - "type": "TASK_ACK", - "data": { - "code": 0, - "msg": "success", - "data": { - "taskId": "1910246754753896450", - "taskStatus": "RUNNING" - } - } -} -``` - -##### PONG - 心跳响应 -响应任务队列的 PING 消息： -```json -{ - "type": "PONG" -} -``` - -#### 任务队列 → 桥接器（任务队列发送的消息） - -##### TASK_ASSIGN - 任务分配消息 -```json -{ - "type": "TASK_ASSIGN", - "data": { - "workflowId": "1904136902449209346", - "nodeInfoList": [ - { - "nodeId": "6", - "fieldName": "text", - "fieldValue": "1 girl in classroom" - }, - { - "nodeId": "3", - "fieldName": "seed", - "fieldValue": "1231231" - } - ], - "webhookUrl": "https://shuzhiren.xueai.art/callback" - } -} -``` - -##### PING - 心跳检测 -任务队列可定期发送： -```json -{ - "type": "PING" -} -``` - -### 3. Webhook 回调接口 - -任务队列后端需要提供一个 HTTP POST 接口用于接收任务完成回调。 - -#### 回调请求格式 -``` -POST {webhookUrl} -Content-Type: application/json - -{ - "event": "TASK_END", - "taskId": "1910246754753896450", - "eventData": "{\"code\":0,\"msg\":\"success\",\"data\":[{\"fileUrl\":\"https://example.com/image.png\",\"fileType\":\"png\",\"taskCostTime\":0,\"nodeId\":\"9\"}]}" -} -``` - -#### eventData 字段说明 -`eventData` 是 JSON 字符串，解析后结构如下： - -**成功时：** -```json -{ - "code": 0, - "msg": "success", - "data": [ - { - "fileUrl": "https://example.com/output.png", - "fileType": "png", - "taskCostTime": 0, - "nodeId": "9" - } - ] -} -``` - -**失败时：** -```json -{ - "code": 1, - "msg": "错误信息", - "data": [] -} -``` - -### 4. 任务调度逻辑 - -任务队列应根据桥接器的可用实例数量来分配任务： - -1. **桥接器注册时**：记录该桥接器的总实例数和可用实例数 -2. **心跳更新时**：更新桥接器的实例状态 -3. **任务分配时**： - - 选择有可用实例的桥接器 - - 根据 `availableInstanceCount` 决定发送多少任务 - - 发送任务后，暂减该桥接器的可用计数 - - 收到 TASK_ACK 或 TASK_END 回调后更新状态 - -### 5. 错误处理 - -#### WebSocket 连接错误 -- 桥接器断开连接后，任务队列应： - - 标记该桥接器为离线 - - 停止向其发送新任务 - - 已发送但未完成的任务根据需要重新分配 - -#### 任务超时 -- 任务队列应设置任务超时机制 -- 超时任务可标记为失败或尝试重新分配 -- 超时时间建议：5-10分钟 - -#### Webhook 回调失败 -- 桥接器会尝试发送回调，但可能失败 -- 任务队列应考虑实现回调重试机制 -- 或提供查询任务状态的接口供桥接器轮询 - -## 实施步骤 - -### 第一步：实现 WebSocket 服务器 - -1. 创建 WebSocket 服务器，监听指定端口 -2. 实现连接管理，支持多个桥接器同时连接 -3. 实现消息解析和分发 - -### 第二步：实现注册和心跳机制 - -1. 处理 REGISTER 消息，记录桥接器信息 -2. 处理 HEARTBEAT 消息，更新桥接器状态 -3. 实现桥接器离线检测（如超过3个心跳周期未收到消息） - -### 第三步：实现任务分配 - -1. 创建任务队列存储 -2. 实现任务分配逻辑，根据可用实例选择桥接器 -3. 发送 TASK_ASSIGN 消息 -4. 处理 TASK_ACK 确认消息 - -### 第四步：实现 Webhook 回调接口 - -1. 创建 HTTP POST 接口接收回调 -2. 解析 eventData 字段 -3. 更新任务状态 -4. 触发后续业务逻辑 - -### 第五步：实现错误处理和重试 - -1. 实现桥接器离线后的任务重分配 -2. 实现任务超时机制 -3. 添加监控和日志 - -## API 接口定义（任务队列后端） - -### WebSocket 端点 - -| 路径 | 方法 | 说明 | -|------|------|------| -| `/ws` | WebSocket | 桥接器连接端点 | - -### HTTP 端点 - -| 路径 | 方法 | 说明 | -|------|------|------| -| `/callback` | POST | 接收任务完成回调 | - -## 消息结构速查表 - -### 任务队列发送的消息 - -| 类型 | 说明 | 触发时机 | -|------|------|----------| -| TASK_ASSIGN | 分配任务 | 有新任务需要处理时 | -| PING | 心跳检测 | 定期发送 | - -### 桥接器发送的消息 - -| 类型 | 说明 | 触发时机 | -|------|------|----------| -| REGISTER | 注册桥接器 | 连接成功后立即 | -| HEARTBEAT | 心跳 | 定期发送（30秒） | -| TASK_ACK | 任务确认 | 收到任务后立即 | -| PONG | 心跳响应 | 收到 PING 后 | - -## 集成测试流程 - -### 测试 1：连接和注册 - -1. 启动任务队列后端 -2. 启动桥接器 -3. 验证桥接器成功连接并发送 REGISTER 消息 -4. 验证任务队列正确记录桥接器信息 - -### 测试 2：任务分配和回调 - -1. 通过任务队列提交一个测试任务 -2. 验证任务被分配到桥接器（TASK_ASSIGN） -3. 验证桥接器发送 TASK_ACK -4. 等待任务完成 -5. 验证收到 Webhook 回调 -6. 验证回调数据格式正确 - -### 测试 3：断开重连 - -1. 建立连接后，断开桥接器 -2. 验证任务队列检测到离线 -3. 等待1分钟，验证桥接器自动重连 -4. 验证重连后正常工作 - -### 测试 4：多实例负载 - -1. 配置桥接器有多个 ComfyUI 实例 -2. 同时提交多个任务 -3. 验证任务按实例数量分配 -4. 验证所有任务正常完成 - -## 注意事项 - -1. **webhookUrl 的传递**：任务队列必须在 TASK_ASSIGN 消息中提供 `webhookUrl`，桥接器通过该 URL 发送回调 -2. **taskId 的对应**：TASK_ACK 和回调中的 taskId 应与任务队列中的任务标识对应 -3. **eventData 的格式**：eventData 是 JSON 字符串，需要先解析再使用 -4. **可用性监控**：任务队列应监控桥接器的在线状态，避免向离线桥接器发送任务 -5. **幂等性**：考虑实现回调接口的幂等性，防止重复处理 diff --git a/backend/package.json b/backend/package.json index 6d36994..e775881 100644 --- a/backend/package.json +++ b/backend/package.json @@ -12,7 +12,6 @@ "dotenv": "^16.3.1", "express": "^4.18.2", "form-data": "^4.0.0", - "ioredis": "^5.3.2", "multer": "^1.4.5-lts.1", "uuid": "^9.0.1", "winston": "^3.11.0", diff --git a/backend/src/config/index.js b/backend/src/config/index.js index 6a5283a..9bf5ecf 100644 --- a/backend/src/config/index.js +++ b/backend/src/config/index.js @@ -7,7 +7,6 @@ import fs from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; import comfyUIMonitor from '../comfyui-monitor/index.js'; -import jsonPersistence from '../json-persistence/index.js'; import logger from '../logger/index.js'; const __filename = fileURLToPath(import.meta.url); @@ -121,8 +120,6 @@ class ConfigManager { this.logConfigChange(oldConfig, this.config, operator); - jsonPersistence.saveConfigSnapshot(this.config); - return true; } catch (error) { logger.error('[Config] 保存配置文件失败:', error); diff --git a/backend/src/data-sync/index.js b/backend/src/data-sync/index.js deleted file mode 100644 index 3344817..0000000 --- a/backend/src/data-sync/index.js +++ /dev/null @@ -1,58 +0,0 @@ -import logger from '../logger/index.js'; -import redisManager from '../redis-manager/index.js'; -import jsonPersistence from '../json-persistence/index.js'; -import clusterManager from '../cluster-manager/index.js'; - -class DataSyncManager { - constructor() { - this.syncInterval = null; - this.init(); - } - - init() { - this.startPeriodicSync(); - logger.info('[Data Sync] 数据同步管理器已初始化'); - } - - startPeriodicSync() { - this.syncInterval = setInterval(() => { - this.syncAll().catch(err => { - logger.error('[Data Sync] 定期同步失败:', err); - }); - }, 300000); - } - - stopPeriodicSync() { - if (this.syncInterval) { - clearInterval(this.syncInterval); - this.syncInterval = null; - } - } - - async syncAll() { - logger.info('[Data Sync] 开始数据同步...'); - - await this.syncInstanceStates(); - - logger.info('[Data Sync] 数据同步完成'); - } - - async syncInstanceStates() { - const instances = clusterManager.getAllInstances(); - const stateData = { - syncedAt: new Date().toISOString(), - instances: instances.map(inst => ({ - id: inst.id, - status: inst.status, - load: inst.load, - currentTasks: inst.currentTasks, - lastHeartbeat: inst.lastHeartbeat - })) - }; - - await jsonPersistence.save('cluster', 'state', stateData, 'system'); - logger.debug('[Data Sync] 实例状态已同步到JSON文件'); - } -} - -export default new DataSyncManager(); diff --git a/backend/src/index.js b/backend/src/index.js index bb63d7a..daf9d34 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -4,9 +4,6 @@ import 'dotenv/config'; import clusterManager from './cluster-manager/index.js'; import logger from './logger/index.js'; -import redisManager from './redis-manager/index.js'; -import jsonPersistence from './json-persistence/index.js'; -import dataSyncManager from './data-sync/index.js'; import taskQueueClient from './task-queue-client/index.js'; const app = express(); diff --git a/backend/src/json-persistence/index.js b/backend/src/json-persistence/index.js deleted file mode 100644 index 3bc6295..0000000 --- a/backend/src/json-persistence/index.js +++ /dev/null @@ -1,121 +0,0 @@ -import fs from 'fs'; -import path from 'path'; -import { fileURLToPath } from 'url'; -import crypto from 'crypto'; -import logger from '../logger/index.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -class JsonPersistence { - constructor() { - this.baseDir = path.resolve(__dirname, '../../data'); - this.ensureBaseDir(); - } - - ensureBaseDir() { - if (!fs.existsSync(this.baseDir)) { - fs.mkdirSync(this.baseDir, { recursive: true }); - } - } - - generateChecksum(data) { - return crypto - .createHash('md5') - .update(JSON.stringify(data)) - .digest('hex'); - } - - getFilePath(module, dataType, timestamp = null) { - const ts = timestamp || Date.now(); - const fileName = `${module}_${dataType}_${ts}.json`; - return path.join(this.baseDir, fileName); - } - - async writeAtomic(filePath, data) { - const tempPath = `${filePath}.tmp`; - const dir = path.dirname(filePath); - - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - - fs.writeFileSync(tempPath, JSON.stringify(data, null, 2), 'utf-8'); - fs.renameSync(tempPath, filePath); - } - - async save(module, dataType, data, operator = 'system') { - const timestamp = Date.now(); - const checksum = this.generateChecksum(data); - - const fileData = { - metadata: { - module, - dataType, - createdAt: new Date(timestamp).toISOString(), - version: '1.0.0', - checksum, - operator - }, - data - }; - - const filePath = this.getFilePath(module, dataType, timestamp); - await this.writeAtomic(filePath, fileData); - - logger.info(`[JSON Persistence] 数据已保存: ${filePath}`); - return filePath; - } - - async load(module, dataType, filePath = null) { - let targetPath = filePath; - - if (!targetPath) { - const files = fs.readdirSync(this.baseDir) - .filter(f => f.startsWith(`${module}_${dataType}_`)) - .sort() - .reverse(); - - if (files.length === 0) { - return null; - } - - targetPath = path.join(this.baseDir, files[0]); - } - - if (!fs.existsSync(targetPath)) { - return null; - } - - const content = fs.readFileSync(targetPath, 'utf-8'); - const fileData = JSON.parse(content); - - const calculatedChecksum = this.generateChecksum(fileData.data); - if (fileData.metadata.checksum !== calculatedChecksum) { - logger.warn(`[JSON Persistence] 数据校验失败: ${targetPath}`); - } - - return fileData.data; - } - - async saveTaskHistory(task) { - return await this.save('task', 'history', task, 'system'); - } - - async saveConfigSnapshot(config) { - return await this.save('config', 'snapshot', config, 'system'); - } - - listFiles(module, dataType) { - if (!fs.existsSync(this.baseDir)) { - return []; - } - - return fs.readdirSync(this.baseDir) - .filter(f => f.startsWith(`${module}_${dataType}_`)) - .sort() - .reverse(); - } -} - -export default new JsonPersistence(); diff --git a/backend/src/redis-manager/index.js b/backend/src/redis-manager/index.js deleted file mode 100644 index 4ebcac8..0000000 --- a/backend/src/redis-manager/index.js +++ /dev/null @@ -1,164 +0,0 @@ -import Redis from 'ioredis'; -import logger from '../logger/index.js'; - -const SYSTEM_PREFIX = 'comfyui:cluster'; - -class RedisManager { - constructor() { - this.client = null; - this.connected = false; - this.init(); - } - - init() { - try { - const host = process.env.REDIS_HOST || 'localhost'; - const port = parseInt(process.env.REDIS_PORT || '6379', 10); - const db = parseInt(process.env.REDIS_DB || '0', 10); - - this.client = new Redis({ - host, - port, - db, - retryStrategy: (times) => { - const delay = Math.min(times * 50, 2000); - return delay; - } - }); - - this.client.on('connect', () => { - this.connected = true; - logger.info('[Redis Manager] Redis连接成功'); - }); - - this.client.on('error', (err) => { - this.connected = false; - logger.error('[Redis Manager] Redis连接错误:', err); - }); - - this.client.on('close', () => { - this.connected = false; - logger.warn('[Redis Manager] Redis连接已关闭'); - }); - - } catch (error) { - logger.error('[Redis Manager] Redis初始化失败:', error); - } - } - - getKey(module, dataType, id = '') { - if (id) { - return `${SYSTEM_PREFIX}:${module}:${dataType}:${id}`; - } - return `${SYSTEM_PREFIX}:${module}:${dataType}`; - } - - async setInstanceStatus(instanceId, status) { - const key = this.getKey('cluster', 'instance', instanceId); - await this.client.hset(key, { - status, - updatedAt: new Date().toISOString() - }); - await this.client.expire(key, 86400); - } - - async getInstanceStatus(instanceId) { - const key = this.getKey('cluster', 'instance', instanceId); - return await this.client.hgetall(key); - } - - async setTask(task) { - const key = this.getKey('task', 'task', task.id); - await this.client.hset(key, { - id: task.id, - promptId: task.promptId || '', - workflow: JSON.stringify(task.workflow), - nodeInfoList: JSON.stringify(task.nodeInfoList || []), - workflowId: task.workflowId || '', - instanceId: task.instanceId, - status: task.status, - progress: task.progress || 0, - createdAt: task.createdAt, - startedAt: task.startedAt || '', - completedAt: task.completedAt || '', - result: JSON.stringify(task.result || null), - error: task.error || '' - }); - await this.client.expire(key, 604800); - } - - async getTask(taskId) { - const key = this.getKey('task', 'task', taskId); - const data = await this.client.hgetall(key); - if (!data || Object.keys(data).length === 0) return null; - - return { - id: data.id, - promptId: data.promptId || null, - workflow: JSON.parse(data.workflow), - nodeInfoList: JSON.parse(data.nodeInfoList), - workflowId: data.workflowId || null, - instanceId: data.instanceId, - status: data.status, - progress: parseInt(data.progress, 10), - createdAt: data.createdAt, - startedAt: data.startedAt || null, - completedAt: data.completedAt || null, - result: JSON.parse(data.result), - error: data.error || null - }; - } - - async getAllTasks() { - const pattern = this.getKey('task', 'task', '*'); - const keys = await this.client.keys(pattern); - const tasks = []; - - for (const key of keys) { - const data = await this.client.hgetall(key); - if (data && data.id) { - tasks.push({ - id: data.id, - promptId: data.promptId || null, - workflow: JSON.parse(data.workflow), - nodeInfoList: JSON.parse(data.nodeInfoList), - workflowId: data.workflowId || null, - instanceId: data.instanceId, - status: data.status, - progress: parseInt(data.progress, 10), - createdAt: data.createdAt, - startedAt: data.startedAt || null, - completedAt: data.completedAt || null, - result: JSON.parse(data.result), - error: data.error || null - }); - } - } - - return tasks.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt)); - } - - async deleteTask(taskId) { - const key = this.getKey('task', 'task', taskId); - return await this.client.del(key); - } - - async setConfigLock(configKey) { - const key = this.getKey('config', 'lock', configKey); - const result = await this.client.set(key, '1', 'NX', 'EX', 30); - return result === 'OK'; - } - - async releaseConfigLock(configKey) { - const key = this.getKey('config', 'lock', configKey); - return await this.client.del(key); - } - - quit() { - if (this.client) { - this.client.quit(); - } - } -} - -export default new RedisManager(); diff --git a/backend/src/task-forwarder/index.js b/backend/src/task-forwarder/index.js index bb4d85f..3ad8226 100644 --- a/backend/src/task-forwarder/index.js +++ b/backend/src/task-forwarder/index.js @@ -3,8 +3,6 @@ import logger from '../logger/index.js'; import clusterManager from '../cluster-manager/index.js'; import webSocketClient from '../websocket-client/index.js'; import axios from 'axios'; -import redisManager from '../redis-manager/index.js'; -import jsonPersistence from '../json-persistence/index.js'; import taskQueueClient from '../task-queue-client/index.js'; import fileUploader from '../file-uploader/index.js'; import config from '../config/index.js'; @@ -13,6 +11,7 @@ import path from 'path'; class TaskForwarder { constructor() { + this.tasks = new Map(); this.setupEventListeners(); } @@ -76,7 +75,7 @@ class TaskForwarder { error: null }; - await redisManager.setTask(task); + this.tasks.set(taskId, task); logger.info(`任务已创建: ${taskId}, 分配到实例: ${instance.id}`); try { @@ -85,7 +84,7 @@ class TaskForwarder { } catch (error) { task.status = 'failed'; task.error = error.message; - await redisManager.setTask(task); + this.tasks.set(taskId, task); logger.error(`任务 ${taskId} 提交失败:`, error); if (webhookUrl) { @@ -110,13 +109,12 @@ class TaskForwarder { } async handleExecutionStart(instanceId, promptId) { - const allTasks = await redisManager.getAllTasks(); - for (const task of allTasks) { + for (const [taskId, task] of this.tasks) { if (task.instanceId === instanceId && !task.promptId && task.status === 'submitted') { task.promptId = promptId; task.status = 'running'; task.startedAt = new Date().toISOString(); - await redisManager.setTask(task); + this.tasks.set(taskId, task); clusterManager.updateInstanceStatus(instanceId, 'busy'); logger.info(`任务 ${task.id} 开始执行, promptId: ${promptId}`); break; @@ -125,12 +123,11 @@ class TaskForwarder { } async handleProgress(instanceId, data) { - const allTasks = await redisManager.getAllTasks(); - for (const task of allTasks) { + for (const [taskId, task] of this.tasks) { if (task.instanceId === instanceId && task.status === 'running') { if (data.max && data.max > 0) { task.progress = Math.round((data.value / data.max) * 100); - await redisManager.setTask(task); + this.tasks.set(taskId, task); } break; } @@ -138,14 +135,12 @@ class TaskForwarder { } async handleExecuted(instanceId, data) { - const allTasks = await redisManager.getAllTasks(); - for (const task of allTasks) { + for (const [taskId, task] of this.tasks) { if (task.promptId === data.prompt_id && task.status === 'running') { task.status = 'completed'; task.completedAt = new Date().toISOString(); task.result = data; - await redisManager.setTask(task); - await jsonPersistence.saveTaskHistory(task); + this.tasks.set(taskId, task); clusterManager.updateInstanceStatus(instanceId, 'online'); logger.info(`任务 ${task.id} 执行完成`); @@ -164,14 +159,12 @@ class TaskForwarder { } async handleExecutionError(instanceId, data) { - const allTasks = await redisManager.getAllTasks(); - for (const task of allTasks) { + for (const [taskId, task] of this.tasks) { if (task.promptId === data.prompt_id && task.status === 'running') { task.status = 'failed'; task.completedAt = new Date().toISOString(); task.error = data.exception_message; - await redisManager.setTask(task); - await jsonPersistence.saveTaskHistory(task); + this.tasks.set(taskId, task); clusterManager.updateInstanceStatus(instanceId, 'online'); logger.error(`任务 ${task.id} 执行失败: ${data.exception_message}`); @@ -280,11 +273,11 @@ class TaskForwarder { } async getTask(taskId) { - return await redisManager.getTask(taskId); + return this.tasks.get(taskId) || null; } async getTasks(status = null) { - let tasks = await redisManager.getAllTasks(); + let tasks = Array.from(this.tasks.values()); if (status) { tasks = tasks.filter(t => t.status === status); } @@ -292,7 +285,7 @@ class TaskForwarder { } async cancelTask(taskId) { - const task = await redisManager.getTask(taskId); + const task = this.tasks.get(taskId); if (!task) { return false; } @@ -314,8 +307,7 @@ class TaskForwarder { task.status = 'cancelled'; task.completedAt = new Date().toISOString(); - await redisManager.setTask(task); - await jsonPersistence.saveTaskHistory(task); + this.tasks.set(taskId, task); logger.info(`任务 ${taskId} 已取消`); return true; } diff --git a/backend/src/task-queue-client/index.js b/backend/src/task-queue-client/index.js index faaa966..f037a82 100644 --- a/backend/src/task-queue-client/index.js +++ b/backend/src/task-queue-client/index.js @@ -65,6 +65,10 @@ class MessageDispatcherClient extends EventEmitter { }); }); + this.ws.on('ping', () => { + this.ws.pong(); + }); + this.ws.on('error', (error) => { logger.error('[MessageDispatcher] WebSocket连接错误:', error); this.isConnected = false; diff --git a/message-dispatcher/.env b/message-dispatcher/.env index dbf6942..53fc6a6 100644 --- a/message-dispatcher/.env +++ b/message-dispatcher/.env @@ -6,3 +6,6 @@ JWT_SECRET=comfyui-cluster-bridge-secret-key-2024 JWT_EXPIRES_IN=24h ADMIN_USERNAME=admin ADMIN_PASSWORD=2233..2233 + +# 任务队列后端 WebSocket 地址 +TASK_QUEUE_WS_URL=ws://localhost:8087 diff --git a/message-dispatcher/nodemon.json b/message-dispatcher/nodemon.json new file mode 100644 index 0000000..0faec95 --- /dev/null +++ b/message-dispatcher/nodemon.json @@ -0,0 +1,5 @@ +{ + "watch": ["src"], + "ignore": ["node_modules"], + "ext": "js,json" +} diff --git a/message-dispatcher/src/index.js b/message-dispatcher/src/index.js index 347f371..59b424e 100644 --- a/message-dispatcher/src/index.js +++ b/message-dispatcher/src/index.js @@ -6,6 +6,8 @@ import authRoutes from './auth/index.js'; import apiRoutes from './api/index.js'; import logger from './logger/index.js'; import websocketServer from './websocket-server/index.js'; +import mdWebSocketClient from './md-websocket-client/index.js'; +import taskScheduler from './task-scheduler/index.js'; const app = express(); const PORT = process.env.PORT || 4000; @@ -33,4 +35,26 @@ const server = app.listen(PORT, () => { console.log('========================================'); }); +server.keepAliveTimeout = 65000; +server.headersTimeout = 66000; + websocketServer.start(server); + +taskScheduler.init().then(() => { + console.log('[TaskScheduler] 任务调度器已初始化'); + return mdWebSocketClient.init(); +}).then(() => { + console.log('[MDWebSocketClient] WebSocket 客户端已初始化'); +}).catch((error) => { + console.error('初始化失败:', error); +}); + +process.on('SIGINT', async () => { + console.log('正在关闭服务...'); + await taskScheduler.shutdown(); + mdWebSocketClient.disconnect(); + server.close(() => { + console.log('服务已关闭'); + process.exit(0); + }); +}); diff --git a/message-dispatcher/src/md-websocket-client/index.js b/message-dispatcher/src/md-websocket-client/index.js new file mode 100644 index 0000000..fbe98d2 --- /dev/null +++ b/message-dispatcher/src/md-websocket-client/index.js @@ -0,0 +1,252 @@ +import WebSocket from 'ws'; +import dotenv from 'dotenv'; +import bridgeManager from '../bridge-manager/index.js'; +import jwt from 'jsonwebtoken'; + +dotenv.config(); + +class MDWebSocketClient { + constructor() { + this.ws = null; + this.connected = false; + this.reconnectAttempts = 0; + this.tokenPushInterval = null; + this.capacityPushInterval = null; + this.heartbeatInterval = null; + this.serverUrl = process.env.TASK_QUEUE_WS_URL || 'ws://localhost:8087'; + this.jwtSecret = process.env.JWT_SECRET || 'comfyui-cluster-bridge-secret-key-2024'; + } + + async init() { + console.log('[MDWebSocketClient] 初始化 WebSocket 客户端'); + await this.connect(); + } + + async connect() { + return new Promise((resolve, reject) => { + console.log(`[MDWebSocketClient] 正在连接到 ${this.serverUrl}`); + + this.ws = new WebSocket(this.serverUrl); + + this.ws.on('open', () => { + console.log('[MDWebSocketClient] WebSocket 连接已建立'); + this.connected = true; + this.reconnectAttempts = 0; + + this.pushJwtToken(); + this.pushCapacityState(); + + this.startHeartbeat(); + this.startTokenPushTimer(); + this.startCapacityPushTimer(); + + resolve(); + }); + + this.ws.on('message', (data) => { + this.handleMessage(data); + }); + + this.ws.on('close', (code, reason) => { + console.log(`[MDWebSocketClient] WebSocket 连接已关闭 (code: ${code})`); + this.connected = false; + this.clearIntervals(); + this.scheduleReconnect(); + }); + + this.ws.on('error', (error) => { + console.error('[MDWebSocketClient] WebSocket 连接错误:', error); + this.connected = false; + }); + }); + } + + disconnect() { + console.log('[MDWebSocketClient] 断开 WebSocket 连接'); + this.clearIntervals(); + if (this.ws) { + this.ws.close(); + this.ws = null; + } + this.connected = false; + } + + clearIntervals() { + if (this.tokenPushInterval) { + clearInterval(this.tokenPushInterval); + this.tokenPushInterval = null; + } + if (this.capacityPushInterval) { + clearInterval(this.capacityPushInterval); + this.capacityPushInterval = null; + } + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); + this.heartbeatInterval = null; + } + } + + scheduleReconnect() { + const delay = Math.min(1000 * Math.pow(2, this.reconnectAttempts), 30000); + this.reconnectAttempts++; + console.log(`[MDWebSocketClient] ${delay}ms 后尝试重连 (第 ${this.reconnectAttempts} 次)`); + + setTimeout(() => { + this.connect().catch(err => { + console.error('[MDWebSocketClient] 重连失败:', err); + }); + }, delay); + } + + pushJwtToken() { + const token = this.generateJwtToken(); + const expiresAt = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(); + + const message = { + type: 'JWT_UPDATE', + data: { + token, + expiresAt, + timestamp: new Date().toISOString() + } + }; + + this.send(message); + console.log('[MDWebSocketClient] 已推送 JWT Token'); + } + + generateJwtToken() { + return jwt.sign( + { + sub: 'message-dispatcher', + iss: 'comfyui-bridge', + iat: Math.floor(Date.now() / 1000) + }, + this.jwtSecret, + { expiresIn: '24h' } + ); + } + + pushCapacityState() { + const bridges = bridgeManager.getAllBridges(); + const summary = this.calculateCapacitySummary(bridges); + + const message = { + type: 'CAPACITY_UPDATE', + data: { + timestamp: new Date().toISOString(), + bridges: bridges.map(b => ({ + bridgeId: b.id, + info: b.info + })), + summary + } + }; + + this.send(message); + console.log('[MDWebSocketClient] 已推送算力状态:', summary); + } + + calculateCapacitySummary(bridges) { + let totalInstances = 0; + let onlineInstances = 0; + let busyInstances = 0; + let offlineInstances = 0; + + for (const bridge of bridges) { + if (bridge.info?.instances) { + totalInstances += bridge.info.instances.length; + onlineInstances += bridge.info.instances.filter(i => i.status === 'online').length; + busyInstances += bridge.info.instances.filter(i => i.status === 'busy').length; + offlineInstances += bridge.info.instances.filter(i => i.status === 'offline').length; + } + } + + const availableCapacity = onlineInstances - busyInstances; + + return { + totalBridges: bridges.length, + totalInstances, + onlineInstances, + busyInstances, + offlineInstances, + availableCapacity + }; + } + + pushInstanceOnline(instanceId, bridgeId) { + const message = { + type: 'INSTANCE_ONLINE', + data: { + instanceId, + bridgeId, + timestamp: new Date().toISOString() + } + }; + this.send(message); + console.log(`[MDWebSocketClient] 已推送实例上线: ${instanceId}`); + } + + pushInstanceOffline(instanceId, bridgeId) { + const message = { + type: 'INSTANCE_OFFLINE', + data: { + instanceId, + bridgeId, + timestamp: new Date().toISOString() + } + }; + this.send(message); + console.log(`[MDWebSocketClient] 已推送实例下线: ${instanceId}`); + } + + send(message) { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify(message)); + } else { + console.warn('[MDWebSocketClient] WebSocket 未连接，无法发送消息'); + } + } + + handleMessage(data) { + try { + const message = JSON.parse(data.toString()); + console.log(`[MDWebSocketClient] 收到消息: ${message.type}`); + + switch (message.type) { + case 'HEARTBEAT_ACK': + break; + default: + console.log('[MDWebSocketClient] 未知消息类型:', message.type); + } + } catch (error) { + console.error('[MDWebSocketClient] 解析消息失败:', error); + } + } + + startHeartbeat() { + this.heartbeatInterval = setInterval(() => { + const message = { + type: 'HEARTBEAT', + data: { + timestamp: new Date().toISOString() + } + }; + this.send(message); + }, 30000); + } + + startTokenPushTimer() { + this.tokenPushInterval = setInterval(() => { + this.pushJwtToken(); + }, 20 * 60 * 60 * 1000); + } + + startCapacityPushTimer() { + this.capacityPushInterval = setInterval(() => { + this.pushCapacityState(); + }, 10000); + } +} + +export default new MDWebSocketClient(); diff --git a/message-dispatcher/src/task-scheduler/index.js b/message-dispatcher/src/task-scheduler/index.js new file mode 100644 index 0000000..f36565a --- /dev/null +++ b/message-dispatcher/src/task-scheduler/index.js @@ -0,0 +1,183 @@ +const TASK_STATES = { + PENDING: 'pending', + PROCESSING: 'processing', + COMPLETED: 'completed', + FAILED: 'failed', + RETRYING: 'retrying' +}; + +class TaskScheduler { + constructor() { + this.pendingTaskQueue = []; + this.processingTasks = new Map(); + this.completedTasks = []; + this.failedTasks = []; + this.currentCapacity = 0; + this.maxCapacity = 0; + this.schedulerLoopInterval = null; + } + + async init() { + console.log('[TaskScheduler] 初始化任务调度器'); + this.startSchedulerLoop(); + } + + setCurrentCapacity(capacity) { + const oldCapacity = this.currentCapacity; + this.currentCapacity = capacity; + this.maxCapacity = Math.max(this.maxCapacity, capacity); + + console.log(`[TaskScheduler] 容量更新: ${oldCapacity} -> ${capacity}`); + + if (capacity < oldCapacity) { + this.handleCapacityReduction(capacity); + } else if (capacity > oldCapacity) { + this.handleCapacityIncrease(capacity); + } + } + + addTaskToPending(task) { + const taskWithState = { + ...task, + state: TASK_STATES.PENDING, + addedAt: new Date().toISOString() + }; + this.pendingTaskQueue.push(taskWithState); + console.log(`[TaskScheduler] 任务已加入等待队列: ${task.taskId}, 当前等待数: ${this.pendingTaskQueue.length}`); + } + + getTaskFromPending() { + if (this.pendingTaskQueue.length === 0) { + return null; + } + return this.pendingTaskQueue.shift(); + } + + markTaskAsProcessing(taskId, instanceId) { + const task = this.processingTasks.get(taskId) || this.pendingTaskQueue.find(t => t.taskId === taskId); + if (task) { + task.state = TASK_STATES.PROCESSING; + task.instanceId = instanceId; + task.startedAt = new Date().toISOString(); + this.processingTasks.set(taskId, task); + + const index = this.pendingTaskQueue.findIndex(t => t.taskId === taskId); + if (index !== -1) { + this.pendingTaskQueue.splice(index, 1); + } + + console.log(`[TaskScheduler] 任务开始处理: ${taskId}, 实例: ${instanceId}`); + } + } + + markTaskAsCompleted(taskId, result) { + const task = this.processingTasks.get(taskId); + if (task) { + task.state = TASK_STATES.COMPLETED; + task.result = result; + task.completedAt = new Date().toISOString(); + this.processingTasks.delete(taskId); + + this.completedTasks.push(task); + if (this.completedTasks.length > 1000) { + this.completedTasks.shift(); + } + + console.log(`[TaskScheduler] 任务完成: ${taskId}`); + this.schedulePendingTasks(); + } + } + + markTaskAsFailed(taskId, error) { + const task = this.processingTasks.get(taskId); + if (task) { + task.state = TASK_STATES.FAILED; + task.error = error; + task.failedAt = new Date().toISOString(); + this.processingTasks.delete(taskId); + + this.failedTasks.push(task); + if (this.failedTasks.length > 100) { + this.failedTasks.shift(); + } + + console.error(`[TaskScheduler] 任务失败: ${taskId}`, error); + this.schedulePendingTasks(); + } + } + + hasAvailableCapacity() { + return this.processingTasks.size < this.currentCapacity; + } + + getAvailableSlots() { + return Math.max(0, this.currentCapacity - this.processingTasks.size); + } + + handleCapacityReduction(newCapacity) { + const currentProcessingCount = this.processingTasks.size; + + if (currentProcessingCount > newCapacity) { + const excessCount = currentProcessingCount - newCapacity; + + const tasksToMoveBack = Array.from(this.processingTasks.values()) + .sort((a, b) => new Date(a.startedAt) - new Date(b.startedAt)) + .slice(0, excessCount); + + for (const task of tasksToMoveBack.reverse()) { + this.processingTasks.delete(task.taskId); + this.pendingTaskQueue.unshift({ + ...task, + state: TASK_STATES.PENDING, + movedBackAt: new Date().toISOString() + }); + } + + console.log(`[TaskScheduler] 算力降低: ${this.currentCapacity} -> ${newCapacity}, 已将 ${excessCount} 个任务移回缓存队列`); + } + + this.currentCapacity = newCapacity; + } + + handleCapacityIncrease(newCapacity) { + console.log(`[TaskScheduler] 算力增加: ${this.currentCapacity} -> ${newCapacity}`); + this.currentCapacity = newCapacity; + this.schedulePendingTasks(); + } + + async schedulePendingTasks() { + const availableSlots = this.getAvailableSlots(); + + if (availableSlots <= 0 || this.pendingTaskQueue.length === 0) { + return; + } + + const tasksToSchedule = this.pendingTaskQueue.splice(0, availableSlots); + + console.log(`[TaskScheduler] 调度 ${tasksToSchedule.length} 个任务`); + + for (const task of tasksToSchedule) { + this.markTaskAsProcessing(task.taskId, 'auto-assigned'); + } + } + + startSchedulerLoop() { + this.schedulerLoopInterval = setInterval(() => { + this.schedulePendingTasks(); + }, 1000); + } + + stopSchedulerLoop() { + if (this.schedulerLoopInterval) { + clearInterval(this.schedulerLoopInterval); + this.schedulerLoopInterval = null; + } + } + + async shutdown() { + console.log('[TaskScheduler] 正在关闭调度器'); + this.stopSchedulerLoop(); + } +} + +export default new TaskScheduler(); diff --git a/message-dispatcher/src/websocket-server/index.js b/message-dispatcher/src/websocket-server/index.js index d1f058e..1d27274 100644 --- a/message-dispatcher/src/websocket-server/index.js +++ b/message-dispatcher/src/websocket-server/index.js @@ -10,7 +10,10 @@ class WebSocketServer { } start(server) { - this.wss = new WSServer({ server }); + this.wss = new WSServer({ + server, + keepalive: true + }); logger.info('WebSocket服务器已启动'); this.wss.on('connection', (ws) => { @@ -20,14 +23,44 @@ class WebSocketServer { handleConnection(ws) { let bridgeId = null; + let pingInterval = null; + let pongTimeout = null; logger.info('新的WebSocket连接已建立'); + const PING_INTERVAL = 30000; + const PONG_TIMEOUT = 10000; + + const sendPing = () => { + if (ws.readyState !== WSServer.OPEN) { + clearInterval(pingInterval); + return; + } + ws.ping(); + pongTimeout = setTimeout(() => { + logger.warn('PONG响应超时，关闭连接'); + ws.terminate(); + }, PONG_TIMEOUT); + }; + + pingInterval = setInterval(sendPing, PING_INTERVAL); + ws.on('message', (data) => { this.handleMessage(ws, data, (id) => { bridgeId = id; }); }); + ws.on('pong', () => { + if (pongTimeout) { + clearTimeout(pongTimeout); + pongTimeout = null; + } + }); + ws.on('close', (code, reason) => { + clearInterval(pingInterval); + if (pongTimeout) { + clearTimeout(pongTimeout); + } if (bridgeId) { bridgeManager.unregisterBridge(bridgeId); this.cleanupPendingRequests(bridgeId); @@ -36,6 +69,10 @@ class WebSocketServer { }); ws.on('error', (error) => { + clearInterval(pingInterval); + if (pongTimeout) { + clearTimeout(pongTimeout); + } logger.error('WebSocket连接错误:', error); }); } diff --git a/任务队列后端/.env b/任务队列后端/.env new file mode 100644 index 0000000..2125e64 --- /dev/null +++ b/任务队列后端/.env @@ -0,0 +1,34 @@ +# 项目前缀 +PROJECT_PREFIX='digitalHuman' + +# token 密钥 +TOKEN_SECRET='1Ag9BJJn0rXDnidCyXqu' + +# WebSocket 端口 +WS_PORT=8086 + +# 回调端口 +CALLBACK_PORT=8066 + +# runninghub API +RunningHub_URL='https://www.runninghub.cn/task/openapi/create' + +# 后端接口地址 +BACKEND_API_URL='http://localhost:8787' # http://www.whjbjm.com/api + +# 回调接口地址 +CALLBACK_URL='http://43.248.131.153:8066/callback/all' +# fNkecvcLonpHtFimE4G1BOjcB82yy4PqiQv9caknQqtQAwT1ZAJeWkG7YjY2YVBP +# http://www.whjbjm.com/taskCallback/callback/all + +# redis 地址 +REDIS_URL = '' + +# Message Dispatcher 配置 +MESSAGE_DISPATCHER_URL=http://localhost:4000/api/task +MESSAGE_DISPATCHER_WS_PORT=8087 +MESSAGE_DISPATCHER_ENABLED=true +MESSAGE_DISPATCHER_TIMEOUT=30000 + +# 外部容量配置 +EXTERNAL_CAPACITY_MAX=10 diff --git a/任务队列后端/Temp/router.js b/任务队列后端/Temp/router.js new file mode 100644 index 0000000..fff2368 --- /dev/null +++ b/任务队列后端/Temp/router.js @@ -0,0 +1,36 @@ +import { Router } from 'express'; +import multer, { diskStorage } from 'multer'; +import { join, extname } from 'path'; +import { existsSync, mkdirSync } from 'fs'; + +const router = Router(); + +// 设置存储引擎 +const storage = diskStorage({ + destination: function (req, file, cb) { + const tempDir = join(__dirname, '../../static/Temp'); + if (!existsSync(tempDir)){ + mkdirSync(tempDir, { recursive: true }); // 添加 recursive 参数以确保目录递归创建 + } + cb(null, tempDir); + }, + filename: function (req, file, cb) { + cb(null, Date.now() + extname(file.originalname)); // 重命名文件以避免冲突 + } +}); + +const upload = multer({ storage: storage }); + +// 添加API路由来处理文件上传 +router.post('/uploadImage', upload.single('file'), (req, res) => { + if (!req.file) { + return res.status(400).send('No file uploaded.'); + } + // 返回图片保存的地址与文件名 + const fileResponse = { + filePath: req.file.path, + fileName: req.file.filename + }; + res.json(fileResponse); +}); +export default router; \ No newline at end of file diff --git a/任务队列后端/check_redis.js b/任务队列后端/check_redis.js new file mode 100644 index 0000000..d92b149 --- /dev/null +++ b/任务队列后端/check_redis.js @@ -0,0 +1,109 @@ +import redis from './redis/index.js'; +import dotenv from 'dotenv'; +import initQueue from './redis/initQueue.js'; + +dotenv.config(); + +const prefix = process.env.PROJECT_PREFIX || 'default'; +const initInfoKey = `${prefix}:InitInfo`; + +async function checkRedisData() { + try { + console.log('正在连接Redis...'); + + // 连接Redis + if (!redis.isOpen) { + await redis.connect(); + console.log('Redis连接成功'); + } + + // 获取初始化信息 + const initInfoResult = await redis.json.get(initInfoKey, { path: '$' }); + console.log('\n初始化信息:', JSON.stringify(initInfoResult, null, 2)); + + // 处理Redis返回的数组格式数据 + const initInfo = Array.isArray(initInfoResult) ? initInfoResult[0] : initInfoResult; + + // 获取平台信息 + const platforms = initInfo?.platforms || {}; + console.log('\n平台信息:', JSON.stringify(platforms, null, 2)); + + // 检查每个平台的配置 + for (const [key, platform] of Object.entries(platforms)) { + console.log(`\n平台 ${key} 详情:`); + console.log(` AIGC: ${platform.AIGC}`); + console.log(` platformName: ${platform.platformName}`); + console.log(` WQtasks: ${platform.WQtasks}`); + console.log(` PQtasks: ${platform.PQtasks}`); + console.log(` MAX_CONCURRENT: ${platform.MAX_CONCURRENT}`); + console.log(` waitQueue: ${platform.waitQueue}`); + + // 检查等待队列中的任务数 + const waitQueueLength = await redis.lLen(platform.waitQueue); + console.log(` 等待队列实际长度: ${waitQueueLength}`); + + // 获取等待队列中的任务ID + const waitQueueTasks = await redis.lRange(platform.waitQueue, 0, -1); + console.log(` 等待队列任务ID: ${waitQueueTasks}`); + } + + // 检查各种队列状态 + console.log('\n=== 队列状态检查 ==='); + + // 检查处理队列 + const processPollingLength = await redis.lLen(initQueue.processPolling); + console.log(`处理轮询队列(${initQueue.processPolling})长度: ${processPollingLength}`); + + const processCallbackLength = await redis.lLen(initQueue.processCallback); + console.log(`处理回调队列(${initQueue.processCallback})长度: ${processCallbackLength}`); + + // 检查结果队列 + const resultListLength = await redis.lLen(initQueue.resultList); + console.log(`结果列表(${initQueue.resultList})长度: ${resultListLength}`); + + // 检查错误队列 + const errorListLength = await redis.lLen(initQueue.errorList); + console.log(`错误列表(${initQueue.errorList})长度: ${errorListLength}`); + + // 检查待发送消息队列 + const pendingMessagesLength = await redis.lLen(initQueue.pendingMessages); + console.log(`待发送消息队列(${initQueue.pendingMessages})长度: ${pendingMessagesLength}`); + + // 获取待发送消息详情 + if (pendingMessagesLength > 0) { + console.log('\n=== 待发送消息详情 ==='); + const pendingMessageKeys = await redis.lRange(initQueue.pendingMessages, 0, -1); + + for (const messageKey of pendingMessageKeys) { + try { + const messageData = await redis.hGetAll(messageKey); + if (messageData) { + console.log(`消息 ${messageKey}:`); + console.log(` backendId: ${messageData.backendId}`); + console.log(` 消息内容: ${messageData.message?.slice(0, 100)}...`); + console.log(` 时间戳: ${new Date(parseInt(messageData.timestamp || 0)).toLocaleString()}`); + console.log(` 重试次数: ${messageData.retryCount || 0}`); + } + } catch (err) { + console.error(`获取消息 ${messageKey} 失败:`, err); + } + } + } + + // 检查回调队列 + const callbackLength = await redis.lLen(initQueue.callback); + console.log(`\n回调队列(${initQueue.callback})长度: ${callbackLength}`); + + } catch (error) { + console.error('检查Redis数据失败:', error); + } finally { + // 断开Redis连接 + if (redis.isOpen) { + await redis.disconnect(); + console.log('\nRedis连接已关闭'); + } + process.exit(); + } +} + +checkRedisData(); diff --git a/任务队列后端/clearRedis.js b/任务队列后端/clearRedis.js new file mode 100644 index 0000000..dcd20f2 --- /dev/null +++ b/任务队列后端/clearRedis.js @@ -0,0 +1,103 @@ +// clearDigitalHumanData.js +import redis from './redis/index.js'; +import initQueue from './redis/initQueue.js'; + +async function clearDigitalHumanData() { + try { + console.log('开始清除数字人相关数据...'); + + // 1. 清除等待队列 + await redis.del('digitalHuman:runninghub:wait'); + await redis.del('digitalHuman:coze:wait'); + console.log('已清除等待队列'); + + // 2. 清除所有数字人相关的任务数据 + let cursor = '0'; + do { + // 调用scan并打印返回结果（便于调试） + const result = await redis.scan(cursor, { + MATCH: `${initQueue.prefix}:task:*`, + COUNT: 100 + }); + // 【核心修复】从scan返回的对象中正确提取cursor和keys + const newCursor = result.cursor; // 取cursor属性 + const keys = result.keys || []; // 取keys属性，兜底为空数组 + + console.log(`当前游标: ${newCursor}, 找到keys数量: ${keys.length}`); // 调试日志 + + if (keys.length > 0) { + // 加强过滤，确保只保留有效字符串key + const validKeys = keys.filter(key => { + return typeof key === 'string' && key.trim() !== ''; + }); + + if (validKeys.length > 0) { + await redis.del(...validKeys); + console.log(`已清除 ${validKeys.length} 个任务数据`); + } + } + + cursor = newCursor; + } while (cursor !== '0'); + + // 3. 清除轮询队列数据 + cursor = '0'; + do { + const result = await redis.scan(cursor, { + MATCH: `${initQueue.prefix}:processPolling:*`, + COUNT: 100 + }); + // 同样修复解构问题 + const newCursor = result.cursor; + const keys = result.keys || []; + + console.log(`当前游标: ${newCursor}, 找到轮询keys数量: ${keys.length}`); + + if (keys.length > 0) { + const validKeys = keys.filter(key => { + return typeof key === 'string' && key.trim() !== ''; + }); + + if (validKeys.length > 0) { + await redis.del(...validKeys); + console.log(`已清除 ${validKeys.length} 个轮询队列数据`); + } + } + + cursor = newCursor; + } while (cursor !== '0'); + + // 4. 清除结果队列数据 + await redis.del(initQueue.resultName); + await redis.del(initQueue.resultList); + console.log('已清除结果队列数据'); + + // 5. 清除错误队列数据 + await redis.del(initQueue.errorName); + await redis.del(initQueue.errorList); + console.log('已清除错误队列数据'); + + // 6. 清除回调队列数据 + await redis.del(initQueue.callback); + console.log('已清除回调队列数据'); + + // 7. 重置平台信息中的数字人相关计数 + await redis.json.set(initQueue.initInfoKey, '$.platforms.digitalHuman:runninghub.WQtasks', '0'); + await redis.json.set(initQueue.initInfoKey, '$.platforms.digitalHuman:runninghub.PQtasks', '0'); + await redis.json.set(initQueue.initInfoKey, '$.platforms.digitalHuman:coze.WQtasks', '0'); + await redis.json.set(initQueue.initInfoKey, '$.platforms.digitalHuman:coze.PQtasks', '0'); + console.log('已重置平台计数'); + + console.log('数字人相关数据清除完成！'); + } catch (error) { + console.error('清除数据时出错:', error); + process.exit(1); // 异常退出进程 + } finally { + if (redis.isOpen) { + await redis.disconnect(); + console.log('Redis 连接已正常关闭'); + } + } +} + +clearDigitalHumanData(); \ No newline at end of file diff --git a/任务队列后端/config/Config.js b/任务队列后端/config/Config.js new file mode 100644 index 0000000..c9ec4a3 --- /dev/null +++ b/任务队列后端/config/Config.js @@ -0,0 +1,23 @@ +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; + +// 获取当前模块文件的目录 +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// 使用相对于当前文件的路径（注意：文件在同一个目录下，不需要 .. ） +const modelPath = path.join(__dirname, 'model.json'); +const modelData = JSON.parse(fs.readFileSync(modelPath, 'utf8')); + +const platformPath = path.join(__dirname, 'Platform.json'); +const platformData = JSON.parse(fs.readFileSync(platformPath, 'utf8')); + +const CostPath = path.join(__dirname, 'cost.json'); +const CostData = JSON.parse(fs.readFileSync(CostPath, 'utf8')); + +// const errorPath = path.join(__dirname, 'error.json'); +// const errorData = JSON.parse(fs.readFileSync(errorPath, 'utf8')); + +export { modelData, platformData, CostData }; \ No newline at end of file diff --git a/任务队列后端/config/Platform.json b/任务队列后端/config/Platform.json new file mode 100644 index 0000000..a160fba --- /dev/null +++ b/任务队列后端/config/Platform.json @@ -0,0 +1,8 @@ +{ + "callback": [ + "runninghub" + ], + "polling": [ + "coze" + ] +} \ No newline at end of file diff --git a/任务队列后端/config/code.json b/任务队列后端/config/code.json new file mode 100644 index 0000000..bf11244 --- /dev/null +++ b/任务队列后端/config/code.json @@ -0,0 +1,11 @@ +{ + "ERROR": { + "JSONError": "消息格式错误，请联系服务商。", + "OpcodeError": "错误提交，请稍后再试。", + "BalanceError": "余额不足，请充值后继续使用。", + "AssessmentError": "任务提交失败，请稍后再试。" + }, + "SUCCESS": { + "AssessmentSuccess": "任务提交成功，正在排队中..." + } +} \ No newline at end of file diff --git a/任务队列后端/config/cost.json b/任务队列后端/config/cost.json new file mode 100644 index 0000000..cb821fb --- /dev/null +++ b/任务队列后端/config/cost.json @@ -0,0 +1,3 @@ +{ + "runninghub": 0.0012 +} diff --git a/任务队列后端/config/messageDispatcher.json b/任务队列后端/config/messageDispatcher.json new file mode 100644 index 0000000..add845f --- /dev/null +++ b/任务队列后端/config/messageDispatcher.json @@ -0,0 +1,14 @@ +{ + "enabled": true, + "priority": true, + "task": { + "timeout": 30000, + "retryCount": 1 + }, + "capacity": { + "external": 10 + }, + "websocket": { + "port": 8087 + } +} diff --git a/任务队列后端/config/model.json b/任务队列后端/config/model.json new file mode 100644 index 0000000..df42dd8 --- /dev/null +++ b/任务队列后端/config/model.json @@ -0,0 +1,12 @@ +{ + "digitalHuman":{ + "runninghub":{ + "apikey":"3c20cd6c85514d1c86d55a5d3bcd53b7", + "concurrency":13 + }, + "coze":{ + "apikey":"", + "concurrency":20 + } + } +} diff --git a/任务队列后端/index.js b/任务队列后端/index.js new file mode 100644 index 0000000..6d504cf --- /dev/null +++ b/任务队列后端/index.js @@ -0,0 +1,36 @@ +import express, { json, urlencoded } from 'express'; +import cors from 'cors'; +import dotenv from 'dotenv'; +import fileRouter from './upload/index.js'; +import recordRouter from './outside/callback.js'; +import mdWebSocketServer from './utils/mdWebSocketServer.js'; + +// 配置 dotenv 加载环境变量 +dotenv.config(); + +const server = express(); // 接口url +const hostname = '0.0.0.0'; // IP地址 +const port = process.env.CALLBACK_PORT || 8060; // 端口号 +server.use(cors()); // 允许跨域 + +//设置静态资源路径 +server.use('/workflow/uploads', express.static('uploads')); + +// 添加 body-parser 中间件来解析 JSON 请求体 +server.use(json()); +server.use(urlencoded({ extended: true })); + +server.use('/workflow/file', fileRouter); +server.use('/callback', recordRouter); + +// 启动服务器 +server.listen(port, hostname, () => { + console.log(`Server running at http://${hostname}:${port}/`); + + // 初始化 WebSocket 服务 + mdWebSocketServer.init().then(() => { + console.log('[MDWebSocketServer] 初始化完成'); + }).catch((error) => { + console.error('[MDWebSocketServer] 初始化失败:', error); + }); +}); diff --git a/任务队列后端/outside/callback.js b/任务队列后端/outside/callback.js new file mode 100644 index 0000000..73ec8fa --- /dev/null +++ b/任务队列后端/outside/callback.js @@ -0,0 +1,44 @@ +import { Router } from 'express' +import redis from '../redis/index.js' +import initQueue from '../redis/initQueue.js' +const router = Router() + +// 添加API路由来处理文件上传 +router.post('/all', async (req, res) => { + // 立即返回响应，避免平台超时 + res.status(200).json({ success: true, message: 'Received' }); + // // console.log('callback',req.body) + processCallbackData(req.body).catch(error => { + console.error('处理回调数据时出错:', error); + }); +}) + +/** + * 异步处理回调数据 + */ +async function processCallbackData(body) { + let remoteTaskId, eventData + remoteTaskId = body.taskId + eventData = body.eventData + + // 通过remoteTaskId查询对应的taskId + const taskId = await redis.get(`${initQueue.callback}:${remoteTaskId}`); + + if (taskId) { + // 将eventData存储到redis的数据里，使用taskId作为键 + const taskKey = `${initQueue.prefix}:task:${taskId}`; + await redis.hSet(taskKey, 'resultData', eventData); + + // 增加值到回调结果队列列表 + await redis.rPush(initQueue.callback, taskId); + await initQueue.addCallbackRQtasks(1); + console.log('taskKey:', taskKey); + console.log('数据已保存到 Redis:', eventData); + } else { + console.error('未找到对应的taskId，remoteTaskId:', remoteTaskId); + // 可以考虑将未找到的remoteTaskId记录下来，以便后续分析 + await redis.set(`callback:missing:${remoteTaskId}`, JSON.stringify(body), { EX: 86400 }); // 保存24小时 + } +} + +export default router diff --git a/任务队列后端/outside/generat.js b/任务队列后端/outside/generat.js new file mode 100644 index 0000000..4655cb4 --- /dev/null +++ b/任务队列后端/outside/generat.js @@ -0,0 +1,111 @@ +import {modelData} from '../config/Config.js'; +import outside from './outPlatforms/outside.js' +import mdWebSocketServer from '../utils/mdWebSocketServer.js'; +import dotenv from 'dotenv' +dotenv.config() + +// 发送请求 +export async function externalPostRequest(task) { // { aigc, tasksData } + const platform = task.platformName + const AIGC = process.env.PROJECT_PREFIX + + let actualPlatform = platform; + let usedInternal = false; + + // 决策逻辑：如果是 runninghub 且内部有算力，尝试使用内部平台 + if (platform === 'runninghub') { + const internalCapacity = mdWebSocketServer.getInternalCapacity(); + const hasJwtToken = !!mdWebSocketServer.getJwtToken(); + const hasConnectedClients = mdWebSocketServer.hasConnectedClients(); + + if (internalCapacity > 0 && hasJwtToken && hasConnectedClients) { + console.log('[externalPostRequest] 尝试使用内部 messageDispatcher 平台'); + actualPlatform = 'messageDispatcher'; + usedInternal = true; + } + } + + const apikey = modelData[AIGC][platform].apikey + + let response; + let success = false; + + try { + const headers = await outside[actualPlatform].getGenerateHeader(apikey) + const url = outside[actualPlatform].getGenerateUrl() + const body = outside[actualPlatform].getGenerateBody({payload:task.taskData, apikey}) + + console.log(`[externalPostRequest] 发送请求到 ${actualPlatform}: ${url}`); + + response = await fetch(url, { method: 'POST', headers, body: body }); + + // 检查响应状态 + if (!response.ok) { + throw new Error(`外部平台返回错误状态: ${response.status} ${response.statusText}`); + } + + success = true; + } catch (error) { + if (usedInternal) { + console.warn('[externalPostRequest] 内部平台失败，降级使用 runninghub:', error.message); + // 降级到 runninghub + actualPlatform = 'runninghub'; + usedInternal = false; + + try { + const headers = await outside[actualPlatform].getGenerateHeader(apikey) + const url = outside[actualPlatform].getGenerateUrl() + const body = outside[actualPlatform].getGenerateBody({payload:task.taskData, apikey}) + + console.log(`[externalPostRequest] 降级发送请求到 ${actualPlatform}: ${url}`); + + response = await fetch(url, { method: 'POST', headers, body: body }); + + if (!response.ok) { + throw new Error(`降级平台返回错误状态: ${response.status} ${response.statusText}`); + } + + success = true; + } catch (fallbackError) { + console.error('[externalPostRequest] 降级也失败:', fallbackError.message); + return { + taskId: task.taskId, + remoteTaskId: { type: 2, message: `请求失败: ${fallbackError.message}` }, + platform, + AIGC + }; + } + } else { + console.error('[externalPostRequest] 外部请求失败:', error); + return { + taskId: task.taskId, + remoteTaskId: { type: 2, message: `外部请求失败: ${error.message}` }, + platform, + AIGC + }; + } + } + + // 处理成功响应 + try { + const successResult = await outside[actualPlatform].getSuccessTasks(response); + console.log(`[externalPostRequest] ${actualPlatform} 响应:`, successResult); + + let remoteTaskId; + if (successResult.type === 2) { + remoteTaskId = successResult; + } else { + remoteTaskId = { type: 1, data: successResult }; + } + + return { taskId: task.taskId, remoteTaskId, platform: actualPlatform, AIGC, workflowId: task.workflowId }; + } catch (parseError) { + console.error('[externalPostRequest] 解析响应失败:', parseError); + return { + taskId: task.taskId, + remoteTaskId: { type: 2, message: `解析响应失败: ${parseError.message}` }, + platform: actualPlatform, + AIGC + }; + } +} diff --git a/任务队列后端/outside/outPlatforms/JimuAI.js b/任务队列后端/outside/outPlatforms/JimuAI.js new file mode 100644 index 0000000..f64f19f --- /dev/null +++ b/任务队列后端/outside/outPlatforms/JimuAI.js @@ -0,0 +1,189 @@ +const API_BASE_URL = process.env.JIMUAI_API_BASE_URL || 'https://api.xueai.art'; + +// 获取生成接口URL +export function getGenerateUrl() { + return `${API_BASE_URL}/workProgresses`; +} + +// 获取生成接口请求头 +export function getGenerateHeader(apikey) { + const headers = { + 'Content-Type': 'application/json' + }; + + if (apikey) { + headers['WXCZ-ACCESS-KEY'] = apikey; + } + + return headers; +} + +// 获取生成接口请求体 +export function getGenerateBody(task) { + const payload = task.payload; + const apikey = task.apikey; + + const posts = { + plat: 'comfyui', + private: true, + stepEvent: true, + standaloneMode: true, + channelName: 'magicps', + workflow: parseInt(payload.workflowId), + params: { + isFullJson: true, + workflowName: 'workflow-#' + payload.workflowId, + resultUpload: { + storageDays: 7, + category: 'generated' + }, + imageCreation: { + byModel: 'workflow-#' + payload.workflowId + }, + valuesMap: {} + } + }; + + const params = posts.params; + + if (payload.inputs && Array.isArray(payload.inputs)) { + for (const input of payload.inputs) { + const res = {}; + + switch(input.type) { + case 'image': + case 'video': + case 'audio': + if (input.value && typeof input.value === 'object') { + res.url = input.value.url; + res.fileSize = input.value.fileSize; + } + break; + + case 'text': + case 'string': + if (typeof input.value === 'string') { + res.value = input.value; + } + break; + + case 'boolean': + if (typeof input.value === 'boolean') { + res.value = input.value; + } + break; + + case 'number': + if (typeof input.value === 'number') { + res.value = input.value; + } + break; + } + + for (const key of ['id', 'up', 'type']) { + if (input[key] !== undefined) { + res[key] = input[key]; + } + } + + params.valuesMap[input.id] = res; + } + } + + return JSON.stringify(posts); +} + +// 获取查询接口URL +export function getQueryUrl(remoteTaskId) { + return `${API_BASE_URL}/workProgresses/${remoteTaskId}/event`; +} + +// 获取查询 接口请求头 +export function getQueryHeader(apikey) { + const headers = { + 'Content-Type': 'application/json' + }; + + if (apikey) { + headers['WXCZ-ACCESS-KEY'] = apikey; + } + + return headers; +} + +export function getTaskStatus(response) { + const data = response.data; + + if (data && data.status) { + switch(data.status) { + case 'ended': + return true; + case 'error': + case 'timeout': + case 'aborted': + return false; + default: + return null; + } + } + + return null; +} + +export async function getSuccessTasks(response) { + const res = await response.json(); + console.log('积木AI提交任务响应:', res); + + if (res.success && res.data && res.data.id) { + return res.data.id; + } else { + return { message: res.message || '任务提交失败', type: 2 }; + } +} + +export async function getTaskResult(response) { + const res = await response.json(); + console.log('积木AI任务结果:', res); + + if (res.success && res.data) { + const data = res.data; + const files = []; + + if (data.resourceIds) { + const resourceIds = data.resourceIds.split(','); + const apiUrl = API_BASE_URL; + + for (const rId of resourceIds) { + files.push(`${apiUrl}/resources/download/${rId}`); + } + } + + if (files.length > 0) { + return { files: files.length === 1 ? files[0] : files, type: 1 }; + } else { + return { message: '未找到生成结果', type: 2 }; + } + } else { + return { message: res.message || '获取任务结果失败', type: 2 }; + } +} + +export function getStopUrl(taskId) { + return `${API_BASE_URL}/workProgresses/${taskId}`; +} + +export function getStopHeader(apikey) { + const headers = { + 'Content-Type': 'application/json' + }; + + if (apikey) { + headers['WXCZ-ACCESS-KEY'] = apikey; + } + + return headers; +} + +export function getStopBody() { + return JSON.stringify({ status: 'abort' }); +} diff --git a/任务队列后端/outside/outPlatforms/coze/coze.js b/任务队列后端/outside/outPlatforms/coze/coze.js new file mode 100644 index 0000000..c89dbd4 --- /dev/null +++ b/任务队列后端/outside/outPlatforms/coze/coze.js @@ -0,0 +1,266 @@ +import axios from 'axios'; +import * as fs from 'fs'; +import * as crypto from 'crypto'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; +import jwt from 'jsonwebtoken'; +import redis from '../../../redis/index.js'; + +// 获取当前文件的目录（ES模块方式） +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Coze配置 +const COZE_OAUTH_CONFIG = { + appId: '1172420148562', + kid: 'noU76VVvKw679eiyjwHUZLcU2zDwKtSD6N-rOsPIwe0', + privateKeyPath: path.join(__dirname, 'private_key.pem') +}; + +// Redis中存储API密钥的键 +const REDIS_COZE_TOKEN_KEY = 'coze:api:token'; +const REDIS_COZE_EXPIRE_KEY = 'coze:api:expireTime'; + +/** + * 生成符合Coze规范的JWT + * @param {number} durationSeconds - 有效期(秒) + * @returns {Promise} JWT token + */ +async function generateCozeJWT(durationSeconds = 900) { + const privateKey = fs.readFileSync(COZE_OAUTH_CONFIG.privateKeyPath, 'utf8'); + const now = Math.floor(Date.now() / 1000); + + const header = { + alg: 'RS256', + typ: 'JWT', + kid: COZE_OAUTH_CONFIG.kid + }; + + const payload = { + iss: COZE_OAUTH_CONFIG.appId, + aud: 'api.coze.cn', + iat: now, + exp: now + durationSeconds, + jti: crypto.randomBytes(32).toString('hex') + }; + + return jwt.sign(payload, privateKey, { + algorithm: 'RS256', + header: header + }); +} + +/** + * 获取OAuth Access Token + * @param {string} jwtToken - 生成的JWT + * @param {number} durationSeconds - Token有效期 + * @returns {Promise} 包含access_token的对象 + */ +async function getOAuthToken(jwtToken, durationSeconds = 3600) { + try { + const response = await axios.post( + 'https://api.coze.cn/api/permission/oauth2/token', + { + grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', + duration_seconds: durationSeconds + }, + { + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${jwtToken}` + } + } + ); + + return { + access_token: response.data.access_token + }; + } catch (error) { + console.error('获取Coze Token失败:', error.response?.data || error.message); + throw error; + } +} + +/** + * 获取有效的API密钥，过期自动刷新 + * @returns {Promise} 有效的API密钥 + */ +async function getValidApiKey() { + const now = Date.now(); + + try { + // 1. 从Redis获取当前token和过期时间 + const [storedToken, storedExpireTime] = await Promise.all([ + redis.get(REDIS_COZE_TOKEN_KEY), + redis.get(REDIS_COZE_EXPIRE_KEY) + ]); + + // 2. 如果token存在且未过期（5分钟内），则直接返回 + if (storedToken && storedExpireTime && now < parseInt(storedExpireTime) - 5 * 60 * 1000) { + return storedToken; + } + + // 3. 否则重新生成token + console.log('Coze API密钥已过期或不存在，重新生成...'); + + const jwtToken = await generateCozeJWT(); + const tokenResponse = await getOAuthToken(jwtToken); + + const newToken = tokenResponse.access_token; + // 设置过期时间为1小时后 + const newExpireTime = now + 3600 * 1000; + + // 4. 将新token和过期时间存储到Redis，使用Promise.all并行执行 + await Promise.all([ + redis.set(REDIS_COZE_TOKEN_KEY, newToken), + redis.set(REDIS_COZE_EXPIRE_KEY, newExpireTime.toString()), + redis.expire(REDIS_COZE_TOKEN_KEY, 3660), + redis.expire(REDIS_COZE_EXPIRE_KEY, 3660) + ]); + + console.log('Coze API密钥生成成功，有效期1小时'); + return newToken; + } catch (error) { + console.error('获取或生成Coze API密钥失败:', error); + throw error; + } +} + +/** + * 获取生成请求的URL + * @returns {string} 生成请求的URL + */ +function getGenerateUrl() { + return 'https://api.coze.cn/v1/workflow/run'; +} + +/** + * 获取生成请求的Headers + * @param {string} apikey - API密钥（可选，自动生成时忽略） + * @returns {Promise} 生成请求的Headers + */ +async function getGenerateHeader(apikey = null) { + const validApiKey = apikey || await getValidApiKey(); + return { + "Authorization": `Bearer ${validApiKey}`, + "Content-Type": "application/json" + }; +} + +/** + * 获取生成请求的Body + * @param {Object} params - 包含payload和apikey的参数对象 + * @returns {string} 生成请求的Body（JSON字符串） + */ +function getGenerateBody(params) { + try { + // 前端发送的payload已经是完整的请求体对象，包含is_async、parameters和workflow_id + // 只需要将其转换为JSON字符串即可 + return params.payload; + } catch (error) { + console.error('构建Coze请求体失败:', error); + // 返回与成功例子一致的基本格式作为备用 + return JSON.stringify({ is_async: true, parameters: {}, workflow_id: '' }); + } +} + +/** + * 获取查询请求的URL + * @param {string} remoteTaskId - 外部任务ID + * @param {string} workflowId - 工作流ID（可选） + * @returns {string} 查询请求的URL + */ +function getQueryUrl(remoteTaskId, workflowId = null) { + if (!workflowId) { + // 如果没有workflowId，使用通用的运行历史查询接口 + return `https://api.coze.cn/v1/workflow/run_histories/${remoteTaskId}`; + } + return `https://api.coze.cn/v1/workflows/${workflowId}/run_histories/${remoteTaskId}`; +} + +/** + * 获取查询请求的Headers + * @param {string} apikey - API密钥（可选，自动生成时忽略） + * @returns {Promise} 查询请求的Headers + */ +async function getQueryHeader(apikey = null) { + const validApiKey = apikey || await getValidApiKey(); + return { + "Authorization": `Bearer ${validApiKey}`, + "Content-Type": "application/json" + }; +} + +/** + * 处理查询响应，判断任务状态 + * @param {Object} response - 外部平台返回的响应数据 + * @returns {boolean|Object} 任务完成返回结果，否则返回false + */ +function getTaskStatus(response) { + if (!response) { + return false; + } + console.log('Coze API响应:', response); + // 处理Coze API的响应格式 + // Coze API可能直接返回data对象，或者在response.data中 + const data = response.data; + + // 首先检查Coze API的调用状态码 + // code: 0 表示调用成功，其他值表示调用失败 + if (response.code !== undefined) { + if (response.code !== 0) { + // 调用失败，返回错误信息 + return { + result: JSON.stringify({ error: data.msg || 'API调用失败', code: data.code }), + status: 'failed' + }; + } else { + // 当code为0时，取data列表里的第一个值的execute_status + const taskData = data[0]; + + // 检查任务状态 + // Coze API使用execute_status字段表示任务状态 + if (taskData && taskData.execute_status) { + switch (taskData.execute_status) { + case 'Success': + // 任务完成，返回结果 + console.log('任务成功完成，返回结果'); + return { + result: taskData.output, + status: 'success' + }; + case 'Running': + // 任务执行中，返回false表示继续轮询 + return false; + case 'Fail': + // 任务失败 + return { + result: JSON.stringify({ error: taskData.error_message || taskData.error || taskData.msg || '任务失败' }), + status: 'failed' + }; + default: + // 其他状态，视为任务仍在处理中 + return false; + } + } else { + console.log('taskData 或 execute_status 不存在，继续轮询'); + } + } + } else { + console.log('data.code 不存在，继续轮询'); + } + + // 其他情况，视为任务仍在处理中 + return false; +} + + + +export default { + getGenerateUrl, + getGenerateHeader, + getGenerateBody, + getQueryUrl, + getQueryHeader, + getTaskStatus +}; diff --git a/任务队列后端/outside/outPlatforms/coze/private_key.pem b/任务队列后端/outside/outPlatforms/coze/private_key.pem new file mode 100644 index 0000000..a084995 --- /dev/null +++ b/任务队列后端/outside/outPlatforms/coze/private_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwjpkxuX8ZQkAJ +4rKO2DokUQtS0WZr3HLmJNCZrPRMc4tkXD4TcC1VY3sb03kYq93Beu51x22sxm0n +fAbLQ3a2DtbHhN2oXwRCfNGF1NaKUsQVQo6Mc6Al0kFuzEFpaDte8hLMoWVg5bnK +ill8utHjRTseq9AuDP36zf+amPG5yk77Wi+hjOmv7un7kRhK9D51qBSovsYClOg5 +1VtAbizb6vHHaI0b4xCwq9PPqfcK4B0Nj62LF/IFT4U48yozUm8mD7OI8G5JEsRu +ISJ9ziGsymtSNu+go33YvI6P9ihC1pBSDNl+7pl4MvCmhTXMZRVPhtfDAhOF4O8o +/P2z97FRAgMBAAECggEAJ3LFEdIjbs5ZppvLT5VKcGDXSdrVqpXn6johjaSSNR6/ +712Y1RkEWAbRM+dtMDD+bEN+UjyL6cWwD9lrXzEkrgrkvFGYgQ0x03U2D1P914wk +mad0WDdhefHfgtUKbHXIhi9KOgR5tUu+1l1RH0hSqxgF3JWA/zkR6l7qlG1F3T/S +nNjHV2I5z+DzYY1nK+4VXQjECEBJQT0f6D6uJHIXE1kGz4i/NHdbxuJ7d7ey9TkO +7PY+p5WEjQKOyXwjTFf84I4TAM5TRD0GxtLzys7ZWP6yOfvWPugbk3/znRyS/Rg3 +KR4xz7yxRoMCbRcWxJoMoAQnNd4pF4qnDqeO5WQirQKBgQDZXG1v+Jao6ZZTlscY +WceVKvr1m+nWLNbWtBsaaU3tDepb2N7YE2O8r/gM6twxqrGs038dZRCAtvmx7ntP +MFSOpUlJSIYqK1ANr/uxlGfQF76B1Ht+lMcZ6h72v73jzc0Y2U7OcD+/VrxZMABo +1NOyXfatpYqRzWG1ks8dqT2EfwKBgQDP8Ud7G3bM4Tif3E4RwE5CDdEHmv75WpZY +VqvSv8zHBrJz1KLekYjsAtSjavIxeROZich4jME2RmVsa+52WuPXXhcSGwx4RmKp +zuHKHOj/41VjYvKPim4MWbAxVA3RSVyKT+5vy1AqTZWxzse6JL8eD3ZokCop6Pax +DvTxEiyiLwKBgBuTCCceSC6hg3qTNCq4qQMZcsDZyK5s/cw7CP0uwr4B9+sy9gI/ +Y3W6dSNeYBTE7MlaA1Q9T/ykOcUC1g3TucZm3Yc4dhy/ZeZ2nt2GUC0r9fUOeaQz +R5bYBpmS9YoCv7QZTVAPGWcyn65I0qR562lDVlntGEkq3uxj9XZz0+QNAoGAUifs +6vm13UqamaZr/d1xze0xigS1+oTM48gSiOiYmoXN2a/ITZFIfJ69rnchi2Rf1wi1 ++NL7v1re1ZBrHb3ZSQz2poOjUJ3We2qukLENaZRC90pvtUCnLB//We3wq6CFfGwK +M4crfBs9KowdIzFDhTfsu3FCB17woJHdOqXIlqcCgYEAv3hYuS4FV+NVr1yUocib +aPQmY8/LwaUQsoiAcfWPr5cieghiqmPxTDMScMjN9sM8VeaJczRfvoyoHrXLwLa2 +U0CXgYACDwpaFjQPIENKEqBjv8E/t/OfggqkpGbWdyAPSFubnsGqYf2XSrzcb2aX +I/oPe484hWWwA+NV+sTjQME= +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/任务队列后端/outside/outPlatforms/messageDispatcher.js b/任务队列后端/outside/outPlatforms/messageDispatcher.js new file mode 100644 index 0000000..379f864 --- /dev/null +++ b/任务队列后端/outside/outPlatforms/messageDispatcher.js @@ -0,0 +1,51 @@ +import mdWebSocketServer from '../../utils/mdWebSocketServer.js'; + +export function getGenerateUrl() { + return process.env.MESSAGE_DISPATCHER_URL; +} + +export function getGenerateHeader(apikey) { + const jwtToken = mdWebSocketServer.getJwtToken(); + return { + 'Content-Type': 'application/json' + }; +} + +export function getGenerateBody(task) { + const taskData = JSON.parse(task.payload); + const jwtToken = mdWebSocketServer.getJwtToken(); + const payload = { ...taskData, apiKey: jwtToken, webhookUrl: process.env.CALLBACK_URL }; + console.log('[messageDispatcher] 请求体:', payload); + return JSON.stringify(payload); +} + +export function getQueryUrl() { + return process.env.CALLBACK_URL; +} + +export async function getSuccessTasks(response) { + try { + const res = await response.json(); + console.log('[messageDispatcher] 响应:\n', res); + if (res.success === true && res.data && res.data.requestId) { + return { msg: 'success', code: 0, data: { taskId: res.data.requestId } }; + } else { + return { message: res, type: 2 }; + } + } catch (error) { + console.error('[messageDispatcher] 解析响应失败:', error); + return { message: error.message, type: 2 }; + } +} + +export async function getTaskResult(response) { + const res = await JSON.parse(response); + const files = []; + if (res.msg === 'success' && res.code === 0) { + for (const file of res.data) + files.push(file.fileUrl); + return { files: files[0], type: 1 }; + } else { + return { message: res.msg, type: 2 }; + } +} diff --git a/任务队列后端/outside/outPlatforms/outside.js b/任务队列后端/outside/outPlatforms/outside.js new file mode 100644 index 0000000..9f29155 --- /dev/null +++ b/任务队列后端/outside/outPlatforms/outside.js @@ -0,0 +1,6 @@ +import * as runninghub from './runninghub.js'; +import * as jimuai from './JimuAI.js'; +import coze from './coze/coze.js'; +import * as messageDispatcher from './messageDispatcher.js'; + +export default { runninghub, jimuai, coze, messageDispatcher }; \ No newline at end of file diff --git a/任务队列后端/outside/outPlatforms/runninghub.js b/任务队列后端/outside/outPlatforms/runninghub.js new file mode 100644 index 0000000..d84e5a8 --- /dev/null +++ b/任务队列后端/outside/outPlatforms/runninghub.js @@ -0,0 +1,55 @@ +// 获取生成接口URL +export function getGenerateUrl() { + return process.env.RunningHub_URL +} + +// 获取生成接口请求头 +export function getGenerateHeader(){ + return { + 'Content-Type': 'application/json', + 'Host': 'www.runninghub.cn' + }; +} + +// 获取生成接口请求体 +export function getGenerateBody(task) { + const taskData = JSON.parse(task.payload) + const payload = {...taskData,apiKey:task.apikey,webhookUrl: process.env.CALLBACK_URL} + // console.log('getGenerteBody', payload); + return JSON.stringify(payload); +} + +// 获取回调接口 +export function getQueryUrl() { + return process.env.CALLBACK_URL +} + +// 获取任务状态 +export function getTaskStatus() { + if(response.task_status === 'SUCCESS') return true; +} + +// 提交任务后对返回的数据处理 从返回的数据里筛选成功的任务数据并返回相应外部平台的任务ID +export async function getSuccessTasks(response) { + const res = await response.json() + console.log('runninghub:\n', res) + if(res.msg === 'success' && res.code === 0) { + return res.data.taskId + } else { + return {message:res, type: 2} + } +} + +// 获取结果后对返回的数据处理 从返回的数据里筛选成功的任务数据并返回相应外部平台的任务ID +export async function getTaskResult(response) { + const res = await JSON.parse(response) + const files = [] + if(res.msg === 'success' && res.code === 0) { + for(const file of res.data) + files.push(file.fileUrl) + // console.log('files',files) + return {files: files[0], type: 1} + } else { + return {message:res.msg, type: 2} + } +} \ No newline at end of file diff --git a/任务队列后端/outside/polling.js b/任务队列后端/outside/polling.js new file mode 100644 index 0000000..04e9ae2 --- /dev/null +++ b/任务队列后端/outside/polling.js @@ -0,0 +1,31 @@ +import {modelData} from '../config/Config.js'; +import outside from './outPlatforms/outside.js' + +// 获取外部平台任务结果 +export async function externalGetRequest(remoteTaskId, value) { + const info = JSON.parse(value) + console.log('info.workflowId:', info.workflowId) + console.log(info) + const url = outside[info.platform].getQueryUrl(remoteTaskId, info.workflowId) + const headers = await outside[info.platform].getQueryHeader(modelData[info.AIGC][info.platform].apikey) + const res = await fetch(url, { method: 'GET', headers }); + const response = await res.json() + + const result = outside[info.platform].getTaskStatus(response) + console.log(`[externalGetRequest] getTaskStatus 返回结果:`, result) + // 判断状态是否返回结果 + if(result) { + const taskResult = { + result: result.result, + status: result.status, + remoteTaskId, + aigc: info.AIGC, + platform: info.platform, + name: info.AIGC + ':' + info.platform, + taskId: info.taskId + } + console.log(`[externalGetRequest] 返回任务结果:`, taskResult) + return taskResult + } + console.log(`[externalGetRequest] 任务未完成，返回 undefined`) +} diff --git a/任务队列后端/outside/record.js b/任务队列后端/outside/record.js new file mode 100644 index 0000000..70d77d2 --- /dev/null +++ b/任务队列后端/outside/record.js @@ -0,0 +1,74 @@ +import { addConsumptionHistory } from '../school/api.js' +import outside from './outPlatforms/outside.js' +import { modelData,CostData } from '../config/Config.js'; + +async function getTaskResult(task) { // 创建一个函数，用于获取runninghub的任务结果得到其费用 + const body = JSON.stringify({ + "apiKey": modelData[task.info.AIGC][task.info.platform].apikey, + "taskId": task.remoteTaskId + }) + console.log('获取任务结果参数', task.remoteTaskId) + for (let i = 0; i < 13; i++){ + const response = await fetch('https://www.runninghub.cn/task/openapi/outputs', { + method: 'POST', + headers:{'Host':'www.runninghub.cn','Content-Type':'application/json'}, + body: body + }); + const res = await response.json() + console.log(res) + if ( res.code === 0 && res.msg === 'success'){ + return res + } + await new Promise(resolve => setTimeout(resolve, 5000)); + } + // console.log('生成时长：',res.data.taskCostTime,'三方平台费用：',res.data.thirdPartyConsumeMoney) +} + +async function defaultBody(task, file) { + return { + platformCode: `${task.info.platformId}`, + platformId: task.info.platformId, + taskRootId: task.info.taskRootId, // 根任务ID + parentTaskId: task.info.taskType === 1 ? '0' : (task.info.parentTaskId || '0'), // 记录父任务 ID + taskId: task.taskId, + title: task.info.title || '', + modelName: task.info.modelName || '', + chargeCode: task.info.chargeCode, + quantity: 1, + status: file.type, // 需要在获取结果后判断 1成功 2失败 + fileType: task.info.fileType || 'image', // 文件类型 + fileUrl: file.files || '', + chargeType: task.info.chargeType, // 需要在提交时判断 + taskType: task.info.taskType, //任务类型：1 - 初始模特穿衣生成（仅根任务可用），2 - 对话修改，3 - 生成视频，4 - 生成模特，5 - 生成姿势，6 - 生成背景 + type: 2, // 需要在提交时判断 + actualAmount: task.info.cost, + createTime: task.info.createTime || '', // 创建时间 + parentCreateTime: task.info.parentCreateTime || '', // 父任务创建时间 + parentIndex: task.info.parentIndex || 0 // 父任务图片索引 + } +} + +export async function record(task) { // 创建一个函数，用于记录任务 + const file = await outside[task.info.platform].getTaskResult(task.resultData) + let errorMessage = null + + if(file.type === 2){ + errorMessage = file.message + return false + } + if (task.info.platform === 'runninghub' && task.info.type === 2){ + let res = null + res = await getTaskResult(task) + // console.log('生成时长：',res.data.taskCostTime,'三方平台费用：',res.data.thirdPartyConsumeMoney) + task.info.cost = Math.round(Number(res.data[0].taskCostTime) * 100 * CostData[task.info.platform] + Number(res.data[0].thirdPartyConsumeMoney || 0) * 100) + console.log('任务费用：',task.info.cost) + } + + let body = null + body = await defaultBody(task, file) + + console.log('记录信息\n', body) + const success = await addConsumptionHistory(body,task.token) + console.log('记录任务成功', success.data) + return 'succes'; // 返回 后端任务ID与对应的平台任务ID， “后端任务ID” 该ID为当前后端创建，非外部平台返回的任务ID +} diff --git a/任务队列后端/package-lock.json b/任务队列后端/package-lock.json new file mode 100644 index 0000000..18dca79 --- /dev/null +++ b/任务队列后端/package-lock.json @@ -0,0 +1,1419 @@ +{ + "name": "服装-优化逻辑版 - 积木AI版 (2)", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "axios": "^1.9.0", + "cors": "^2.8.5", + "dotenv": "^16.5.0", + "express": "^4.21.2", + "fs": "^0.0.1-security", + "jsonwebtoken": "^9.0.2", + "multer": "^1.4.5-lts.1", + "path": "^0.12.7", + "redis": "^5.10.0", + "uuid": "^11.0.5", + "ws": "^8.18.3" + } + }, + "node_modules/@redis/bloom": { + "version": "5.10.0", + "resolved": "https://registry.npmmirror.com/@redis/bloom/-/bloom-5.10.0.tgz", + "integrity": "sha512-doIF37ob+l47n0rkpRNgU8n4iacBlKM9xLiP1LtTZTvz8TloJB8qx/MgvhMhKdYG+CvCY2aPBnN2706izFn/4A==", + "license": "MIT", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@redis/client": "^5.10.0" + } + }, + "node_modules/@redis/client": { + "version": "5.10.0", + "resolved": "https://registry.npmmirror.com/@redis/client/-/client-5.10.0.tgz", + "integrity": "sha512-JXmM4XCoso6C75Mr3lhKA3eNxSzkYi3nCzxDIKY+YOszYsJjuKbFgVtguVPbLMOttN4iu2fXoc2BGhdnYhIOxA==", + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@redis/json": { + "version": "5.10.0", + "resolved": "https://registry.npmmirror.com/@redis/json/-/json-5.10.0.tgz", + "integrity": "sha512-B2G8XlOmTPUuZtD44EMGbtoepQG34RCDXLZbjrtON1Djet0t5Ri7/YPXvL9aomXqP8lLTreaprtyLKF4tmXEEA==", + "license": "MIT", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@redis/client": "^5.10.0" + } + }, + "node_modules/@redis/search": { + "version": "5.10.0", + "resolved": "https://registry.npmmirror.com/@redis/search/-/search-5.10.0.tgz", + "integrity": "sha512-3SVcPswoSfp2HnmWbAGUzlbUPn7fOohVu2weUQ0S+EMiQi8jwjL+aN2p6V3TI65eNfVsJ8vyPvqWklm6H6esmg==", + "license": "MIT", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@redis/client": "^5.10.0" + } + }, + "node_modules/@redis/time-series": { + "version": "5.10.0", + "resolved": "https://registry.npmmirror.com/@redis/time-series/-/time-series-5.10.0.tgz", + "integrity": "sha512-cPkpddXH5kc/SdRhF0YG0qtjL+noqFT0AcHbQ6axhsPsO7iqPi1cjxgdkE9TNeKiBUUdCaU1DbqkR/LzbzPBhg==", + "license": "MIT", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@redis/client": "^5.10.0" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmmirror.com/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", + "license": "MIT" + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/axios/-/axios-1.9.0.tgz", + "integrity": "sha512-re4CqKTJaURpzbLHtIi6XpDv20/CnpXOtjRY5/CU32L8gU8ek9UIivcfvSWvmKEngmVbrUtPpdDwWDWL7DNHvg==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmmirror.com/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/call-bound/-/call-bound-1.0.3.tgz", + "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "get-intrinsic": "^1.2.6" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmmirror.com/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmmirror.com/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmmirror.com/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmmirror.com/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dotenv": { + "version": "16.5.0", + "resolved": "https://registry.npmmirror.com/dotenv/-/dotenv-16.5.0.tgz", + "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmmirror.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmmirror.com/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.21.2", + "resolved": "https://registry.npmmirror.com/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.12", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmmirror.com/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmmirror.com/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==", + "license": "ISC" + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.7", + "resolved": "https://registry.npmmirror.com/get-intrinsic/-/get-intrinsic-1.2.7.tgz", + "integrity": "sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0", + "function-bind": "^1.1.2", + "get-proto": "^1.0.0", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/jsonwebtoken": { + "version": "9.0.2", + "resolved": "https://registry.npmmirror.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", + "integrity": "sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==", + "license": "MIT", + "dependencies": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jsonwebtoken/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/jwa": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/jwa/-/jwa-1.4.2.tgz", + "integrity": "sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "3.2.2", + "resolved": "https://registry.npmmirror.com/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "license": "MIT", + "dependencies": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmmirror.com/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmmirror.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmmirror.com/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmmirror.com/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmmirror.com/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/multer": { + "version": "1.4.5-lts.1", + "resolved": "https://registry.npmmirror.com/multer/-/multer-1.4.5-lts.1.tgz", + "integrity": "sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.0.0", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmmirror.com/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmmirror.com/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path": { + "version": "0.12.7", + "resolved": "https://registry.npmmirror.com/path/-/path-0.12.7.tgz", + "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==", + "license": "MIT", + "dependencies": { + "process": "^0.11.1", + "util": "^0.10.3" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "license": "MIT" + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmmirror.com/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmmirror.com/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/redis": { + "version": "5.10.0", + "resolved": "https://registry.npmmirror.com/redis/-/redis-5.10.0.tgz", + "integrity": "sha512-0/Y+7IEiTgVGPrLFKy8oAEArSyEJkU0zvgV5xyi9NzNQ+SLZmyFbUsWIbgPcd4UdUh00opXGKlXJwMmsis5Byw==", + "license": "MIT", + "dependencies": { + "@redis/bloom": "5.10.0", + "@redis/client": "5.10.0", + "@redis/json": "5.10.0", + "@redis/search": "5.10.0", + "@redis/time-series": "5.10.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmmirror.com/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmmirror.com/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmmirror.com/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmmirror.com/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/util": { + "version": "0.10.4", + "resolved": "https://registry.npmmirror.com/util/-/util-0.10.4.tgz", + "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==", + "license": "MIT", + "dependencies": { + "inherits": "2.0.3" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/util/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "license": "ISC" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "11.0.5", + "resolved": "https://registry.npmmirror.com/uuid/-/uuid-11.0.5.tgz", + "integrity": "sha512-508e6IcKLrhxKdBbcA2b4KQZlLVp2+J5UwQ6F7Drckkc5N9ZJwFa4TgWtsww9UG8fGHbm6gbV19TdM5pQ4GaIA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/ws": { + "version": "8.19.0", + "resolved": "https://registry.npmmirror.com/ws/-/ws-8.19.0.tgz", + "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/任务队列后端/package.json b/任务队列后端/package.json new file mode 100644 index 0000000..6206868 --- /dev/null +++ b/任务队列后端/package.json @@ -0,0 +1,17 @@ +{ + "type": "module", + "dependencies": { + "axios": "^1.9.0", + "cors": "^2.8.5", + "dotenv": "^16.5.0", + "express": "^4.21.2", + "form-data": "^4.0.5", + "fs": "^0.0.1-security", + "jsonwebtoken": "^9.0.2", + "multer": "^1.4.5-lts.1", + "path": "^0.12.7", + "redis": "^5.10.0", + "uuid": "^11.0.5", + "ws": "^8.18.3" + } +} diff --git a/任务队列后端/pm2Index.config.cjs b/任务队列后端/pm2Index.config.cjs new file mode 100644 index 0000000..81c7509 --- /dev/null +++ b/任务队列后端/pm2Index.config.cjs @@ -0,0 +1,59 @@ +module.exports = { + apps: [{ + name: 'digitalHuman-callbackTask-v2', + script: './index.js', + cwd: './', + args: '', + interpreter: 'node', + interpreter_args: '', + + // 监听文件修改 + watch: true, + ignore_watch: ['config', 'logs', 'node_modules', 'redis', 'school', 'static', 'worker_threads', + '.env', 'package.json', 'package-lock.json', 'pnpm-lock.yaml', 'webSocket.js', + 'outside/outPlatforms', 'outside/generat.js', 'outside/polling.js', 'outside/record.js', + 'pm2Index.config.cjs','pm2Websocket.config.cjs'], + + // 实例数 + instances: 1, + exec_mode: 'fork', + + // 自动重启设置 + autorestart: true, + max_restarts: 30, + min_uptime: '10s', + + // 内存限制重启 + // max_memory_restart: '1G', + + // 日志配置 + out_file: './logs/index/out/out.log', + error_file: './logs/index/error/error.log', + // log_file: './logs/combined.log', + log_type: 'raw', // 或 'json' + log_date_format: 'YYYY-MM-DD HH:mm:ss Z', + logrotate: { + max_size: '5M', // 日志文件最大大小 + retain: 30, // 保留最近7天的日志 + compress: true, // 压缩旧日志 + date_format: 'YYYY-MM-DD' // 日期格式 + }, + + // 合并日志 + // combine_logs: true, + + // 监控和重启设置 + kill_timeout: 1600, + restart_delay: 4000, + + // 环境变量 + // env: { + // NODE_ENV: 'development', + // PORT: 8080 + // }, + env_production: { + NODE_ENV: 'production', + PORT: 8080 + } + }], +}; \ No newline at end of file diff --git a/任务队列后端/pm2Websocket.config.cjs b/任务队列后端/pm2Websocket.config.cjs new file mode 100644 index 0000000..757cbff --- /dev/null +++ b/任务队列后端/pm2Websocket.config.cjs @@ -0,0 +1,44 @@ +module.exports = { + apps: [{ + name: 'digitalHuman-websocketTask-v2', + script: './webSocket.js', + cwd: './', + args: '', + interpreter: 'node', + interpreter_args: '', + + // 监听文件修改 + watch: true, + ignore_watch: ['logs', 'node_modules', 'static', + 'package.json', 'package-lock.json', 'pnpm-lock.yaml', 'index.js', + 'outside/callback.js', 'pm2Index.config.cjs', 'pm2Websocket.config.cjs'], + + // 实例数 - 单实例建议使用 fork 模式 + instances: 1, + exec_mode: 'fork', // cluster 模式下日志轮转需要特殊处理，单实例用 fork 更稳定 + + // 自动重启设置 + autorestart: true, + max_restarts: 30, + min_uptime: '10s', + + // 内存限制重启 + // max_memory_restart: '1G', + + // 日志配置 - 移除原生 logrotate 配置（改用插件） + out_file: './logs/websocket/out/out.log', + error_file: './logs/websocket/error/error.log', + log_type: 'raw', + log_date_format: 'YYYY-MM-DD HH:mm:ss Z', + + // 监控和重启设置 + kill_timeout: 1600, + restart_delay: 4000, + + // 环境变量 + env_production: { + NODE_ENV: 'production', + PORT: 8080 + } + }] +}; \ No newline at end of file diff --git a/任务队列后端/pnpm-lock.yaml b/任务队列后端/pnpm-lock.yaml new file mode 100644 index 0000000..dfe604b --- /dev/null +++ b/任务队列后端/pnpm-lock.yaml @@ -0,0 +1,1040 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + axios: + specifier: ^1.9.0 + version: 1.12.2 + cors: + specifier: ^2.8.5 + version: 2.8.5 + dotenv: + specifier: ^16.5.0 + version: 16.6.1 + express: + specifier: ^4.21.2 + version: 4.21.2 + form-data: + specifier: ^4.0.5 + version: 4.0.5 + fs: + specifier: ^0.0.1-security + version: 0.0.1-security + jsonwebtoken: + specifier: ^9.0.2 + version: 9.0.2 + multer: + specifier: ^1.4.5-lts.1 + version: 1.4.5-lts.2 + path: + specifier: ^0.12.7 + version: 0.12.7 + redis: + specifier: ^5.10.0 + version: 5.10.0 + uuid: + specifier: ^11.0.5 + version: 11.1.0 + ws: + specifier: ^8.18.3 + version: 8.18.3 + +packages: + + '@redis/bloom@5.10.0': + resolution: {integrity: sha512-doIF37ob+l47n0rkpRNgU8n4iacBlKM9xLiP1LtTZTvz8TloJB8qx/MgvhMhKdYG+CvCY2aPBnN2706izFn/4A==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/client@5.10.0': + resolution: {integrity: sha512-JXmM4XCoso6C75Mr3lhKA3eNxSzkYi3nCzxDIKY+YOszYsJjuKbFgVtguVPbLMOttN4iu2fXoc2BGhdnYhIOxA==} + engines: {node: '>= 18'} + + '@redis/json@5.10.0': + resolution: {integrity: sha512-B2G8XlOmTPUuZtD44EMGbtoepQG34RCDXLZbjrtON1Djet0t5Ri7/YPXvL9aomXqP8lLTreaprtyLKF4tmXEEA==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/search@5.10.0': + resolution: {integrity: sha512-3SVcPswoSfp2HnmWbAGUzlbUPn7fOohVu2weUQ0S+EMiQi8jwjL+aN2p6V3TI65eNfVsJ8vyPvqWklm6H6esmg==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/time-series@5.10.0': + resolution: {integrity: sha512-cPkpddXH5kc/SdRhF0YG0qtjL+noqFT0AcHbQ6axhsPsO7iqPi1cjxgdkE9TNeKiBUUdCaU1DbqkR/LzbzPBhg==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + + append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + + array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.12.2: + resolution: {integrity: sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==} + + body-parser@1.20.3: + resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + concat-stream@1.6.2: + resolution: {integrity: sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==} + engines: {'0': node >= 0.8} + + content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + cookie-signature@1.0.6: + resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + + cookie@0.7.1: + resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} + engines: {node: '>= 0.6'} + + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + + cors@2.8.5: + resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} + engines: {node: '>= 0.10'} + + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + + express@4.21.2: + resolution: {integrity: sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==} + engines: {node: '>= 0.10.0'} + + finalhandler@1.3.1: + resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} + engines: {node: '>= 0.8'} + + follow-redirects@1.15.11: + resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + + fs@0.0.1-security: + resolution: {integrity: sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==} + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + http-errors@2.0.0: + resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} + engines: {node: '>= 0.8'} + + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + + inherits@2.0.3: + resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + + jsonwebtoken@9.0.2: + resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} + engines: {node: '>=12', npm: '>=6'} + + jwa@1.4.2: + resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==} + + jws@3.2.2: + resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} + + lodash.includes@4.3.0: + resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + + lodash.isboolean@3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + + lodash.isinteger@4.0.4: + resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} + + lodash.isnumber@3.0.3: + resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} + + lodash.isplainobject@4.0.6: + resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} + + lodash.isstring@4.0.1: + resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} + + lodash.once@4.1.1: + resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + + methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + mkdirp@0.5.6: + resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} + hasBin: true + + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + multer@1.4.5-lts.2: + resolution: {integrity: sha512-VzGiVigcG9zUAoCNU+xShztrlr1auZOlurXynNvO9GiWD1/mTBbUljOKY+qMeazBqXgRnjzeEgJI/wyjJUHg9A==} + engines: {node: '>= 6.0.0'} + deprecated: Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version. + + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + + path-to-regexp@0.1.12: + resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} + + path@0.12.7: + resolution: {integrity: sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==} + + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + qs@6.13.0: + resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} + engines: {node: '>=0.6'} + + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@2.5.2: + resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} + engines: {node: '>= 0.8'} + + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + + redis@5.10.0: + resolution: {integrity: sha512-0/Y+7IEiTgVGPrLFKy8oAEArSyEJkU0zvgV5xyi9NzNQ+SLZmyFbUsWIbgPcd4UdUh00opXGKlXJwMmsis5Byw==} + engines: {node: '>= 18'} + + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + + semver@7.7.3: + resolution: {integrity: sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==} + engines: {node: '>=10'} + hasBin: true + + send@0.19.0: + resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} + engines: {node: '>= 0.8.0'} + + serve-static@1.16.2: + resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} + engines: {node: '>= 0.8.0'} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + side-channel-list@1.0.0: + resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.0: + resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} + engines: {node: '>= 0.4'} + + statuses@2.0.1: + resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} + engines: {node: '>= 0.8'} + + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + + typedarray@0.0.6: + resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + util@0.10.4: + resolution: {integrity: sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==} + + utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + + ws@8.18.3: + resolution: {integrity: sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + +snapshots: + + '@redis/bloom@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/client@5.10.0': + dependencies: + cluster-key-slot: 1.1.2 + + '@redis/json@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/search@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/time-series@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + + append-field@1.0.0: {} + + array-flatten@1.1.1: {} + + asynckit@0.4.0: {} + + axios@1.12.2: + dependencies: + follow-redirects: 1.15.11 + form-data: 4.0.5 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + body-parser@1.20.3: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.13.0 + raw-body: 2.5.2 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + buffer-equal-constant-time@1.0.1: {} + + buffer-from@1.1.2: {} + + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + + bytes@3.1.2: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + + cluster-key-slot@1.1.2: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + concat-stream@1.6.2: + dependencies: + buffer-from: 1.1.2 + inherits: 2.0.4 + readable-stream: 2.3.8 + typedarray: 0.0.6 + + content-disposition@0.5.4: + dependencies: + safe-buffer: 5.2.1 + + content-type@1.0.5: {} + + cookie-signature@1.0.6: {} + + cookie@0.7.1: {} + + core-util-is@1.0.3: {} + + cors@2.8.5: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + + debug@2.6.9: + dependencies: + ms: 2.0.0 + + delayed-stream@1.0.0: {} + + depd@2.0.0: {} + + destroy@1.2.0: {} + + dotenv@16.6.1: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + ee-first@1.1.1: {} + + encodeurl@1.0.2: {} + + encodeurl@2.0.0: {} + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + escape-html@1.0.3: {} + + etag@1.8.1: {} + + express@4.21.2: + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.3 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.7.1 + cookie-signature: 1.0.6 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.3.1 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.3 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.12 + proxy-addr: 2.0.7 + qs: 6.13.0 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.19.0 + serve-static: 1.16.2 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + finalhandler@1.3.1: + dependencies: + debug: 2.6.9 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.1 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + follow-redirects@1.15.11: {} + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + forwarded@0.2.0: {} + + fresh@0.5.2: {} + + fs@0.0.1-security: {} + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + gopd@1.2.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + http-errors@2.0.0: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.1 + toidentifier: 1.0.1 + + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + + inherits@2.0.3: {} + + inherits@2.0.4: {} + + ipaddr.js@1.9.1: {} + + isarray@1.0.0: {} + + jsonwebtoken@9.0.2: + dependencies: + jws: 3.2.2 + lodash.includes: 4.3.0 + lodash.isboolean: 3.0.3 + lodash.isinteger: 4.0.4 + lodash.isnumber: 3.0.3 + lodash.isplainobject: 4.0.6 + lodash.isstring: 4.0.1 + lodash.once: 4.1.1 + ms: 2.1.3 + semver: 7.7.3 + + jwa@1.4.2: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@3.2.2: + dependencies: + jwa: 1.4.2 + safe-buffer: 5.2.1 + + lodash.includes@4.3.0: {} + + lodash.isboolean@3.0.3: {} + + lodash.isinteger@4.0.4: {} + + lodash.isnumber@3.0.3: {} + + lodash.isplainobject@4.0.6: {} + + lodash.isstring@4.0.1: {} + + lodash.once@4.1.1: {} + + math-intrinsics@1.1.0: {} + + media-typer@0.3.0: {} + + merge-descriptors@1.0.3: {} + + methods@1.1.2: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mime@1.6.0: {} + + minimist@1.2.8: {} + + mkdirp@0.5.6: + dependencies: + minimist: 1.2.8 + + ms@2.0.0: {} + + ms@2.1.3: {} + + multer@1.4.5-lts.2: + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 1.6.2 + mkdirp: 0.5.6 + object-assign: 4.1.1 + type-is: 1.6.18 + xtend: 4.0.2 + + negotiator@0.6.3: {} + + object-assign@4.1.1: {} + + object-inspect@1.13.4: {} + + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + + parseurl@1.3.3: {} + + path-to-regexp@0.1.12: {} + + path@0.12.7: + dependencies: + process: 0.11.10 + util: 0.10.4 + + process-nextick-args@2.0.1: {} + + process@0.11.10: {} + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + proxy-from-env@1.1.0: {} + + qs@6.13.0: + dependencies: + side-channel: 1.1.0 + + range-parser@1.2.1: {} + + raw-body@2.5.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + + redis@5.10.0: + dependencies: + '@redis/bloom': 5.10.0(@redis/client@5.10.0) + '@redis/client': 5.10.0 + '@redis/json': 5.10.0(@redis/client@5.10.0) + '@redis/search': 5.10.0(@redis/client@5.10.0) + '@redis/time-series': 5.10.0(@redis/client@5.10.0) + + safe-buffer@5.1.2: {} + + safe-buffer@5.2.1: {} + + safer-buffer@2.1.2: {} + + semver@7.7.3: {} + + send@0.19.0: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + + serve-static@1.16.2: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.19.0 + transitivePeerDependencies: + - supports-color + + setprototypeof@1.2.0: {} + + side-channel-list@1.0.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.0 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + + statuses@2.0.1: {} + + streamsearch@1.1.0: {} + + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + + toidentifier@1.0.1: {} + + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + + typedarray@0.0.6: {} + + unpipe@1.0.0: {} + + util-deprecate@1.0.2: {} + + util@0.10.4: + dependencies: + inherits: 2.0.3 + + utils-merge@1.0.1: {} + + uuid@11.1.0: {} + + vary@1.1.2: {} + + ws@8.18.3: {} + + xtend@4.0.2: {} diff --git a/任务队列后端/redis/index.js b/任务队列后端/redis/index.js new file mode 100644 index 0000000..a44d38e --- /dev/null +++ b/任务队列后端/redis/index.js @@ -0,0 +1,51 @@ +import { createClient } from "redis"; + +// 创建 Redis 客户端，优化配置 +const redis = createClient({ + RESP: 3, + url: process.env.REDIS_URL || 'redis://localhost:16379', + password: process.env.REDIS_PASSWORD || '654321', + // 优化连接配置 + socket: { + // 连接超时时间 + connectTimeout: 10000, + // 保持活动状态 + keepAlive: 30000, + // 重试策略 + reconnectStrategy: (retries) => { + // 最多重试 5 次 + if (retries > 5) { + return new Error('Redis reconnect failed after 5 attempts'); + } + // 指数退避策略 + return Math.min(retries * 1000, 5000); + } + }, + // 禁用不必要的功能 + legacyMode: false, + // 优化命令队列 + enableReadyCheck: true, + // 最大命令队列长度 + maxRetriesPerRequest: 3 +}); + +// 连接事件 +redis.on('connect', () => { + console.log('Redis 连接成功'); +}); + +redis.on('error', (err) => { + console.error('Redis 连接错误:', err); +}); + +redis.on('reconnecting', () => { + console.log('Redis 正在重连...'); +}); + +redis.on('end', () => { + console.log('Redis 连接已关闭'); +}); + +// 导出前自动连接 +redis.connect().catch(console.error); +export default redis; diff --git a/任务队列后端/redis/initQueue.js b/任务队列后端/redis/initQueue.js new file mode 100644 index 0000000..3574fa4 --- /dev/null +++ b/任务队列后端/redis/initQueue.js @@ -0,0 +1,377 @@ +import redis from './index.js'; +import { modelData, platformData } from '../config/Config.js'; +import dotenv from 'dotenv'; + +dotenv.config(); + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +class InitQueues { + constructor() { + this.prefix = process.env.PROJECT_PREFIX || 'default'; + this.processPolling = `${this.prefix}:process:Polling`; // 轮询处理队列名 {remoteTaskId: "JSON.stringify{taskid, platform, AIGC}"} + this.processCallback = `${this.prefix}:process:callback`; // 回调队列名 + this.resultName = `${this.prefix}:result:queue`; // 结果队列名 { taskid:"JSON.stringify(result)" } + this.resultList = `${this.prefix}:result:list`; // 结果列表，存储任务ID + this.callback = `${this.prefix}:callback`; // 回调队列名 + this.errorName = `${this.prefix}:error:queue`; // 错误队列名 + this.errorList = `${this.prefix}:error:list`; // 错误列表，存储任务ID + this.initInfoKey = `${this.prefix}:InitInfo`; // 初始化信息键名 + this.pendingMessages = `${this.prefix}:pending:messages`; // 待发送消息队列，存储断连时未发送的消息 + } + + // 初始化各个队列 + async init(){ + logger.info('开始初始化队列...'); + + const waitQueues = []; // 存储等待队列名称 + const platforms = {}; // 存储平台相关信息 + + try { + // 初始化各个平台的等待队列 + for (const [AIGC, modelObj] of Object.entries(modelData)) { + for (const [platformName, info] of Object.entries(modelObj)) { + // 初始化等待队列名称，不需要实际创建空队列（Redis会自动创建） + const waitName = `${AIGC}:${platformName}:wait`; + waitQueues.push(waitName); + logger.debug(`等待队列创建成功: ${waitName}`); + + // 记录各平台信息 + const platform = { + AIGC, + platformName, + WQtasks: 0, + PQtasks: 0, + MAX_CONCURRENT: info.concurrency, // 最大并发数 + waitQueue: waitName + }; + platforms[`${AIGC}:${platformName}`] = platform; + } + } + + // 检查InitInfo是否已存在 + const existingInfo = await redis.json.get(this.initInfoKey, { path: '$' }); + if (!existingInfo) { + // 初始化配置信息，只需要设置一次 + const initInfo = { + waitQueues: waitQueues, + processPolling: this.processPolling, + processCallback: this.processCallback, + resultName: this.resultName, + PQtasksALL: 0, // 初始值为0 + RQtasksALL: 0, + CQtasksALL: 0, + EQtaskALL: 0, + platforms: platforms + }; + + await redis.json.set(this.initInfoKey, '$', initInfo); + logger.info('Redis初始化完成，创建了InitInfo配置'); + } else { + logger.info('Redis已存在初始化信息，检查并更新配置'); + + // 清理等待队列中的旧任务ID + for (const waitName of waitQueues) { + const queueLength = await redis.lLen(waitName); + if (queueLength > 0) { + await redis.del(waitName); + logger.info(`已清理等待队列 ${waitName}，删除了 ${queueLength} 个旧任务`); + } + } + + // 更新各平台配置，确保MAX_CONCURRENT和其他属性正确设置 + for (const [key, platform] of Object.entries(existingInfo[0].platforms)) { + // 从新生成的platforms中获取最新配置 + const newPlatformConfig = platforms[key]; + if (newPlatformConfig) { + // 更新平台配置，包括MAX_CONCURRENT + await redis.json.set(this.initInfoKey, `$.platforms.${key}`, { + ...platform, + MAX_CONCURRENT: newPlatformConfig.MAX_CONCURRENT, + WQtasks: 0, // 清零等待队列任务数 + waitQueue: newPlatformConfig.waitQueue + }); + logger.debug(`已更新平台 ${key} 配置：MAX_CONCURRENT=${newPlatformConfig.MAX_CONCURRENT}`); + } + } + + logger.info('已更新各平台配置并清零WQtasks计数器'); + } + + logger.info('队列初始化完成'); + } catch (error) { + logger.error('队列初始化失败:', error); + throw error; // 抛出错误以便上层处理 + } + } + + // // 加载现有配置（用于Worker线程） + // async loadExistingConfig() { + // // 重新加载平台信息 + // waitQueues = []; + // platforms.clear(); + + // for (const [AIGC, modelObj] of Object.entries(modelData)) { + // for (const [KEY, info] of Object.entries(modelObj)) { + // const platformName = KEY; + // const waitName = this.toQueue(AIGC, platformName, 'wait'); + // waitQueues.push(waitName); + + // // 从Redis获取当前任务数信息，这里简化处理 + // const platform = { + // AIGC, + // platformName, + // WQtasks: 0, + // PQtasks: 0, + // MAX_CONCURRENT: info.concurrency, + // waitQueue: waitName + // } + // platforms.set(AIGC + ':' + platformName, platform) + // } + // } + // } + + // 获取队列名称 + toQueue(AIGC, model, queue) { + return `${AIGC}:${model}:${queue}`; + } + + // // 获取等待队列名称 + // async getWaitQueueNames(){ + // const res = await redis.json.get("InitInfo", { path: '$.waitQueues' }) + // return res[0] + // } + + // // 获取结果队列名称 + // async getResultName(){ + // const res = await redis.json.get("InitInfo", { path: '$.resultName' }) + // return res[0] + // } + + // 获取平台相关信息 + async getPlatforms() { + try { + const res = await redis.json.get(this.initInfoKey, { path: '$.platforms' }); + return res ? res[0] : {}; + } catch (error) { + logger.error('获取平台信息失败:', error); + return {}; + } + } + + // 获取轮询的处理队列总任务数 + async getPQtasksALL() { + try { + const res = await redis.json.get(this.initInfoKey, { path: '$.PQtasksALL' }); + return res ? res[0] : 0; + } catch (error) { + logger.error('获取轮询处理队列总任务数失败:', error); + return 0; + } + } + + // 获取结果队列任务数 + async getRQtasksALL() { + try { + const res = await redis.json.get(this.initInfoKey, { path: '$.RQtasksALL' }); + return res ? res[0] : 0; + } catch (error) { + logger.error('获取结果队列任务数失败:', error); + return 0; + } + } + + // 获取回调队列任务数 + async getCQtasksALL() { + try { + const res = await redis.json.get(this.initInfoKey, { path: '$.CQtasksALL' }); + return res ? res[0] : 0; + } catch (error) { + logger.error('获取回调队列任务数失败:', error); + return 0; + } + } + + // 获取错误队列任务数 + async getEQtaskALL() { + try { + const res = await redis.json.get(this.initInfoKey, { path: '$.EQtaskALL' }); + return res ? res[0] : 0; + } catch (error) { + logger.error('获取错误队列任务数失败:', error); + return 0; + } + } + + // 增加平台相关信息处理队列 正在处理的任务数 + async addPlatformsProcess(taskCountMap) { + try { + const multi = redis.multi(); + let PQcount = 0; + + for (const [key, count] of taskCountMap.entries()) { + const [aigc, platform] = key.split(':'); + multi.json.numIncrBy(this.initInfoKey, `$.platforms.${key}.PQtasks`, count); + + if (platformData.polling.includes(platform)) { + PQcount++; + } + logger.debug(`增加相关平台处理队列: AIGC=${aigc}, Platform=${platform}, Count=${count}`); + } + + multi.json.numIncrBy(this.initInfoKey, '$.PQtasksALL', PQcount); + await multi.exec(); // 等待multi命令执行完成 + } catch (error) { + logger.error('增加平台处理队列任务数失败:', error); + } + } + + // 减少单个平台处理队列任务数（带边界检查） + async reducePlatformsProcessSingle(platformKey) { + const key = `${this.prefix}:platforms:${platformKey}`; + + try { + const platformInfo = await redis.json.get(this.initInfoKey, { path: `$.platforms.${platformKey}` }); + if (!platformInfo || !platformInfo[0]) { + logger.warn(`[CapacityManager] 平台不存在: ${platformKey}`); + return 0; + } + + const current = platformInfo[0].PQtasks; + let newValue = parseInt(current) - 1; + + if (newValue < 0) { + logger.warn(`[CapacityManager] 检测到负值: ${platformKey} PQtasks = ${newValue}, 已修正为 0`); + newValue = 0; + } + + await redis.json.set(this.initInfoKey, `$.platforms.${platformKey}.PQtasks`, newValue); + logger.debug(`[CapacityManager] ${platformKey} PQtasks: ${current} -> ${newValue}`); + + return newValue; + } catch (error) { + logger.error(`[CapacityManager] 更新 PQtasks 失败:`, error); + throw error; + } + } + + // 减少平台相关信息处理队列 正在处理的任务数 + async reducePlatformsProcess(taskCountMap) { + try { + const multi = redis.multi(); + let PQcount = 0; + + for (const [key, count] of taskCountMap.entries()) { + const [aigc, platform] = key.split(':'); + + const platformKey = key; + const platformInfo = await redis.json.get(this.initInfoKey, { path: `$.platforms.${platformKey}` }); + if (platformInfo && platformInfo[0]) { + const current = platformInfo[0].PQtasks; + let newValue = parseInt(current) - count; + + if (newValue < 0) { + logger.warn(`[CapacityManager] 检测到负值: ${platformKey} PQtasks = ${newValue}, 已修正为 0`); + newValue = 0; + } + + multi.json.set(this.initInfoKey, `$.platforms.${platformKey}.PQtasks`, newValue); + logger.debug(`[CapacityManager] ${platformKey} PQtasks: ${current} -> ${newValue}`); + } + + if (platformData.polling.includes(platform)) { + PQcount++; + } + logger.debug(`减少相关平台处理队列: AIGC=${aigc}, Platform=${platform}, Count=${count}`); + } + + multi.json.numIncrBy(this.initInfoKey, '$.PQtasksALL', -PQcount); + multi.json.numIncrBy(this.initInfoKey, '$.RQtasksALL', PQcount); + await multi.exec(); + } catch (error) { + logger.error('减少平台处理队列任务数失败:', error); + } + } + + // 增加平台等待队列的 等待中任务数 + async addPlatformsWait(aigc, platform, count) { + try { + await redis.json.numIncrBy(this.initInfoKey, `$.platforms.${aigc}:${platform}.WQtasks`, count); + logger.debug(`增加平台等待队列任务数: AIGC=${aigc}, Platform=${platform}, Count=${count}`); + } catch (error) { + logger.error('增加平台等待队列任务数失败:', error); + } + } + + // 减少平台等待队列的 等待中任务数 + async reducePlatformsWait(taskCountMap) { + try { + const multi = redis.multi(); + + for (const [key, count] of taskCountMap.entries()) { + multi.json.numIncrBy(this.initInfoKey, `$.platforms.${key}.WQtasks`, -count); + } + + await multi.exec(); // 等待multi命令执行完成 + } catch (error) { + logger.error('减少平台等待队列任务数失败:', error); + } + } + + // 增加回调队列的 完成任务数 + async addCallbackRQtasks(count) { + try { + await redis.json.numIncrBy(this.initInfoKey, '$.CQtasksALL', count); + logger.debug(`增加回调队列任务数: ${count}`); + } catch (error) { + logger.error('增加回调队列任务数失败:', error); + } + } + + // 减少回调队列任务数 + async reduceCQtasksALL(count) { + try { + await redis.json.numIncrBy(this.initInfoKey, '$.CQtasksALL', -count); + logger.debug(`减少回调队列任务数: ${count}`); + } catch (error) { + logger.error('减少回调队列任务数失败:', error); + } + } + + // 增加错误队列的 完成任务数 + async addEQtaskALL(count) { + try { + await redis.json.numIncrBy(this.initInfoKey, '$.EQtaskALL', count); + logger.debug(`增加错误队列任务数: ${count}`); + } catch (error) { + logger.error('增加错误队列任务数失败:', error); + } + } + + // 减少错误队列任务数 + async reduceEQtaskALL(count) { + try { + await redis.json.numIncrBy(this.initInfoKey, '$.EQtaskALL', -count); + logger.debug(`减少错误队列任务数: ${count}`); + } catch (error) { + logger.error('减少错误队列任务数失败:', error); + } + } + +} + +export default new InitQueues(); diff --git a/任务队列后端/redis/messagePersistence.js b/任务队列后端/redis/messagePersistence.js new file mode 100644 index 0000000..ebcc48c --- /dev/null +++ b/任务队列后端/redis/messagePersistence.js @@ -0,0 +1,123 @@ +import redis from './index.js'; +import initQueue from './initQueue.js'; + +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +class MessagePersistence { + constructor() { + this.pendingMessagesKey = initQueue.pendingMessages; + } + + async savePendingMessage(backendId, message) { + try { + const messageData = { + backendId, + message, + timestamp: Date.now(), + retryCount: 0 + }; + + const messageKey = `${this.pendingMessagesKey}:${backendId}:${Date.now()}`; + await redis.hSet(messageKey, messageData); + await redis.lPush(this.pendingMessagesKey, messageKey); + + logger.debug(`保存待发送消息: backendId=${backendId}, messageKey=${messageKey}`); + return messageKey; + } catch (error) { + logger.error(`保存待发送消息失败: backendId=${backendId}`, error); + throw error; + } + } + + async getPendingMessages(backendId) { + try { + const allMessageKeys = await redis.lRange(this.pendingMessagesKey, 0, -1); + const pendingMessages = []; + + for (const messageKey of allMessageKeys) { + if (messageKey.includes(`:${backendId}:`)) { + const messageData = await redis.hGetAll(messageKey); + if (messageData && messageData.message) { + pendingMessages.push({ + key: messageKey, + backendId: messageData.backendId, + message: messageData.message, + timestamp: parseInt(messageData.timestamp), + retryCount: parseInt(messageData.retryCount || 0) + }); + } + } + } + + logger.debug(`获取待发送消息: backendId=${backendId}, count=${pendingMessages.length}`); + return pendingMessages; + } catch (error) { + logger.error(`获取待发送消息失败: backendId=${backendId}`, error); + return []; + } + } + + async removePendingMessage(messageKey) { + try { + await redis.del(messageKey); + await redis.lRem(this.pendingMessagesKey, 1, messageKey); + logger.debug(`删除已发送消息: messageKey=${messageKey}`); + } catch (error) { + logger.error(`删除待发送消息失败: messageKey=${messageKey}`, error); + } + } + + async incrementRetryCount(messageKey) { + try { + await redis.hIncrBy(messageKey, 'retryCount', 1); + logger.debug(`增加重试次数: messageKey=${messageKey}`); + } catch (error) { + logger.error(`增加重试次数失败: messageKey=${messageKey}`, error); + } + } + + async cleanupOldMessages(maxAge = 7 * 24 * 60 * 60 * 1000) { + try { + const allMessageKeys = await redis.lRange(this.pendingMessagesKey, 0, -1); + const now = Date.now(); + const keysToDelete = []; + + for (const messageKey of allMessageKeys) { + const messageData = await redis.hGetAll(messageKey); + if (messageData && messageData.timestamp) { + const messageAge = now - parseInt(messageData.timestamp); + if (messageAge > maxAge) { + keysToDelete.push(messageKey); + } + } + } + + if (keysToDelete.length > 0) { + const multi = redis.multi(); + for (const key of keysToDelete) { + multi.del(key); + multi.lRem(this.pendingMessagesKey, 1, key); + } + await multi.exec(); + logger.info(`清理过期消息: count=${keysToDelete.length}`); + } + } catch (error) { + logger.error('清理过期消息失败:', error); + } + } +} + +export default new MessagePersistence(); diff --git a/任务队列后端/reset_pqtasks.js b/任务队列后端/reset_pqtasks.js new file mode 100644 index 0000000..4740369 --- /dev/null +++ b/任务队列后端/reset_pqtasks.js @@ -0,0 +1,114 @@ +import redis from './redis/index.js'; +import dotenv from 'dotenv'; +import initQueue from './redis/initQueue.js'; + +dotenv.config(); + +const prefix = process.env.PROJECT_PREFIX || 'default'; +const initInfoKey = `${prefix}:InitInfo`; + +async function clearAllProjectData() { + try { + console.log('正在连接Redis...'); + + // 连接Redis + if (!redis.isOpen) { + await redis.connect(); + console.log('Redis连接成功'); + } + + console.log(`\n开始清除项目 "${prefix}" 的所有Redis数据...`); + + // 1. 获取并显示当前初始化信息 + try { + const initInfoResult = await redis.json.get(initInfoKey, { path: '$' }); + if (initInfoResult) { + console.log('\n当前初始化信息:', JSON.stringify(initInfoResult, null, 2)); + } else { + console.log('\n未找到初始化信息'); + } + } catch (error) { + console.log('获取初始化信息失败（可能不存在）:', error.message); + } + + // 2. 删除所有相关的Redis键 + const keysToDelete = []; + + // 使用SCAN模式删除所有相关键 + const patterns = [ + `${prefix}:*`, // 所有项目前缀的键 + `${initQueue?.prefix || prefix}:*`, // 兼容initQueue的前缀 + ]; + + for (const pattern of patterns) { + console.log(`\n正在搜索模式: ${pattern}`); + let cursor = '0'; + let totalDeleted = 0; + + do { + try { + const result = await redis.scan(cursor, { + MATCH: pattern, + COUNT: 100 + }); + + const newCursor = result.cursor; + const keys = result.keys || []; + + console.log(` 游标: ${newCursor}, 找到键数量: ${keys.length}`); + + if (keys.length > 0) { + // 过滤有效键 + const validKeys = keys.filter(key => { + return typeof key === 'string' && key.trim() !== ''; + }); + + if (validKeys.length > 0) { + await redis.del(...validKeys); + totalDeleted += validKeys.length; + console.log(` 已删除 ${validKeys.length} 个键`); + } + } + + cursor = newCursor; + } catch (error) { + console.error(` 搜索过程中出错:`, error.message); + break; + } + } while (cursor !== '0'); + + console.log(`模式 "${pattern}" 共删除 ${totalDeleted} 个键`); + } + + // 3. 特殊处理：删除等待队列（可能不包含前缀） + try { + const platforms = await redis.json.get(initInfoKey, { path: '$.platforms' }); + if (platforms && platforms[0]) { + for (const [key, platform] of Object.entries(platforms[0])) { + if (platform.waitQueue) { + console.log(`\n删除等待队列: ${platform.waitQueue}`); + await redis.del(platform.waitQueue); + } + } + } + } catch (error) { + console.log('删除等待队列失败:', error.message); + } + + console.log('\n========================================'); + console.log('项目所有Redis数据已清除完成！'); + console.log('========================================'); + + } catch (error) { + console.error('清除Redis数据失败:', error); + } finally { + // 断开Redis连接 + if (redis.isOpen) { + await redis.disconnect(); + console.log('\nRedis连接已关闭'); + } + process.exit(); + } +} + +clearAllProjectData(); diff --git a/任务队列后端/school/api.js b/任务队列后端/school/api.js new file mode 100644 index 0000000..6c3879a --- /dev/null +++ b/任务队列后端/school/api.js @@ -0,0 +1,71 @@ +import service from '../utils/request.js' + +// 检查用户token +export async function checkUsertoken(token) { + // console.log('开始验证token:', token); // 添加开始验证的日志 + + try { + const res = await service.get(`/auth/check/token`,{ + headers: { + Authorization: `Bearer ${token}` + } + }); + + console.log('checkTokenValid:', res); + + if (res.code === '401' || res.success === false) { + console.error('Token is invalid:', res.message); + return false; + } + + // console.log('令牌有效'); + return res; + } catch (error) { + console.error('验证token时发生错误:', error); + console.error('错误详情:', error.message, error.stack); + return false; + } +} + +// 判断余额或免费次数是否充足 +/** + * @param {object} data 预扣费接口参数 + * @param {string} data.platformCode 平台编码 + * @param {number} data.platformId 生成平台 必需 + * @param {string} data.chargeCode 必需 + * @param {number} data.quantity 必需 + * @param {string} data.chargeType 生成类型（0：token，1：图片，3：音频，4：视频，5：3D，6：教学智能体，7：deep_research，8：多模态，9：AI编程，10：智能体开发） 必需 + * @param {number} data.taskType ( 文生：1，图生：2，音效生成：3，参考生视频：4，白膜贴图：5 ) 必需 + * @param {number} data.type 记录类型（0充点 1 次数 2 点数） 必需 + * @param {string} data.preDeductAmount 预扣减金额 必需 + * @returns + */ +export async function checkBalance(data,token) { + // console.log('开始验证余额:', token); + return await service.post(`/billing/judgeBalanceWithAmount`, data, {headers: { 'Authorization': `Bearer ${token}` }}); +} + +// 添加用户充值消费历史记录并扣费 +/** + * @param {object} data judgeBalance接口参数 + * @param {string} data.platformCode 平台编码 必需 + * @param {number} data.platformId 生成平台 ( 音频：1，视频：2，3D：3 ) 必需 + * @param {string} data.taskId 任务ID 必需 + * @param {string} data.title 文件名 必需 + * @param {string} data.chargeCode 可选 + * @param {number} data.quantity 可选 + * @param {string} data.status 任务状态 (0:进行中, 1:成功, 2:失败) 必需 + * @param {string} data.result 生成的文字内容（文字类型时填写） 可选 + * @param {string} data.tokens 消费的token数（文字类型时填写） 可选 + * @param {string} data.fileUrl 文件路径（音视频图片类型时填写） 可选 + * @param {string} data.errorMessage 错误消息（失败时填写） 可选 + * @param {number} data.chargeType 生成类型（ 0：token，1：图片，3：音频，4：视频，5：AI3D ） 必需 + * @param {number} data.taskType ( 文生：1，图生：2，音效生成：3，参考生视频：4 ) 必需 + * @param {number} data.type 记录类型（0充点 1 次数 2 点数） 必需 + * @param {string} data.fileType 文件类型 可选 + * @param {number} data.actualAmount 实际消费金额 必需 + * @returns + */ +export function addConsumptionHistory(data,token) { + return service.post(`/billing/callbackWithAmount`, data, {headers: { 'Authorization': `Bearer ${token}` }}); +} diff --git a/任务队列后端/upload/index.js b/任务队列后端/upload/index.js new file mode 100644 index 0000000..caf3e68 --- /dev/null +++ b/任务队列后端/upload/index.js @@ -0,0 +1,109 @@ +import { Router } from 'express'; +const router = Router(); +import { statSync, createReadStream, existsSync, mkdirSync, unlinkSync } from 'fs'; +import { join, dirname, extname } from 'path'; +import { fileURLToPath } from 'url'; // 添加这行 +import multer, { diskStorage } from 'multer'; +import axios from 'axios'; +import FormData from 'form-data'; + +const apikey = '3c20cd6c85514d1c86d55a5d3bcd53b7' + +/** + * 上传文件到外部平台 + * @param {string} filePath + * @returns {Promise} 文件名或null + */ +async function send_file(filePath) { + try { + const url = 'https://www.runninghub.cn/task/openapi/upload'; + + // 使用 form-data 库创建表单数据 + const form = new FormData(); + const fileStats = statSync(filePath); + console.log(`文件大小: ${fileStats.size} bytes`); + console.log(`文件路径: ${filePath}`); + + form.append('file', createReadStream(filePath)); + form.append('apiKey', apikey); + form.append('fileType', 'input'); + + const response = await axios.post(url, form, { + headers: { + ...form.getHeaders(), // 重要：使用 form-data 提供的头部信息 + "Host": "www.runninghub.cn", + }, + timeout: 30000, + }); + + console.log('****************************11*******************************'); + console.log(response.data); + + if (response.data.code === 0 && response.data.msg === "success") { + console.log("File URL:", response.data.data.fileName); + return response.data.data.fileName; + } + else if (response.data.code === 301 && response.data.msg === "PARAMS_INVALID") { + console.log("请重新上传文件"); + return false; + } + else { + console.log("请重新上传文件"); + return false; + } + } catch (error) { + console.error('文件上传失败:', error); + return null; + } +} + +// 使用 fileURLToPath 获取目录路径 +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// 确保文件目录存在 +const uploadDir = join(__dirname, '../uploads'); +if (!existsSync(uploadDir)) { + mkdirSync(uploadDir, { recursive: true }); +} + +// 配置multer存储 +const storage = diskStorage({ + destination: function (req, file, cb) { + cb(null, uploadDir); + }, + filename: function (req, file, cb) { + const randomStr = Math.random().toString(36).substring(2, 8); + cb(null, `${Date.now()}-${randomStr}${extname(file.originalname)}`); + } +}); + +const upload = multer({ storage: storage }); + +// 文件上传接口 +router.post('/upload', upload.single('file'), async (req, res) => { + if (!req.file) { + return res.status(400).json({ error: 'No file uploaded' }); + } + + try { + // 调用外部平台接口 + const result = await send_file(req.file.path); + // 删除本地文件 + unlinkSync(req.file.path); + + if (result) { + res.json({ + success: true, + url: result + }); + } else { + res.status(500).json({ error: 'File upload to external service failed' }); + } + } catch (error) { + console.error('处理失败:', error); + res.status(500).json({ error: '文件处理失败' }); + } +}); + +export default router; \ No newline at end of file diff --git a/任务队列后端/uploads/1772596380109-3h5x8l.mp4 b/任务队列后端/uploads/1772596380109-3h5x8l.mp4 new file mode 100644 index 0000000..cdb428d Binary files /dev/null and b/任务队列后端/uploads/1772596380109-3h5x8l.mp4 differ diff --git a/任务队列后端/uploads/1772596407045-0jxqwl.png b/任务队列后端/uploads/1772596407045-0jxqwl.png new file mode 100644 index 0000000..0cf22e2 Binary files /dev/null and b/任务队列后端/uploads/1772596407045-0jxqwl.png differ diff --git a/任务队列后端/uploads/1772596407050-b16aem.png b/任务队列后端/uploads/1772596407050-b16aem.png new file mode 100644 index 0000000..e04b155 Binary files /dev/null and b/任务队列后端/uploads/1772596407050-b16aem.png differ diff --git a/任务队列后端/uploads/1772596441891-xiizrj.png b/任务队列后端/uploads/1772596441891-xiizrj.png new file mode 100644 index 0000000..0cf22e2 Binary files /dev/null and b/任务队列后端/uploads/1772596441891-xiizrj.png differ diff --git a/任务队列后端/uploads/1772596441896-8mi7vj.png b/任务队列后端/uploads/1772596441896-8mi7vj.png new file mode 100644 index 0000000..7733939 Binary files /dev/null and b/任务队列后端/uploads/1772596441896-8mi7vj.png differ diff --git a/任务队列后端/uploads/1772596447670-98bomn.png b/任务队列后端/uploads/1772596447670-98bomn.png new file mode 100644 index 0000000..0cf22e2 Binary files /dev/null and b/任务队列后端/uploads/1772596447670-98bomn.png differ diff --git a/任务队列后端/uploads/1772596447675-veyky4.png b/任务队列后端/uploads/1772596447675-veyky4.png new file mode 100644 index 0000000..82d8c96 Binary files /dev/null and b/任务队列后端/uploads/1772596447675-veyky4.png differ diff --git a/任务队列后端/uploads/1772596701505-7fxnii.png b/任务队列后端/uploads/1772596701505-7fxnii.png new file mode 100644 index 0000000..0cf22e2 Binary files /dev/null and b/任务队列后端/uploads/1772596701505-7fxnii.png differ diff --git a/任务队列后端/uploads/1772596701510-vjq9dh.png b/任务队列后端/uploads/1772596701510-vjq9dh.png new file mode 100644 index 0000000..5bfa27d Binary files /dev/null and b/任务队列后端/uploads/1772596701510-vjq9dh.png differ diff --git a/任务队列后端/uploads/1772596774749-rzid65.png b/任务队列后端/uploads/1772596774749-rzid65.png new file mode 100644 index 0000000..3a0edb5 Binary files /dev/null and b/任务队列后端/uploads/1772596774749-rzid65.png differ diff --git a/任务队列后端/uploads/1772596793101-lb8mve.png b/任务队列后端/uploads/1772596793101-lb8mve.png new file mode 100644 index 0000000..3a0edb5 Binary files /dev/null and b/任务队列后端/uploads/1772596793101-lb8mve.png differ diff --git a/任务队列后端/uploads/1772596810158-9mlgkv.png b/任务队列后端/uploads/1772596810158-9mlgkv.png new file mode 100644 index 0000000..3a0edb5 Binary files /dev/null and b/任务队列后端/uploads/1772596810158-9mlgkv.png differ diff --git a/任务队列后端/uploads/1772596832609-sjndgb.png b/任务队列后端/uploads/1772596832609-sjndgb.png new file mode 100644 index 0000000..3a0edb5 Binary files /dev/null and b/任务队列后端/uploads/1772596832609-sjndgb.png differ diff --git a/任务队列后端/uploads/1772596866272-z7ytxb.png b/任务队列后端/uploads/1772596866272-z7ytxb.png new file mode 100644 index 0000000..3a0edb5 Binary files /dev/null and b/任务队列后端/uploads/1772596866272-z7ytxb.png differ diff --git a/任务队列后端/uploads/1772596884988-5mjemd.png b/任务队列后端/uploads/1772596884988-5mjemd.png new file mode 100644 index 0000000..34c621e Binary files /dev/null and b/任务队列后端/uploads/1772596884988-5mjemd.png differ diff --git a/任务队列后端/utils/capacityGuard.js b/任务队列后端/utils/capacityGuard.js new file mode 100644 index 0000000..1fcb767 --- /dev/null +++ b/任务队列后端/utils/capacityGuard.js @@ -0,0 +1,33 @@ +class CapacityGuard { + constructor() { + this.updateLock = false; + this.pendingUpdates = []; + } + + async acquireLock() { + while (this.updateLock) { + await new Promise(resolve => setTimeout(resolve, 10)); + } + this.updateLock = true; + } + + releaseLock() { + this.updateLock = false; + + if (this.pendingUpdates.length > 0) { + const nextUpdate = this.pendingUpdates.shift(); + nextUpdate(); + } + } + + async executeWithLock(fn) { + await this.acquireLock(); + try { + return await fn(); + } finally { + this.releaseLock(); + } + } +} + +export default new CapacityGuard(); diff --git a/任务队列后端/utils/mdWebSocketServer.js b/任务队列后端/utils/mdWebSocketServer.js new file mode 100644 index 0000000..fcbc46a --- /dev/null +++ b/任务队列后端/utils/mdWebSocketServer.js @@ -0,0 +1,132 @@ +import { WebSocketServer as WSServer } from 'ws'; +import dotenv from 'dotenv'; + +dotenv.config(); + +class MDWebSocketServer { + constructor() { + this.wss = null; + this.connectedClients = new Map(); + this.currentJwtToken = null; + this.currentCapacity = { internal: 0, external: 0 }; + this.instances = new Map(); + this.port = process.env.MESSAGE_DISPATCHER_WS_PORT || 8087; + } + + async init() { + return new Promise((resolve, reject) => { + this.wss = new WSServer({ port: this.port }); + + this.wss.on('listening', () => { + console.log(`[MDWebSocketServer] WebSocket 服务已启动，端口: ${this.port}`); + resolve(); + }); + + this.wss.on('connection', (ws) => { + console.log('[MDWebSocketServer] 新的 WebSocket 连接已建立'); + + const clientId = Date.now().toString(); + this.connectedClients.set(clientId, ws); + + ws.on('message', (data) => { + this.handleMessage(ws, data); + }); + + ws.on('close', () => { + console.log('[MDWebSocketServer] WebSocket 连接已关闭'); + this.connectedClients.delete(clientId); + }); + + ws.on('error', (error) => { + console.error('[MDWebSocketServer] WebSocket 连接错误:', error); + this.connectedClients.delete(clientId); + }); + }); + + this.wss.on('error', (error) => { + console.error('[MDWebSocketServer] WebSocket 服务错误:', error); + reject(error); + }); + }); + } + + handleMessage(ws, data) { + try { + const message = JSON.parse(data.toString()); + console.log(`[MDWebSocketServer] 收到消息: ${message.type}`); + + switch (message.type) { + case 'JWT_UPDATE': + this.handleJwtUpdate(message.data); + break; + case 'CAPACITY_UPDATE': + this.handleCapacityUpdate(message.data); + break; + case 'INSTANCE_ONLINE': + this.handleInstanceOnline(message.data); + break; + case 'INSTANCE_OFFLINE': + this.handleInstanceOffline(message.data); + break; + case 'HEARTBEAT': + this.handleHeartbeat(message.data, ws); + break; + default: + console.log('[MDWebSocketServer] 未知消息类型:', message.type); + } + } catch (error) { + console.error('[MDWebSocketServer] 解析消息失败:', error); + } + } + + handleJwtUpdate(data) { + this.currentJwtToken = data.token; + console.log('[MDWebSocketServer] JWT Token 已更新'); + } + + handleCapacityUpdate(data) { + if (data.summary) { + this.currentCapacity.internal = data.summary.onlineInstances - data.summary.busyInstances; + console.log(`[MDWebSocketServer] 算力状态已更新: 内部可用 = ${this.currentCapacity.internal}`); + } + } + + handleInstanceOnline(data) { + this.instances.set(data.instanceId, { ...data, status: 'online' }); + console.log(`[MDWebSocketServer] 实例上线: ${data.instanceId}`); + } + + handleInstanceOffline(data) { + this.instances.set(data.instanceId, { ...data, status: 'offline' }); + console.log(`[MDWebSocketServer] 实例下线: ${data.instanceId}`); + } + + handleHeartbeat(data, ws) { + ws.send(JSON.stringify({ + type: 'HEARTBEAT_ACK', + data: { timestamp: new Date().toISOString() } + })); + } + + getJwtToken() { + return this.currentJwtToken; + } + + getInternalCapacity() { + return this.currentCapacity.internal; + } + + getExternalCapacity() { + return parseInt(process.env.EXTERNAL_CAPACITY_MAX) || 10; + } + + getInstances() { + return Array.from(this.instances.values()); + } + + hasConnectedClients() { + return this.connectedClients.size > 0; + } +} + +export default new MDWebSocketServer(); diff --git a/任务队列后端/utils/request.js b/任务队列后端/utils/request.js new file mode 100644 index 0000000..924360c --- /dev/null +++ b/任务队列后端/utils/request.js @@ -0,0 +1,22 @@ +import axios from 'axios'; +import dotenv from 'dotenv' + +// 配置 dotenv 加载环境变量 +dotenv.config() + +console.log('BACKEND_API_URL:', process.env.BACKEND_API_URL); +// 创建axios实例 +const service = axios.create({ + baseURL: process.env.BACKEND_API_URL, + timeout: 50000, // 请求超时时间 + headers: { 'Content-Type': 'application/json' } +}) + +// response 拦截器 +service.interceptors.response.use( + (response) => { + return response.data + } +) + +export default service; \ No newline at end of file diff --git a/任务队列后端/utils/taskDistributor.js b/任务队列后端/utils/taskDistributor.js new file mode 100644 index 0000000..5c3f1aa --- /dev/null +++ b/任务队列后端/utils/taskDistributor.js @@ -0,0 +1,22 @@ +import dotenv from 'dotenv'; + +dotenv.config(); + +async function getExternalCapacityFromConfig() { + return parseInt(process.env.EXTERNAL_CAPACITY_MAX) || 10; +} + +export async function distributeTasks(tasks, mdWebSocketServer) { + const internalCapacity = mdWebSocketServer.getInternalCapacity(); + const externalCapacity = await getExternalCapacityFromConfig(); + + console.log(`[TaskDistributor] 任务分流 - 内部容量: ${internalCapacity}, 外部容量: ${externalCapacity}, 待分发任务数: ${tasks.length}`); + + const internalTasks = tasks.slice(0, internalCapacity); + const externalTasks = tasks.slice(internalCapacity, internalCapacity + externalCapacity); + const remainingTasks = tasks.slice(internalCapacity + externalCapacity); + + console.log(`[TaskDistributor] 分流结果 - 内部: ${internalTasks.length}, 外部: ${externalTasks.length}, 剩余: ${remainingTasks.length}`); + + return { internalTasks, externalTasks, remainingTasks }; +} diff --git a/任务队列后端/webSocket.js b/任务队列后端/webSocket.js new file mode 100644 index 0000000..f21ef65 --- /dev/null +++ b/任务队列后端/webSocket.js @@ -0,0 +1,332 @@ +import dotenv from 'dotenv' +import WebSocket, { WebSocketServer } from 'ws' +import { Worker } from 'worker_threads' +import { checkUsertoken } from './school/api.js' +import redis from './redis/index.js' +import initQueue from './redis/initQueue.js' +import messagePersistence from './redis/messagePersistence.js' +import code from './config/code.json' with { type: 'json' } + +// 配置 dotenv 加载环境变量 +dotenv.config() + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +let wss = null; +const workers = []; + +// 初始化函数 +async function initialize() { + logger.info('***************初始化队列开始***************'); + try { + // 确保 Redis 连接后再初始化队列 + if (!redis.isOpen) { + await redis.connect(); + logger.info('Redis 连接成功'); + } + + await initQueue.init(); + logger.info('***************初始化队列完成***************'); + + // 创建 WebSocket 服务器 + createWebSocketServer(); + + // 启动定期清理过期消息的任务（每天执行一次） + setInterval(async () => { + try { + await messagePersistence.cleanupOldMessages(2 * 24 * 60 * 60 * 1000); // 清理7天前的消息 + } catch (error) { + logger.error('定期清理过期消息失败:', error); + } + }, 24 * 60 * 60 * 1000); // 每24小时执行一次 + + } catch (err) { + logger.error('初始化失败:', err); + process.exit(1); // 初始化失败退出进程，让进程管理器重启 + } +} + +const socketMap = new Map(); + +// 创建并管理worker线程 +function createWorker(scriptPath) { + const worker = new Worker(scriptPath); + worker.setMaxListeners(20); + worker.on('error', (error) => { + logger.error(`Worker ${scriptPath} 错误:`, error); + }); + worker.on('exit', (code) => { + if (code !== 0) { + logger.error(`Worker ${scriptPath} 异常退出，退出码: ${code}`); + // 可以考虑重启worker + } + }); + workers.push(worker); + return worker; +} + +// 初始化所有worker线程 +const assessment = createWorker('./worker_threads/assessment/assessment.js'); +const wait = createWorker('./worker_threads/wait/waiting.js'); +const polling = createWorker('./worker_threads/process/process.js'); +const result = createWorker('./worker_threads/result/result.js'); +const callback_result = createWorker('./worker_threads/callback_result/result.js'); +const error = createWorker('./worker_threads/error/error.js'); + +// 发送消息给客户端的工具函数 +async function sendMessageToClient(id, message, close = false, closeCode = 1000, closeReason = '') { + let socket; + + // 尝试通过id查找socket，id可能是taskId或backendId + if (typeof id === 'string' && id) { + socket = socketMap.get(id); + } + + if (socket && socket.readyState === WebSocket.OPEN && message) { + try { + socket.send(message); + const messagePreview = typeof message === 'string' ? message.slice(0, 50) : JSON.stringify(message).slice(0, 50); + logger.debug(`成功发送消息到客户端，id: ${id}, 消息: ${messagePreview}...`); + if (close) { + socket.close(closeCode, closeReason); + } + return true; + } catch (error) { + logger.error(`发送消息给客户端失败，id: ${id}`, error); + return false; + } + } else { + if (!message) { + logger.debug(`消息为空，无法发送，id: ${id}`); + return false; + } else { + logger.debug(`未找到目标客户端或连接已关闭，保存消息到待发送队列，id: ${id}`); + try { + await messagePersistence.savePendingMessage(id, message); + logger.info(`消息已保存到待发送队列，等待重试: backendId=${id}`); + return false; + } catch (error) { + logger.error(`保存待发送消息失败: backendId=${id}`, error); + return false; + } + } + } +} + +// 创建 WebSocket 服务器函数 +function createWebSocketServer() { + wss = new WebSocketServer({ + port: process.env.WS_PORT || 8086, + verifyClient: async (info, callback) => { + try { + const urlParams = new URLSearchParams(info.req.url.split('?')[1]); + const token = urlParams.get('token'); + const id = urlParams.get('id'); + + if (!token) { + logger.info('缺少令牌'); + callback(false, 401, '缺少令牌'); + return; + } else if (token !== process.env.TOKEN_SECRET){ + logger.info('验证后端失败'); + callback(false, 401, 'Token is invalid'); + return; + } + + info.req.id = id; + logger.info(`用户ID: token 验证成功`); + callback(true); + } catch (error) { + logger.error('验证后端失败:', error); + callback(false, 401, 'Token is invalid'); + } + } + }); + + // 日志显示WebSocket服务器端口 + logger.info(`WebSocket server is running on port: ${process.env.WS_PORT || 8082}`); + + // 添加服务器错误处理 + wss.on('error', (error) => { + logger.error('WebSocket服务器错误:', error); + }); + + // 当有客户端连接时触发 + wss.on('connection', async (socket, req) => { + const id = req.id; + logger.info(`${id}号后端 连接成功`); + socketMap.set(id, socket); + // 连接成功后，只发送一条请求taskId的消息 + socket.send('please give me tasks'); + + // 重试发送之前未发送的消息 + try { + const pendingMessages = await messagePersistence.getPendingMessages(id); + if (pendingMessages.length > 0) { + logger.info(`${id}号后端 发现 ${pendingMessages.length} 条待发送消息，开始重试发送`); + + for (const pendingMsg of pendingMessages) { + try { + socket.send(pendingMsg.message); + await messagePersistence.removePendingMessage(pendingMsg.key); + logger.debug(`成功重试发送消息: backendId=${id}, messageKey=${pendingMsg.key}`); + } catch (error) { + logger.error(`重试发送消息失败: backendId=${id}, messageKey=${pendingMsg.key}`, error); + await messagePersistence.incrementRetryCount(pendingMsg.key); + } + } + + logger.info(`${id}号后端 待发送消息重试完成`); + } + } catch (error) { + logger.error(`获取或发送待发送消息失败: backendId=${id}`, error); + } + + // 处理收到的消息 + socket.on('message', (message) => { + const messageStr = typeof message === 'string' ? message : message.toString(); + // 首先检查是否为心跳消息 + if (messageStr === 'ping') { + socket.send('pong'); // 回复心跳 + return; + } + + try { + // 只检查前面100个字符是否包含 `"type": "generate"`，提高大消息处理性能 + const prefix = messageStr.slice(0, 50); + if (prefix.includes('"type":"generate"') || prefix.includes("'type':'generate'")) { + // 在此处添加处理消息的逻辑 + assessment.postMessage({ + type: 'submit', + data: messageStr + }); + } else { + // 记录日志，不关闭连接 + logger.debug(`收到未知消息类型: ${prefix}`); + } + } catch (e) { + logger.error('处理消息出错:', e); + // 发送错误消息，不关闭连接 + socket.send(JSON.stringify({ + error: '处理消息出错', + details: e.message + })); + } + }); + + // 定期发送心跳 + const heartbeatInterval = setInterval(() => { + if (socket.readyState === WebSocket.OPEN) { + socket.send('ping'); + logger.debug(`向 ${id} 号后端发送心跳`); + } + }, 30000); // 每30秒发送一次心跳 + + // 处理连接关闭 + socket.on('close', (code, reason) => { + // 清理心跳定时器 + clearInterval(heartbeatInterval); + logger.info(`${id}号后端 连接关闭，关闭码: ${code}，原因: ${reason}`); + }); + + // 处理连接错误 + socket.on('error', (error) => { + logger.error(`${id}号后端 连接错误:`, error); + // 不关闭连接，尝试继续通信 + }); + }); + + // 任务检验的工作线程响应处理 + assessment.on('message', async (message) => { + logger.debug(`收到assessment worker消息: ${JSON.stringify(message)}`); + if (message.type === 'AssessmentSuccess') { + await sendMessageToClient(message.backendId, code.SUCCESS[message.type]); + } else { + await sendMessageToClient(message.backendId, code.ERROR[message.type], false, 4401, code.ERROR[message.type]); + } + }); + + // 获取结果线程响应处理 + result.on('message', async (message) => { + logger.debug(`收到result worker消息: ${JSON.stringify(message)}`); + if (message.type === 'success') { + await sendMessageToClient(message.backendId, message.message, false, 1000, 'success'); + } else { + await sendMessageToClient(message.backendId, '获取结果失败，可在历史记录区刷新查看结果', false, 4401, code.ERROR[message.type]); + } + }); + + // 获取回调结果线程响应处理 + callback_result.on('message', async (message) => { + logger.debug(`收到callback_result worker消息: ${JSON.stringify(message)}`); + if (message.type === 'success') { + await sendMessageToClient(message.backendId, message.message, false, 1000, 'success'); + } else { + await sendMessageToClient(message.backendId, '获取结果失败，可在历史记录区刷新查看结果', false, 4401); + } + }); + + error.on('message', async (message) => { + logger.debug(`收到error worker消息: ${JSON.stringify(message)}`); + await sendMessageToClient(message.backendId, message.message, false, 4402, 'false'); + }); +} + +// 优雅关闭机制 +function gracefulShutdown() { + logger.info('开始优雅关闭...'); + + // 关闭WebSocket服务器，拒绝新连接 + if (wss) { + wss.close(() => { + logger.info('WebSocket服务器已关闭'); + }); + + // 关闭所有现有连接 + wss.clients.forEach((client) => { + client.close(1001, '服务器正在关闭'); + }); + } + + // 终止所有worker线程 + workers.forEach((worker, index) => { + logger.info(`终止worker线程 ${index}`); + worker.terminate(); + }); + + // 关闭Redis连接 + if (redis.isOpen) { + redis.disconnect() + .then(() => { + logger.info('Redis连接已关闭'); + process.exit(0); + }) + .catch((error) => { + logger.error('关闭Redis连接失败:', error); + process.exit(1); + }); + } else { + process.exit(0); + } +} + +// 监听终止信号 +process.on('SIGINT', gracefulShutdown); +process.on('SIGTERM', gracefulShutdown); + +// 启动服务器 +initialize(); diff --git a/任务队列后端/worker_threads/assessment/PreproTask.js b/任务队列后端/worker_threads/assessment/PreproTask.js new file mode 100644 index 0000000..02ad1aa --- /dev/null +++ b/任务队列后端/worker_threads/assessment/PreproTask.js @@ -0,0 +1,177 @@ +import { parentPort } from 'worker_threads' +import initQueue from '../../redis/initQueue.js' +import redis from '../../redis/index.js' +import dotenv from 'dotenv' +// 配置 dotenv 加载环境变量 +dotenv.config() + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +console.log('***********************************预处理线程启动成功**********************************************') +let id + +// 参数检查函数 +function validateTaskParams(task) { + // 定义必填参数列表 + const requiredParams = [ + 'taskId', 'platform', 'payload' + ]; + + // 检查task是否存在 + if (!task) { + return { valid: false, message: '缺少必填参数: task' }; + } + + // 检查每个必填参数 + for (const param of requiredParams) { + if (param === 'taskType') { + // taskType可以是0，所以需要特殊处理 + if (task[param] === undefined || task[param] === null) { + return { valid: false, message: `缺少必填参数: task.${param}` }; + } + } else { + if (!task[param]) { + return { valid: false, message: `缺少必填参数: task.${param}` }; + } + } + } + + return { valid: true, message: '参数检查通过' }; +} + +// 处理任务信息 +function handleTask(data,backendId) { + // console.log('data:', data, backendId); + const task = { + backendId: backendId, + AIGC: process.env.PROJECT_PREFIX, // AIGC名称 { digitalHuman（数字人） } + platform: data.platform, + taskId: data.taskId, + payload: data.payload, // 任务参数 + workflowId: data.workflowId? data.workflowId : '', // 工作流ID + status: 'pending', + resultData: null + } + return task +} + +// 提交任务 +async function storeTask(task) { + const waitName = initQueue.toQueue(task.AIGC, task.platform, 'wait') // 判断任务所属队列 + // 将任务存储到 Hash 中，便于通过 任务ID（taskId） 查询，使用项目前缀 + const taskKey = `${initQueue.prefix}:task:${task.taskId}`; + + const multi = redis.multi() + // 1. 将任务存储到 Hash 中，使用键值对形式 + multi.hSet(taskKey, 'taskId', task.taskId); + // console.log('taskKey:', taskKey); + multi.hSet(taskKey, 'payload', JSON.stringify(task.payload)); + multi.hSet(taskKey, 'backendId', task.backendId); + multi.hSet(taskKey, 'AIGC', task.AIGC); + multi.hSet(taskKey, 'platform', task.platform); + multi.hSet(taskKey, 'status', task.status); + // 存储workflowId + multi.hSet(taskKey, 'workflowId', task.workflowId || ''); + + // 2. 将任务ID添加到处理队列中（List结构） + multi.rPush(waitName, task.taskId); + + // 3. 设置任务的过期时间为2小时（7200秒） + multi.expire(taskKey, 7200); + + await multi.exec(); + + // 增加平台相关信息等待队列的任务数 + initQueue.addPlatformsWait(task.AIGC, task.platform, 1) + logger.info(`任务已加入排队队列：${task.taskId}，并设置了2小时过期时间`); +} + +// 处理任务数据 +async function pre_deducted_fee (task, backendId) { + // console.log('task:', task); + // 调用参数检查函数 + const validationResult = validateTaskParams(task); + if (!validationResult.valid) { + console.error('任务参数检查失败:', validationResult.message); + parentPort.postMessage({ + type: 'error', + id: id, + backendId: backendId, + data: validationResult.message + }); + return; + } + + const taskinfo = handleTask(task, backendId); + + await storeTask(taskinfo); // 使用新的存储方法 +} + +// 启动处理循环 +function startProcessingLoop() { + setInterval(async () => { + const queues = await redis.lPop(`${initQueue.prefix}:assessment:${id}`, 100) + if (queues) { + // console.log('队列非空，开始提交任务...'); + let tasks = []; + + // 处理返回值：可能是字符串或数组 + if (Array.isArray(queues)) { + // 如果是数组，直接使用 + tasks = queues; + } else if (typeof queues === 'string') { + // 如果是字符串，放入数组中 + tasks = [queues]; + } + + const promises = tasks.map(async (task) => { + let taskObj; + try { + taskObj = JSON.parse(task); + // 检查taskObj.data是否存在 + // console.log('taskObj:', taskObj); + if (!taskObj || !taskObj.data) { + throw new Error('无效的任务数据格式'); + } + await pre_deducted_fee(taskObj.data, taskObj.backendId); + } catch (error) { + console.error('单个任务处理失败:', error); + // 继续处理其他任务 + parentPort.postMessage({ + type: 'error', + id: id, + backendId: taskObj.backendId, + data: '任务处理失败，请稍后再试。' + }); + } + }); + await Promise.all(promises); + } else { + await new Promise(resolve => setTimeout(resolve, 10000)); + } + }, 500); +} + +parentPort.on('message', async (message) => { + id = message.id + parentPort.postMessage({ + type: 'ok', + id: id + }); + // 启动处理循环 + startProcessingLoop(); +}) diff --git a/任务队列后端/worker_threads/assessment/assessment.js b/任务队列后端/worker_threads/assessment/assessment.js new file mode 100644 index 0000000..80f2b99 --- /dev/null +++ b/任务队列后端/worker_threads/assessment/assessment.js @@ -0,0 +1,73 @@ +import { parentPort, Worker } from 'worker_threads' +import redis from '../../redis/index.js' +import initQueue from '../../redis/initQueue.js' + +class QueryThreadPool { + constructor(size = 1) { + this.size = size; // 线程池大小，默认为6 + this.workers = []; // 存储worker对象的数组 + this.initWorkers(); // 初始化worker线程 + } + + initWorkers() { + // // console.log('初始化线程池') + for (let i = 0; i < this.size; i++) { + const worker = new Worker('./worker_threads/assessment/PreproTask.js') + worker.postMessage({type: 'once',id: i}); + // 为每个 worker 添加事件监听器 + worker.on('message', (message) => { + if (message.type === 'ok'){ + // console.log(`Worker ${message.id} 已准备就绪`); + } else{ + parentPort.postMessage(message) + this.workers[message.id].cont -= 1 + } + }) + + this.workers.push( + { + worker, + cont: 0, + } + ) + } + } + + // 将任务发送给任务数最少的线程 + executeTask(message) { + // // console.log('分配任务给各线程') + + let minWorkerIndex = 0; + for (let i = 1; i < this.workers.length; i++) { + if (this.workers[i].cont < this.workers[minWorkerIndex].cont) { + minWorkerIndex = i; + } + } + + // 将任务分配给任务数最少的worker + const minWorker = this.workers[minWorkerIndex]; + // 将message转换为JSON字符串后再存储到Redis，因为Redis的rPush命令要求参数必须是字符串或Buffer类型 + redis.rPush(`${initQueue.prefix}:assessment:${minWorkerIndex}`, message) // message: 前端完全返回 + minWorker.cont += 1 + // // console.log('分配给线程：', minWorkerIndex) + } + // 清理资源 + terminate() { + this.workers.forEach((worker) => { + worker.worker.terminate(); + }); + } +} + +// 创建线程池实例 +const threadPool = new QueryThreadPool(3); // 3个线程的线程池 + +// 监听主线程消息 +parentPort.on('message', async (message) => { + // // console.log('接收到主线程消息：', message) + if (message.type === 'submit') { + // 提交任务给提交线程池 + // // console.log('提交任务给线程池'); + threadPool.executeTask(message.data); // 提交任务 + } +}) \ No newline at end of file diff --git a/任务队列后端/worker_threads/callback_result/RecordThreadPool.js b/任务队列后端/worker_threads/callback_result/RecordThreadPool.js new file mode 100644 index 0000000..0bf723b --- /dev/null +++ b/任务队列后端/worker_threads/callback_result/RecordThreadPool.js @@ -0,0 +1,87 @@ +import { Worker } from 'worker_threads'; + +// 创建固定大小的线程池（3个线程） +class RecordThreadPool { + constructor(size = 1) { + this.size = size; // 线程池大小，默认为6 + this.workers = []; // 存储worker对象的数组 + this.taskQueue = []; // 任务队列，存储等待处理的任务 + this.initWorkers(); // 初始化worker线程 + } + + initWorkers() { + for (let i = 0; i < this.size; i++) { + const worker = new Worker(new URL('./recordTask.js', import.meta.url)); // 确保路径正确 + this.workers.push({ + worker, // worker对象 + busy: false // 是否正在处理任务的标志 + }); + } + } + + async executeTask(taskBatch) { + return new Promise((resolve, reject) => { + // 寻找空闲的worker + const availableWorker = this.workers.find(w => !w.busy); + + if (availableWorker) { + // 有空闲worker，直接执行任务 + this.runTaskOnWorker(availableWorker, taskBatch, resolve, reject); + } else { + // 没有空闲worker，将任务加入队列 + this.taskQueue.push({ taskBatch, resolve, reject }); + } + }); + } + + runTaskOnWorker(workerObj, taskBatch, resolve, reject) { + workerObj.busy = true; + + workerObj.worker.once('message', (result) => { + workerObj.busy = false; + resolve(result); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.once('error', (error) => { + workerObj.busy = false; + reject(error); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.postMessage(taskBatch); + } + + processQueuedTasks() { + if (this.taskQueue.length > 0) { + const availableWorker = this.workers.find(w => !w.busy); + if (availableWorker) { + const queuedTask = this.taskQueue.shift(); + this.runTaskOnWorker( + availableWorker, + queuedTask.taskBatch, + queuedTask.resolve, + queuedTask.reject + ); + } + } + } + + async executeAllTasks(taskBatches) { + const promises = taskBatches.map(batch => this.executeTask(batch)); + return Promise.all(promises); + } + + // 清理资源 + terminate() { + this.workers.forEach(({ worker }) => { + worker.terminate(); + }); + } +} + +export default new RecordThreadPool(); \ No newline at end of file diff --git a/任务队列后端/worker_threads/callback_result/recordTask.js b/任务队列后端/worker_threads/callback_result/recordTask.js new file mode 100644 index 0000000..5791655 --- /dev/null +++ b/任务队列后端/worker_threads/callback_result/recordTask.js @@ -0,0 +1,86 @@ +// recordTask.js +import { parentPort } from 'worker_threads' +import { record } from '../../outside/record.js' +import redis from '../../redis/index.js' +import initQueue from '../../redis/initQueue.js' + +// 从Redis获取完整的任务信息 +async function getTaskInfo(taskId) { + try { + // 从哈希存储获取完整任务信息，使用项目前缀 + const taskInfo = await redis.hGetAll(`${initQueue.prefix}:task:${taskId}`); + + if (taskInfo && taskInfo.taskId) { + // 解析JSON格式的字段 + taskInfo.payload = JSON.parse(taskInfo.payload) + taskInfo.info = JSON.parse(taskInfo.info) + return taskInfo + } + + return null; + } catch (error) { + console.error(`获取任务信息失败: ${taskId}`, error); + return null; + } +} + +async function recordTask(taskIds) { + const recordTasks = [] + const taskCountMap = new Map() + + for (const taskId of taskIds) { + // 对每个taskId获取完整任务信息，然后执行record函数 + const taskPromise = (async () => { + const task = await getTaskInfo(taskId); + if (task) { + // 记录任务信息，并统计需要减少的平台任务数 + await record(task); + + // 统计需要减少的平台任务数 + const key = `${task.info.AIGC}:${task.info.platform}`; + console.log('key:', key); + if(taskCountMap.has(key)){ + taskCountMap.set(key, taskCountMap.get(key) + 1); + } else { + taskCountMap.set(key, 1); + } + + return task; + } else { + console.error(`任务信息不存在: ${taskId}`); + return null; + } + })(); + + recordTasks.push(taskPromise); + } + + try { + await Promise.all(recordTasks); + + // 任务记录完成后，减少平台处理队列任务数 + if (taskCountMap.size > 0) { + await initQueue.reducePlatformsProcess(taskCountMap); + } + } catch (error) { + console.error('Error:', error); + throw error; // 重新抛出错误以便上层处理 + } +} + +parentPort.on('message', async (message) => { + try { + // 处理processTasks类型的消息 + if (message && typeof message === 'object' && message.type === 'processTasks' && message.taskIds && Array.isArray(message.taskIds)) { + await recordTask(message.taskIds); + parentPort.postMessage({ status: 'completed' }); + } + // 兼容处理直接发送的任务ID数组 + else if (Array.isArray(message)) { + await recordTask(message); + parentPort.postMessage({ status: 'completed' }); + } + } catch (error) { + parentPort.postMessage({ status: 'error', error: error.message }); + } +}) diff --git a/任务队列后端/worker_threads/callback_result/recordWorkerManager.js b/任务队列后端/worker_threads/callback_result/recordWorkerManager.js new file mode 100644 index 0000000..91a18f0 --- /dev/null +++ b/任务队列后端/worker_threads/callback_result/recordWorkerManager.js @@ -0,0 +1,45 @@ +// recordWorkerManager.js +import { parentPort } from 'worker_threads'; +import RecordThreadPool from './RecordThreadPool.js'; + +// 在当前 Worker 中初始化线程池 +const threadPool = RecordThreadPool; // 3个线程的线程池 + +// 处理任务的函数 +async function processTasks(tasks) { + // 每100个任务为一组 + const batchSize = 100; + const batches = []; + + // 将任务数据按批次分割 + for (let i = 0; i < tasks.length; i += batchSize) { + batches.push(tasks.slice(i, i + batchSize)); + } + + try { + await threadPool.executeAllTasks(batches); + } catch (error) { + console.error('线程池处理任务时出错:', error); + } +} + +// 监听来自主线程的消息 +parentPort.on('message', async (message) => { + if (message && typeof message === 'object') { + // 处理processTasks类型的消息 + if (message.type === 'processTasks' && message.taskIds && Array.isArray(message.taskIds)) { + await processTasks(message.taskIds); + parentPort.postMessage({ status: 'completed' }); + } + // 处理直接发送的数组 + else if (Array.isArray(message) && message.length > 0) { + await processTasks(message); + parentPort.postMessage({ status: 'completed' }); + } + // 处理退出消息 + else if (message.type === 'terminate') { + threadPool.terminate(); + process.exit(0); + } + } +}); \ No newline at end of file diff --git a/任务队列后端/worker_threads/callback_result/result.js b/任务队列后端/worker_threads/callback_result/result.js new file mode 100644 index 0000000..ac1c1e5 --- /dev/null +++ b/任务队列后端/worker_threads/callback_result/result.js @@ -0,0 +1,148 @@ +import { parentPort } from 'worker_threads' +import redis from '../../redis/index.js' +import initQueue from '../../redis/initQueue.js' + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +// 批量获取处理任务的信息 +async function getTasks() { + try { + // 从回调结果队列列表获取最多50个数据 + const taskIds = await redis.lRange(initQueue.callback, 0, 49); + const taskCountMap = new Map() + + if (taskIds.length === 0) { + // 如果没有任务ID，强制将回调队列任务数设置为0 + redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', 0); + logger.debug('回调结果队列为空，已重置任务数为0'); + return []; + } + + logger.debug('回调结果队列任务ID:', taskIds); + + // 批量获取任务的backendId和resultData字段 + const multi = redis.multi(); + for (const taskId of taskIds) { + // 只获取需要的字段 + multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'backendId'); + multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'resultData'); + multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'AIGC'); + multi.hGet(`${initQueue.prefix}:task:${taskId}`, 'platform'); + } + + const results = await multi.exec(); + + // 发送结果给客户端 + const processedTaskIds = []; + for (let i = 0; i < taskIds.length; i++) { + // 从结果数组中提取对应字段（每4个结果对应一个任务） + const backendId = results[i * 4] || ''; + const resultData = results[i * 4 + 1] || ''; + const aigc = results[i * 4 + 2] || 'default'; + const platform = results[i * 4 + 3] || 'default'; + const taskId = taskIds[i]; + + if (backendId) { + try { + // 直接打包结果和任务ID，发送给主线程 + const resultWithTaskId = { + taskId: taskId, + result: resultData + }; + + parentPort.postMessage({ + type: 'success', + backendId: backendId, + message: JSON.stringify(resultWithTaskId) + }); + + logger.debug(`成功发送结果给客户端，taskId: ${taskId}`); + + // 统计需要减少的平台任务数 + const key = `${aigc}:${platform}`; + console.log('key:', key); + if(taskCountMap.has(key)){ + taskCountMap.set(key, taskCountMap.get(key) + 1); + } else { + taskCountMap.set(key, 1); + } + processedTaskIds.push(taskId); + } catch (parseError) { + logger.error(`发送结果给客户端失败: ${taskId}`, parseError); + } + } + } + + // 只有在成功发送结果后才执行后续操作 + if (processedTaskIds.length > 0) { + // 使用原子操作执行多项任务 + const multi = redis.multi(); + + // 1. 移除已获取的任务ID + multi.lTrim(initQueue.callback, processedTaskIds.length, -1); + + // 2. 执行所有Redis操作 + await multi.exec(); + + // 3. 更新平台任务数（如果有需要更新的任务） + if (taskCountMap.size > 0) { + await initQueue.reducePlatformsProcess(taskCountMap); + } + + // 4. 更新回调队列任务数 + await initQueue.reduceCQtasksALL(processedTaskIds.length); + + logger.debug(`已处理${processedTaskIds.length}个回调结果任务，发送结果后结束`); + } + + return taskIds; + } catch (error) { + logger.error('处理回调结果任务失败:', error); + // 出错时，强制将回调队列任务数设置为0，避免死循环 + await redis.json.set(initQueue.initInfoKey, '$.CQtasksALL', 0); + return []; + } +} + +// 持续执行批量处理 +(async () => { + while (true) { + try { + // 判断结果队列是否有可发送的任务 + const rqTasksAll = await initQueue.getCQtasksALL(); + + + if (rqTasksAll !== 0) { + logger.info('回调结果队列有任务可处理，数量:', rqTasksAll); + // logger.debug('回调结果队列任务数量:', rqTasksAll); + // 先处理任务，再减少队列任务数 + await getTasks(); // 处理结果队列的任务 + + // 添加延迟，避免高频率执行 + await new Promise(resolve => setTimeout(resolve, 1000)); + } else { + // 回调结果队列无任务可处理，等待10秒后重试 + await new Promise(resolve => setTimeout(resolve, 10000)); + } + + } catch (error) { + logger.error('持续处理回调结果任务失败:', error); + // 出错后等待5秒再重试 + await new Promise(resolve => setTimeout(resolve, 5000)); + } + } +})() \ No newline at end of file diff --git a/任务队列后端/worker_threads/error/error.js b/任务队列后端/worker_threads/error/error.js new file mode 100644 index 0000000..320eb9b --- /dev/null +++ b/任务队列后端/worker_threads/error/error.js @@ -0,0 +1,125 @@ +import { parentPort } from 'worker_threads'; +import redis from '../../redis/index.js' +import initQueue from '../../redis/initQueue.js' + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +// 批量获取错误任务的信息 +async function getTasks() { + try { + const taskIds = await redis.lRange(initQueue.errorList, 0, -1); + + if (taskIds.length === 0) { + return true; + } + + logger.debug('错误队列任务ID:', taskIds); + + // 批量获取错误任务信息 + const multi = redis.multi(); + for (const taskId of taskIds) { + multi.hGetAll(`${initQueue.prefix}:task:${taskId}`); + } + + const results = await multi.exec(); + + let processedCount = 0; + const taskCountMap = new Map(); + + // 处理结果 + for (let i = 0; i < taskIds.length; i++) { + const taskId = taskIds[i]; + const taskInfo = results[i]; + + if (taskInfo && taskInfo.taskId && taskInfo.resultData) { + try { + logger.debug('错误队列任务数据:', taskInfo); + + // 直接打包错误信息和任务ID，发送给主线程 + const resultWithTaskId = { + taskId: taskInfo.taskId, + result: taskInfo.resultData + }; + + parentPort.postMessage({ + type: 'error', + backendId: taskInfo.backendId, + message: JSON.stringify(resultWithTaskId) + }); + processedCount++; + + // 统计需要减少计数的任务 + const key = `${taskInfo.AIGC}:${taskInfo.platform}`; + if(taskCountMap.has(key)){ + taskCountMap.set(key, taskCountMap.get(key) + 1); + } else { + taskCountMap.set(key, 1); + } + } catch (parseError) { + logger.error(`解析错误任务数据失败: ${taskInfo.resultData}`, parseError); + } + } + } + + // 删除已处理的错误任务 + if (processedCount > 0) { + const deleteMulti = redis.multi(); + for (const taskId of taskIds) { + deleteMulti.lRem(initQueue.errorList, 1, taskId); + // 同时删除tasks中的任务信息（从哈希存储中删除），使用项目前缀 + deleteMulti.del(`${initQueue.prefix}:task:${taskId}`); + } + await deleteMulti.exec(); + + await initQueue.reduceEQtaskALL(processedCount); + logger.info(`处理了 ${processedCount} 个错误任务`); + + // 减少等待队列的计数（错误任务来自等待队列） + if (taskCountMap.size > 0) { + await initQueue.reducePlatformsWait(taskCountMap); + } + } + + return true; + } catch (error) { + logger.error('获取错误任务失败:', error); + return false; + } +} + +// 持续执行批量处理 +(async () => { + while (true) { + try { + const errorTasksCount = await initQueue.getEQtaskALL(); + + // 判断是否有可处理的错误任务 + if (errorTasksCount > 0) { + logger.info('错误队列有可处理任务，数量:', errorTasksCount); + await getTasks(); + } else { + // 没有可处理的错误任务，等待15秒后重试 + await new Promise(resolve => setTimeout(resolve, 15000)); + logger.debug('错误队列无任务可处理'); + } + } catch (error) { + logger.error('持续处理错误任务失败:', error); + // 出错后等待5秒再重试 + await new Promise(resolve => setTimeout(resolve, 5000)); + } + } +})() diff --git a/任务队列后端/worker_threads/process/PollingThreadPool.js b/任务队列后端/worker_threads/process/PollingThreadPool.js new file mode 100644 index 0000000..e93983b --- /dev/null +++ b/任务队列后端/worker_threads/process/PollingThreadPool.js @@ -0,0 +1,87 @@ +import { Worker } from 'worker_threads'; + +// 创建固定大小的线程池（6个线程） +class QueryThreadPool { + constructor(size = 6) { + this.size = size; // 线程池大小，默认为6 + this.workers = []; // 存储worker对象的数组 + this.taskQueue = []; // 任务队列，存储等待处理的任务 + this.initWorkers(); // 初始化worker线程 + } + + initWorkers() { + for (let i = 0; i < this.size; i++) { + const worker = new Worker(new URL('./pollingTask.js', import.meta.url)); // 创建指向generatTask.js的worker + this.workers.push({ + worker, // worker对象 + busy: false // 是否正在处理任务的标志 + }); + } + } + + async executeTask(taskBatch) { + return new Promise((resolve, reject) => { + // 寻找空闲的worker + const availableWorker = this.workers.find(w => !w.busy); + + if (availableWorker) { + // 有空闲worker，直接执行任务 + this.runTaskOnWorker(availableWorker, taskBatch, resolve, reject); + } else { + // 没有空闲worker，将任务加入队列 + this.taskQueue.push({ taskBatch, resolve, reject }); + } + }); + } + + runTaskOnWorker(workerObj, taskBatch, resolve, reject) { + workerObj.busy = true; + + workerObj.worker.once('message', (result) => { + workerObj.busy = false; + resolve(result); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.once('error', (error) => { + workerObj.busy = false; + reject(error); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.postMessage(taskBatch); + } + + processQueuedTasks() { + if (this.taskQueue.length > 0) { + const availableWorker = this.workers.find(w => !w.busy); + if (availableWorker) { + const queuedTask = this.taskQueue.shift(); + this.runTaskOnWorker( + availableWorker, + queuedTask.taskBatch, + queuedTask.resolve, + queuedTask.reject + ); + } + } + } + + async executeAllTasks(taskBatches) { + const promises = taskBatches.map(batch => this.executeTask(batch)); + return Promise.all(promises); + } + + // 清理资源 + terminate() { + this.workers.forEach(({ worker }) => { + worker.terminate(); + }); + } +} + +export default new QueryThreadPool(); diff --git a/任务队列后端/worker_threads/process/pollingTask.js b/任务队列后端/worker_threads/process/pollingTask.js new file mode 100644 index 0000000..cd5aab6 --- /dev/null +++ b/任务队列后端/worker_threads/process/pollingTask.js @@ -0,0 +1,114 @@ +import { parentPort } from 'worker_threads'; +import redis from '../../redis/index.js'; +import initQueue from '../../redis/initQueue.js'; +import { externalGetRequest } from '../../outside/polling.js'; + +async function getTask(tasks) { + console.log(`[pollingTask] 开始处理 ${Object.keys(tasks).length} 个轮询任务`); + console.log(`[pollingTask] 轮询任务数据: ${JSON.stringify(tasks)}`); + const queryTasks = [] + for (const [remoteTaskId, value] of Object.entries(tasks)) { + console.log(`[pollingTask] 准备查询任务: remoteTaskId=${remoteTaskId}, value=${value}`); + // 查询外部平台任务结果 + const queryTaskPromise = externalGetRequest(remoteTaskId, value) // { platform, taskid, AIGC } + queryTasks.push(queryTaskPromise) + } + + try { + const responseTasks = await Promise.all(queryTasks) + console.log(`[pollingTask] 轮询查询完成，收到 ${responseTasks.length} 个响应`); + console.log(`[pollingTask] 轮询响应详情: ${JSON.stringify(responseTasks)}`); + return responseTasks + } catch (error) { + console.error('[pollingTask] 轮询查询出错:', error); + } +} + +// 批量将完成的任务移动到结果队列中 +async function storeSuccessTasks(SuccessTasks) { + console.log(`[pollingTask] 开始存储 ${SuccessTasks.length} 个成功完成的任务到结果队列`); + const taskCountMap = new Map(); + const taskErrorCountMap = new Map(); + + // 准备批量操作 + const multi = redis.multi(); + + for (const task of SuccessTasks) { + const taskId = task.taskid || task.taskId; + const remoteTaskId = task.remoteTaskId; + const aigc = task.aigc; + const platform = task.platform; + + console.log(`[pollingTask] 处理任务: taskId=${taskId}, remoteTaskId=${remoteTaskId}, status=${task.status}`); + + // 存储结果到 task 的 resultData 里 + multi.hSet(`${initQueue.prefix}:task:${taskId}`, 'resultData', task.result); + multi.hSet(`${initQueue.prefix}:task:${taskId}`, 'status', task.status); + + // 判断是否为错误任务 + if (task.status === 'failed') { + // 推送任务 ID 到错误列表 + multi.lPush(initQueue.errorList, taskId); + + // 计算错误队列任务数 + const key = `${aigc}:${platform}`; + if(taskErrorCountMap.has(key)){ + taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1); + } else { + taskErrorCountMap.set(key, 1); + } + } else { + // 推送任务 ID 到结果列表 + multi.lPush(initQueue.resultList, taskId); + + // 计算各平台处理队列的已完成待释放任务数 + const key = `${aigc}:${platform}`; + if(taskCountMap.has(key)){ + taskCountMap.set(key, taskCountMap.get(key) + 1); + } else { + taskCountMap.set(key, 1); + } + } + + // 按平台+AIGC类型删除轮询任务 + const platformKey = `${initQueue.prefix}:processPolling:${aigc}:${platform}`; + multi.hDel(platformKey, remoteTaskId); + } + + // 执行所有Redis操作 + await multi.exec(); + console.log(`[pollingTask] 已完成Redis批量操作，删除了轮询任务并存储到结果队列`); + + // 更新平台计数（使用原子操作） + if (taskCountMap.size > 0) { + await initQueue.reducePlatformsProcess(taskCountMap); + console.log(`[pollingTask] 已更新平台计数: ${JSON.stringify(Array.from(taskCountMap.entries()))}`); + } + + // 更新错误队列计数 + if (taskErrorCountMap.size > 0) { + const totalErrorCount = Object.values(taskErrorCountMap).reduce((a, b) => a + b, 0); + initQueue.addEQtaskALL(totalErrorCount); + console.log(`[pollingTask] 已更新错误队列计数: ${JSON.stringify(Array.from(taskErrorCountMap.entries()))}`); + } + + return true +} + +parentPort.on('message', async (tasks) => { + console.log(`[pollingTask] 收到主线程消息，任务数量: ${Object.keys(tasks).length}`); + try { + const responseTasks = await getTask(tasks); + // 过滤掉 undefined 元素 + const successTasks = responseTasks.filter(task => task !== undefined); + console.log(`[pollingTask] 过滤后的成功任务数量: ${successTasks.length}`); + if (successTasks.length > 0) { + await storeSuccessTasks(successTasks); + } + parentPort.postMessage({ status: 'completed', processed: successTasks.length }); + console.log(`[pollingTask] 任务处理完成，已通知主线程`); + } catch (error) { + console.error('[pollingTask] 处理轮询任务时出错:', error); + parentPort.postMessage({ status: 'error', error: error.message }); + } +}); diff --git a/任务队列后端/worker_threads/process/process.js b/任务队列后端/worker_threads/process/process.js new file mode 100644 index 0000000..af10077 --- /dev/null +++ b/任务队列后端/worker_threads/process/process.js @@ -0,0 +1,172 @@ +import { parentPort } from 'worker_threads'; +import redis from '../../redis/index.js'; +import initQueue from '../../redis/initQueue.js'; +import QueryThreadPool from './PollingThreadPool.js'; + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +// 创建线程池实例 +const threadPool = QueryThreadPool; // 6个线程的线程池 + +// 批量获取处理任务的信息 +async function getTasks(tasks) { + // 每100个任务为一组 + const batchSize = 100; + const batches = []; + + // 将任务数据按批次分割 + const taskEntries = Object.entries(tasks); + for (let i = 0; i < taskEntries.length; i += batchSize) { + const batchEntries = taskEntries.slice(i, i + batchSize); + batches.push(Object.fromEntries(batchEntries)); + } + + try { + // 等待所有子线程完成处理 + const workerResults = await threadPool.executeAllTasks(batches); + return workerResults.flat(); + } catch (error) { + logger.error('子线程处理任务时出错:', error); + throw error; + } +} + +// 获取系统负载（简化版，实际可根据CPU/内存使用率调整） +async function getSystemLoad() { + return 0.5; // 模拟系统负载，0-1之间 +} + +// 动态计算轮询间隔 +function getDynamicInterval(taskCount) { + if (taskCount > 100) return 5000; // 任务多，缩短间隔 + if (taskCount > 0) return 10000; // 有任务，正常间隔 + return 30000; // 无任务，延长间隔 +} + +// 获取支持的平台和AIGC类型组合 +async function getSupportedPlatforms() { + // 从redis获取所有平台AIGC组合 + const platforms = new Set(); + try { + // 直接从轮询队列中获取平台信息，这样更准确 + const pollingKeys = await redis.keys(`${initQueue.prefix}:processPolling:*`); + logger.debug(`[getSupportedPlatforms] 找到轮询队列键: ${JSON.stringify(pollingKeys)}`); + for (const key of pollingKeys) { + // 从轮询键中提取AIGC和platform + const match = key.match(/processPolling:(.+?):(.+)$/); + if (match) { + const [, aigc, platform] = match; + platforms.add(`${aigc}:${platform}`); + logger.debug(`[getSupportedPlatforms] 添加平台组合: ${aigc}:${platform}`); + } + } + + // 如果轮询队列中没有任务，从所有任务中获取平台信息作为备用 + if (platforms.size === 0) { + logger.debug('[getSupportedPlatforms] 轮询队列为空，从任务数据中获取平台信息'); + const taskKeys = await redis.keys(`${initQueue.prefix}:task:*`); + logger.debug(`[getSupportedPlatforms] 找到任务键: ${taskKeys.length} 个`); + for (const key of taskKeys) { + // 获取任务信息 + const infoStr = await redis.hGet(key, 'info'); + if (infoStr) { + try { + const info = JSON.parse(infoStr); + platforms.add(`${info.AIGC}:${info.platform}`); + logger.debug(`[getSupportedPlatforms] 从任务添加平台组合: ${info.AIGC}:${info.platform}`); + } catch (parseError) { + logger.error('解析任务信息失败:', parseError); + } + } + } + } + + logger.info(`[getSupportedPlatforms] 最终支持的平台组合: ${JSON.stringify(Array.from(platforms))}`); + } catch (error) { + logger.error('获取支持的平台组合失败:', error); + } + return Array.from(platforms); +} + +// 轮询单个平台组合 +async function pollPlatform(platformKey) { + logger.info(`开始轮询平台组合: ${platformKey}`); + + while(true) { + try { + // 获取该平台的轮询任务 + const [aigc, platform] = platformKey.split(':'); + const pollingKey = `${initQueue.prefix}:processPolling:${aigc}:${platform}`; + + // 检查是否有任务 + const taskCount = await redis.hLen(pollingKey); + logger.debug(`[pollPlatform] 检查轮询队列: ${pollingKey}, 任务数量: ${taskCount}`); + + if(taskCount > 0) { + logger.info(`平台 ${platformKey} 有可处理任务，数量: ${taskCount}`); + + // 动态计算批量大小 + const systemLoad = await getSystemLoad(); + const batchSize = Math.max(50, Math.min(200, 100 - systemLoad * 50)); + + // 获取所有任务 + const tasks = await redis.hGetAll(pollingKey); + logger.debug(`批量获取平台 ${platformKey} 任务信息: ${JSON.stringify(Object.keys(tasks))}`); + + // 处理任务 + await getTasks(tasks); + logger.info(`平台 ${platformKey} 处理完成`); + } + + // 动态调整轮询间隔 + const interval = getDynamicInterval(taskCount); + logger.debug(`平台 ${platformKey} 轮询间隔: ${interval}ms`); + await new Promise(resolve => setTimeout(resolve, interval)); + } catch (error) { + logger.error(`处理平台 ${platformKey} 任务时出错:`, error); + // 出错后等待5秒再重试 + await new Promise(resolve => setTimeout(resolve, 5000)); + } + } +} + +// 持续执行批量处理 +(async () => { + logger.info('[process.js] 轮询线程启动'); + // 获取支持的平台组合 + const platforms = await getSupportedPlatforms(); + logger.info(`支持的平台组合: ${platforms.join(', ')}`); + + // 为每个平台组合启动独立的轮询线程 + for (const platform of platforms) { + logger.info(`[process.js] 启动平台轮询线程: ${platform}`); + pollPlatform(platform); + } + + // 定期检查新增的平台组合 + setInterval(async () => { + logger.debug('[process.js] 检查新增的平台组合'); + const currentPlatforms = await getSupportedPlatforms(); + for (const platform of currentPlatforms) { + if (!platforms.includes(platform)) { + platforms.push(platform); + logger.info(`发现新平台组合: ${platform}`); + pollPlatform(platform); + } + } + }, 60000); // 每分钟检查一次 +})() diff --git a/任务队列后端/worker_threads/result/RecordThreadPool.js b/任务队列后端/worker_threads/result/RecordThreadPool.js new file mode 100644 index 0000000..6772778 --- /dev/null +++ b/任务队列后端/worker_threads/result/RecordThreadPool.js @@ -0,0 +1,87 @@ +import { Worker } from 'worker_threads'; + +// 创建固定大小的线程池（3个线程） +class QueryThreadPool { + constructor(size = 1) { + this.size = size; // 线程池大小，默认为6 + this.workers = []; // 存储worker对象的数组 + this.taskQueue = []; // 任务队列，存储等待处理的任务 + this.initWorkers(); // 初始化worker线程 + } + + initWorkers() { + for (let i = 0; i < this.size; i++) { + const worker = new Worker(new URL('./recordTask.js', import.meta.url)); // 确保路径正确 + this.workers.push({ + worker, // worker对象 + busy: false // 是否正在处理任务的标志 + }); + } + } + + async executeTask(taskBatch) { + return new Promise((resolve, reject) => { + // 寻找空闲的worker + const availableWorker = this.workers.find(w => !w.busy); + + if (availableWorker) { + // 有空闲worker，直接执行任务 + this.runTaskOnWorker(availableWorker, taskBatch, resolve, reject); + } else { + // 没有空闲worker，将任务加入队列 + this.taskQueue.push({ taskBatch, resolve, reject }); + } + }); + } + + runTaskOnWorker(workerObj, taskBatch, resolve, reject) { + workerObj.busy = true; + + workerObj.worker.once('message', (result) => { + workerObj.busy = false; + resolve(result); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.once('error', (error) => { + workerObj.busy = false; + reject(error); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.postMessage(taskBatch); + } + + processQueuedTasks() { + if (this.taskQueue.length > 0) { + const availableWorker = this.workers.find(w => !w.busy); + if (availableWorker) { + const queuedTask = this.taskQueue.shift(); + this.runTaskOnWorker( + availableWorker, + queuedTask.taskBatch, + queuedTask.resolve, + queuedTask.reject + ); + } + } + } + + async executeAllTasks(taskBatches) { + const promises = taskBatches.map(batch => this.executeTask(batch)); + return Promise.all(promises); + } + + // 清理资源 + terminate() { + this.workers.forEach(({ worker }) => { + worker.terminate(); + }); + } +} + +export default new QueryThreadPool(); \ No newline at end of file diff --git a/任务队列后端/worker_threads/result/recordTask.js b/任务队列后端/worker_threads/result/recordTask.js new file mode 100644 index 0000000..9107d06 --- /dev/null +++ b/任务队列后端/worker_threads/result/recordTask.js @@ -0,0 +1,28 @@ +// recordTask.js +import { parentPort } from 'worker_threads' +import { record } from '../../outside/record.js' + +async function recordTask(tasks) { + const recordTasks = [] + for (const task of tasks) { + // 对每个task执行record函数 + const recordTaskPromise = record(task) + recordTasks.push(recordTaskPromise) + } + + try { + await Promise.all(recordTasks) + } catch (error) { + console.error('Error:', error) + throw error // 重新抛出错误以便上层处理 + } +} + +parentPort.on('message', async (tasks) => { + try { + await recordTask(tasks) + parentPort.postMessage({ status: 'completed' }) + } catch (error) { + parentPort.postMessage({ status: 'error', error: error.message }) + } +}) diff --git a/任务队列后端/worker_threads/result/recordWorkerManager.js b/任务队列后端/worker_threads/result/recordWorkerManager.js new file mode 100644 index 0000000..d69b814 --- /dev/null +++ b/任务队列后端/worker_threads/result/recordWorkerManager.js @@ -0,0 +1,40 @@ +// recordWorkerManager.js +import { parentPort } from 'worker_threads'; +import RecordThreadPool from './RecordThreadPool.js'; + +// 在当前 Worker 中初始化线程池 +const threadPool = RecordThreadPool(); // 3个线程的线程池 + +// 处理任务的函数 +async function processTasks(tasks) { + // 每100个任务为一组 + const batchSize = 100; + const batches = []; + + // 将任务数据按批次分割 + for (let i = 0; i < tasks.length; i += batchSize) { + batches.push(tasks.slice(i, i + batchSize)); + } + + try { + await threadPool.executeAllTasks(batches); + } catch (error) { + console.error('线程池处理任务时出错:', error); + } +} + +// 监听来自主线程的消息 +parentPort.on('message', async (tasks) => { + if (tasks && Array.isArray(tasks) && tasks.length > 0) { + await processTasks(tasks); + parentPort.postMessage({ status: 'completed' }); + } +}); + +// 监听退出消息，清理资源 +parentPort.on('message', (message) => { + if (message.type === 'terminate') { + threadPool.terminate(); + process.exit(0); + } +}); \ No newline at end of file diff --git a/任务队列后端/worker_threads/result/result.js b/任务队列后端/worker_threads/result/result.js new file mode 100644 index 0000000..d67269b --- /dev/null +++ b/任务队列后端/worker_threads/result/result.js @@ -0,0 +1,121 @@ +import { parentPort } from 'worker_threads' +import redis from '../../redis/index.js' +import initQueue from '../../redis/initQueue.js'; + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +// 批量获取处理任务的信息 +async function getTasks() { + const tasks = []; + const processedTaskIds = []; + const taskCountMap = new Map() + try { + // 1. 首先获取所有任务ID + const taskIDs = await redis.lRange(initQueue.resultList, 0, -1); + + if (taskIDs.length === 0) { + return tasks; + } + + // 2. 批量获取完整任务信息 + const taskInfoPromises = []; + for (const taskID of taskIDs) { + taskInfoPromises.push(redis.hGetAll(`${initQueue.prefix}:task:${taskID}`)); + } + + const taskInfos = await Promise.all(taskInfoPromises); + + // 3. 处理结果 + for (let i = 0; i < taskIDs.length; i++) { + const taskId = taskIDs[i]; + const taskInfo = taskInfos[i]; + + if (taskInfo && taskInfo.taskId && taskInfo.resultData) { + try { + // 解析JSON格式的字段 + const task = { + taskId: taskInfo.taskId, + status: taskInfo.status, + resultData: taskInfo.resultData, + token: taskInfo.token + }; + + // 直接打包结果和任务ID，发送给主线程 + const resultWithTaskId = { + taskId: task.taskId, + result: task.resultData + }; + + parentPort.postMessage({ + type: 'success', + backendId: taskInfo.backendId, + message: JSON.stringify(resultWithTaskId) + }); + + tasks.push(task); + processedTaskIds.push(taskId); + } catch (parseError) { + logger.error(`解析任务结果失败: ${taskInfo.resultData}`, parseError); + } + } + } + + // 4. 只删除已成功处理的任务结果 + if (processedTaskIds.length > 0) { + const deleteMulti = redis.multi(); + for (const taskId of processedTaskIds) { + deleteMulti.lRem(initQueue.resultList, 1, taskId); + } + await deleteMulti.exec(); + + // 5. 更新结果队列计数 + // 从结果队列中减少任务数 + try { + await redis.json.numIncrBy(initQueue.initInfoKey || 'default:InitInfo', '$.RQtasksALL', -processedTaskIds.length); + logger.debug(`减少结果队列任务数: ${processedTaskIds.length}`); + } catch (error) { + logger.error('减少结果队列任务数失败:', error); + } + } + + } catch (error) { + logger.error('获取任务结果失败:', error); + } + + return tasks; +} + +// 持续执行批量处理 +(async () => { + while (true) { + try { + // 直接检查结果列表长度，而不是依赖计数 + const resultListLength = await redis.lLen(initQueue.resultList); + if (resultListLength > 0) { + logger.info('结果队列有可处理任务，数量: ' + resultListLength); + await getTasks(); // 处理结果队列的任务，发送结果后结束 + } else { + // 结果队列为空，等待15秒后重试 + await new Promise(resolve => setTimeout(resolve, 15000)); + } + } catch (error) { + logger.error('处理结果任务时出错:', error); + // 出错后等待5秒再重试 + await new Promise(resolve => setTimeout(resolve, 5000)); + } + } +})() diff --git a/任务队列后端/worker_threads/wait/GenerateThreadPool.js b/任务队列后端/worker_threads/wait/GenerateThreadPool.js new file mode 100644 index 0000000..51cd50e --- /dev/null +++ b/任务队列后端/worker_threads/wait/GenerateThreadPool.js @@ -0,0 +1,87 @@ +import { Worker } from 'worker_threads'; + +// 创建固定大小的线程池（6个线程） +class TaskThreadPool { + constructor(size = 1) { + this.size = size; // 线程池大小，默认为6 + this.workers = []; // 存储worker对象的数组 + this.taskQueue = []; // 任务队列，存储等待处理的任务 + this.initWorkers(); // 初始化worker线程 + } + + initWorkers() { + for (let i = 0; i < this.size; i++) { + const worker = new Worker(new URL('./generatTask.js', import.meta.url)); // 创建指向generatTask.js的worker + this.workers.push({ + worker, // worker对象 + busy: false // 是否正在处理任务的标志 + }); + } + } + + async executeTask(taskBatch) { + return new Promise((resolve, reject) => { + // 寻找空闲的worker + const availableWorker = this.workers.find(w => !w.busy); + + if (availableWorker) { + // 有空闲worker，直接执行任务 + this.runTaskOnWorker(availableWorker, taskBatch, resolve, reject); + } else { + // 没有空闲worker，将任务加入队列 + this.taskQueue.push({ taskBatch, resolve, reject }); + } + }); + } + + runTaskOnWorker(workerObj, taskBatch, resolve, reject) { + workerObj.busy = true; + + workerObj.worker.once('message', (result) => { + workerObj.busy = false; + resolve(result); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.once('error', (error) => { + workerObj.busy = false; + reject(error); + + // 检查是否有排队的任务 + this.processQueuedTasks(); + }); + + workerObj.worker.postMessage(taskBatch); + } + + processQueuedTasks() { + if (this.taskQueue.length > 0) { + const availableWorker = this.workers.find(w => !w.busy); + if (availableWorker) { + const queuedTask = this.taskQueue.shift(); + this.runTaskOnWorker( + availableWorker, + queuedTask.taskBatch, + queuedTask.resolve, + queuedTask.reject + ); + } + } + } + + async executeAllTasks(taskBatches) { + const promises = taskBatches.map(batch => this.executeTask(batch)); + return Promise.all(promises); + } + + // 清理资源 + terminate() { + this.workers.forEach(({ worker }) => { + worker.terminate(); + }); + } +} + +export default new TaskThreadPool(); \ No newline at end of file diff --git a/任务队列后端/worker_threads/wait/GenerateWorkerManager.js b/任务队列后端/worker_threads/wait/GenerateWorkerManager.js new file mode 100644 index 0000000..4446401 --- /dev/null +++ b/任务队列后端/worker_threads/wait/GenerateWorkerManager.js @@ -0,0 +1,41 @@ +// wait/WaitWorkerManager.js +import { parentPort } from 'worker_threads'; +import GenerateThreadPool from './GenerateThreadPool.js'; + +// 在独立的 Worker 中初始化线程池 +const threadPool = GenerateThreadPool; // 6个线程的线程池 + +// 处理任务的函数 +async function processTasks(tasks) { + // 每100个任务为一组 + const batchSize = 100; + const batches = []; + + // 将任务数据按批次分割 + for (let i = 0; i < tasks.length; i += batchSize) { + batches.push(tasks.slice(i, i + batchSize)); + } + + try { + await threadPool.executeAllTasks(batches); + } catch (error) { + console.error('线程池处理任务时出错:', error); + } +} + +// 监听来自主线程的消息 +parentPort.on('message', async (tasks) => { + // // console.log('接收到来自主线程的消息:', tasks); + if (tasks && Array.isArray(tasks) && tasks.length > 0) { + await processTasks(tasks); + parentPort.postMessage({ status: 'completed' }); + } +}); + +// 监听退出消息，清理资源 +parentPort.on('message', (message) => { + if (message.type === 'terminate') { + threadPool.terminate(); + process.exit(0); + } +}); \ No newline at end of file diff --git a/任务队列后端/worker_threads/wait/generatTask.js b/任务队列后端/worker_threads/wait/generatTask.js new file mode 100644 index 0000000..592da36 --- /dev/null +++ b/任务队列后端/worker_threads/wait/generatTask.js @@ -0,0 +1,218 @@ +import { parentPort } from 'worker_threads'; +import redis from '../../redis/index.js'; +import initQueue from '../../redis/initQueue.js'; +import { externalPostRequest } from '../../outside/generat.js'; +import { platformData } from '../../config/Config.js'; + +// 批量转发待处理任务到各外部平台 +async function generatTask(tasksData) { + // console.log('开始转发任务'); + const generatTasks = [] + for (const task of tasksData) { + // 2. 获取任务所属平台的生成接口地址 + const generatTaskPromise = externalPostRequest(task) // { aigc, tasksData } + generatTasks.push(generatTaskPromise) + } + + try { + const responseTasks = await Promise.all(generatTasks) + return responseTasks + } catch (error) { + console.error('Error:', error); + return []; // 确保总是返回数组 + } +} + +// 批量储存外部平台返回的任务数据到处理队列 +async function storeGeneratTasks(tasks) { + // 确保tasks是数组 + if (!tasks || !Array.isArray(tasks)) { + console.error('storeGeneratTasks函数接收到无效的tasks参数:', tasks); + return; + } + + const multi = redis.multi(); + let errorCount = 0; + const taskErrorCountMap = new Map(); + const taskCountMap = new Map(); + for (const task of tasks) { + // console.log('\n***************',task) + //错误任务 + if(task.remoteTaskId?.type === 2){ + console.log('储存在错误队列', task); + const aigc = task.AIGC || task.aigc; + const platform = task.platform || task.platformName; + + // 存储错误信息到任务数据中 + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', JSON.stringify(task.remoteTaskId.message)); + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed'); + + // 推送任务ID到错误列表 + multi.lPush(initQueue.errorList, task.taskId); + + errorCount++; + // 存储相关平台信息 + const key = `${aigc}:${platform}`; + if(taskErrorCountMap.has(key)){ + taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1); + } else { + taskErrorCountMap.set(key, 1); + } + continue // 跳过错误任务 + } + + // 处理成功的任务 + let externalTaskId; + if (task.remoteTaskId?.type === 1 && task.remoteTaskId?.data) { + // 使用解析后的响应数据提取外部平台任务ID + try { + const responseData = task.remoteTaskId.data; + // console.log('处理成功任务，响应数据:', responseData); + + // 直接处理响应数据，提取任务ID + const platform = task.platform || task.platformName; + if ((responseData.msg === 'success' || platform === 'coze') && responseData.code === 0) { + // Coze平台返回的是execute_id，其他平台返回的是data.taskId + if (platform === 'coze') { + externalTaskId = responseData.execute_id; + } else { + externalTaskId = responseData.data?.taskId; + } + + if (externalTaskId) { + console.log('成功提取外部平台任务ID:', externalTaskId); + } else { + console.error('无法从响应中提取外部平台任务ID:', responseData); + // 视为错误任务 + const errorMessage = JSON.stringify({ message: '无法从响应中提取外部平台任务ID', response: responseData }); + + // 存储错误信息到任务数据中 + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage); + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed'); + + // 推送任务ID到错误列表 + multi.lPush(initQueue.errorList, task.taskId); + + errorCount++; + // 存储相关平台信息 + const key = `${task.AIGC}:${task.platform}`; + if(taskErrorCountMap.has(key)){ + taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1); + } else { + taskErrorCountMap.set(key, 1); + } + continue; // 跳过错误任务 + } + } else { + console.error('外部平台返回错误:', responseData); + // 视为错误任务 + const aigc = task.AIGC || task.aigc; + const platform = task.platform || task.platformName; + const errorMessage = JSON.stringify(responseData); + + // 存储错误信息到任务数据中 + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage); + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed'); + + // 推送任务ID到错误列表 + multi.lPush(initQueue.errorList, task.taskId); + + errorCount++; + // 存储相关平台信息 + const key = `${aigc}:${platform}`; + if(taskErrorCountMap.has(key)){ + taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1); + } else { + taskErrorCountMap.set(key, 1); + } + continue; // 跳过错误任务 + } + } catch (extractError) { + console.error('提取外部平台任务ID失败:', extractError); + // 视为错误任务 + const aigc = task.AIGC || task.aigc; + const platform = task.platform || task.platformName; + const errorMessage = JSON.stringify({ message: '提取外部平台任务ID失败', error: extractError.message }); + + // 存储错误信息到任务数据中 + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'resultData', errorMessage); + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'status', 'failed'); + + // 推送任务ID到错误列表 + multi.lPush(initQueue.errorList, task.taskId); + + errorCount++; + // 存储相关平台信息 + const key = `${aigc}:${platform}`; + if(taskErrorCountMap.has(key)){ + taskErrorCountMap.set(key, taskErrorCountMap.get(key) + 1); + } else { + taskErrorCountMap.set(key, 1); + } + continue; // 跳过错误任务 + } + } else { + // 直接使用remoteTaskId作为外部平台任务ID + externalTaskId = task.remoteTaskId; + } + + //回调任务 + const aigc = task.AIGC || task.aigc; + const platform = task.platform || task.platformName; + if(platformData.callback.includes(platform)) { + console.log('储存在回调队列', externalTaskId, task.taskId); + multi.set(`${initQueue.callback}:${externalTaskId}`, task.taskId) + } else { // 轮询任务 + // 按平台+AIGC类型存储轮询任务 + const pollingKey = `${initQueue.prefix}:processPolling:${aigc}:${platform}`; + // 从task中提取workflow_id，优先使用task.workflowId + let workflowId = task.workflowId || ''; + try { + if (!workflowId && task.taskData) { + // taskData 已经是字符串，直接解析 + const taskDataParsed = JSON.parse(task.taskData); + workflowId = taskDataParsed.workflow_id || ''; + } + } catch (e) { + console.error('[generatTask] 解析taskData获取workflow_id失败:', e); + } + console.log(`[generatTask] 提取到的workflowId: ${workflowId}`); + // 确保workflowId被传递到轮询任务中 + const pollingData = { + taskId: task.taskId, + platform: platform, + AIGC: aigc, + workflowId: workflowId // 包含workflowId，为空则使用空字符串 + }; + console.log(`[generatTask] 添加轮询任务: pollingKey=${pollingKey}, externalTaskId=${externalTaskId}, pollingData=${JSON.stringify(pollingData)}`); + multi.hSet(pollingKey, externalTaskId, JSON.stringify(pollingData)) + } + + // 更新任务信息，添加 remoteTaskId 字段 + multi.hSet(`${initQueue.prefix}:task:${task.taskId}`, 'remoteTaskId', externalTaskId); + // 确保任务有2小时的过期时间 + multi.expire(`${initQueue.prefix}:task:${task.taskId}`, 7200); + + // 记录相关队列处理的任务数 + const key = `${aigc}:${platform}`; + if(taskCountMap.has(key)){ + taskCountMap.set(key, taskCountMap.get(key) + 1); + } else { + taskCountMap.set(key, 1); + } + } + + // 更新平台信息 + if(errorCount > 0){ + initQueue.addEQtaskALL(errorCount) // 添加错误队列任务数量 + } + // 注意：这里不再调用addPlatformsProcess，因为PQtasks计数已经在updateTaskCounts函数中处理过了 + // 避免同一个任务被两次增加PQtasks计数 + await multi.exec(); +} + +parentPort.on('message', async (tasksData) => { + await generatTask(tasksData) + .then (tasks => storeGeneratTasks(tasks)) + parentPort.postMessage({ status: 'completed' }); +}); diff --git a/任务队列后端/worker_threads/wait/waiting.js b/任务队列后端/worker_threads/wait/waiting.js new file mode 100644 index 0000000..d67e5f4 --- /dev/null +++ b/任务队列后端/worker_threads/wait/waiting.js @@ -0,0 +1,253 @@ +import { parentPort, Worker } from 'worker_threads'; +import redis from '../../redis/index.js'; +import initQueue from '../../redis/initQueue.js'; + +// 日志工具函数 +const logger = { + info: (message) => { + const timestamp = new Date().toISOString(); + console.log(`[${timestamp}] INFO: ${message}`); + }, + error: (message, error) => { + const timestamp = new Date().toISOString(); + console.error(`[${timestamp}] ERROR: ${message}`, error || ''); + }, + debug: (message) => { + const timestamp = new Date().toISOString(); + console.debug(`[${timestamp}] DEBUG: ${message}`); + } +}; + +// 创建专门的线程池管理 Worker +const generateWorker = new Worker(new URL('./GenerateWorkerManager.js', import.meta.url)); + +// 判断并发数，获取可进行任务处理的等待队列 +async function judgConcurrency() { + try { + // 获取平台相关信息包括等待队列名与并发数，当前任务数等 + const platforms = await initQueue.getPlatforms(); + // 储存可进行任务处理的等待队列 + const wDeficiency = []; + + logger.debug('获取到的平台信息：', platforms); + + // 检查每个平台的实际队列长度 + for(const [aigcPfName, info] of Object.entries(platforms)) { + try { + // 直接检查 Redis 队列的实际长度 + const actualQueueLength = await redis.lLen(info.waitQueue); + logger.debug(`平台 ${aigcPfName} 信息：PQtasks=${info.PQtasks}, MAX_CONCURRENT=${info.MAX_CONCURRENT}, 实际队列长度=${actualQueueLength}`); + + // 判断是否可以处理任务：并发数未满且队列中有任务 + if (info.PQtasks < info.MAX_CONCURRENT && actualQueueLength > 0) { + let count = info.MAX_CONCURRENT - info.PQtasks; + // 可处理的任务数不能大于队列实际长度 + if(count > actualQueueLength) { + count = actualQueueLength; + } + wDeficiency.push({ aigcPfName, info, count }); // 储存可进行任务处理的等待队列 + logger.debug(`平台 ${aigcPfName} 满足处理条件，可处理 ${count} 个任务`); + } else { + logger.debug(`平台 ${aigcPfName} 不满足处理条件：PQtasks < MAX_CONCURRENT = ${info.PQtasks < info.MAX_CONCURRENT}, 队列长度 > 0 = ${actualQueueLength > 0}`); + } + } catch (error) { + logger.error(`检查平台 ${aigcPfName} 队列长度失败:`, error); + } + } + + return wDeficiency; // 返回可处理的队列列表 + } catch (error) { + logger.error('判断并发数失败:', error); + return []; + } +} + +// 批量获取等待队列任务的任务ID（仅获取，不移除） +async function getBatchWaitTasksID(platforms) { + try { + const multi = redis.multi(); + // 从等待队列批量获取任务ID，但不立即移除 + for(const platform of platforms) { + multi.lRange(platform.info.waitQueue, 0, platform.count - 1); + } + + // 执行所有命令 + const results = await multi.exec(); + console.log('批量获取等待队列任务ID结果:', results); + + // 将任务ID与处理队列关联 + for(let i = 0; i < results.length; i++) { + const taskIDs = results[i] || []; + const platform = platforms[i]; + platform.waitTaskID = taskIDs; + } + + logger.debug('批量获取等待队列任务ID：', platforms); + return platforms; + } catch (error) { + logger.error('批量获取等待队列任务ID失败:', error); + return platforms; + } +} + +// 批量获取多个等待队列中的任务数据 +async function getBatchWaitTasks(aigcPfTasks) { + const tasksData = []; + + try { + // 收集所有需要获取的任务ID + const allTaskIds = []; + const taskIdMap = new Map(); // 用于映射任务ID到平台信息 + + for(const aigcPfTask of aigcPfTasks) { + for(const taskId of aigcPfTask.waitTaskID) { + if (taskId) { + allTaskIds.push(taskId); + taskIdMap.set(taskId, { + platformName: aigcPfTask.info.platformName, + aigc: aigcPfTask.info.AIGC, + aigcPfName: aigcPfTask.aigcPfName + }); + } + } + } + + if (allTaskIds.length === 0) { + return tasksData; + } + + // 批量获取任务数据 + const multi = redis.multi(); + for(const taskId of allTaskIds) { + multi.hGetAll(`${initQueue.prefix}:task:${taskId}`); + } + + const results = await multi.exec(); + + // 处理结果 + for(let i = 0; i < results.length; i++) { + const taskInfo = results[i]; + const taskId = allTaskIds[i]; + const platformInfo = taskIdMap.get(taskId); + + if (taskInfo) { + try { + tasksData.push({ + backendId: taskInfo.backendId, + taskId: taskInfo.taskId, // 单个任务ID + platformName: platformInfo.platformName, + aigc: platformInfo.aigc, + aigcPfName: platformInfo.aigcPfName, + taskData: taskInfo.payload, + workflowId: taskInfo.workflowId || '', + }); + // logger.debug(`已获取任务 ${taskId} 数据：platform=${platformInfo.platformName}, aigc=${platformInfo.aigc}`); + } catch (error) { + logger.error(`解析任务${taskId}数据失败:`, error); + } + } else { + logger.warn(`任务 ${taskId} 数据不存在`); + } + } + + // logger.debug('批量获取多个等待队列中的任务数据：', tasksData); + return tasksData; + } catch (error) { + logger.error('批量获取任务数据失败:', error); + return tasksData; + } +} + +// 批量移除任务ID、更新等待队列计数并增加处理队列任务数 +async function updateTaskCounts(wDeficiency) { + try { + const taskCountMap = new Map(); + const multi = redis.multi(); + + // 1. 准备批量移除任务ID和更新计数 + for(const aigcPfTask of wDeficiency) { + const { waitTaskID, info } = aigcPfTask; + const key = aigcPfTask.aigcPfName; + const count = waitTaskID.length; + + if (count > 0) { + // 移除已处理的任务ID + multi.lTrim(info.waitQueue, count, -1); + + // 更新计数映射 + if(taskCountMap.has(key)) { + taskCountMap.set(key, taskCountMap.get(key) + count); + } else { + taskCountMap.set(key, count); + } + } + } + + // 2. 执行Redis命令，移除任务ID + await multi.exec(); + + // 3. 更新平台计数 + if (taskCountMap.size > 0) { + // 减少平台等待队列的待处理任务数 + await initQueue.reducePlatformsWait(taskCountMap); + // 增加平台处理队列的正在处理任务数 + await initQueue.addPlatformsProcess(taskCountMap); + logger.debug('更新任务计数完成'); + } + } catch (error) { + logger.error('更新任务计数失败:', error); + } +} + +// 持续执行批量处理 +(async () => { + while(true) { + try { + // 判断并发数，获取可进行任务处理的等待队列 + const wDeficiency = await judgConcurrency(); + + // 判断是否有可处理的队列 + if(wDeficiency.length > 0) { + // logger.debug('可进行任务处理的等待队列：', wDeficiency); + logger.info('有可进行处理的队列，数量: ' + wDeficiency.length); + console.log(wDeficiency); + + // 批量获取多个等待队列中的任务ID + const tasksWithIds = await getBatchWaitTasksID(wDeficiency); + + // 通过任务ID批量获取多个等待队列中的任务数据 + const tasksData = await getBatchWaitTasks(tasksWithIds); + + // 更新任务计数 - 无论是否获取到任务数据，都需要更新计数，因为任务ID已经从Redis队列中移除 + await updateTaskCounts(tasksWithIds); + + // 将任务发送给生成 Worker 处理 + if (tasksData.length > 0) { + logger.info('发送任务给生成Worker处理，数量: ' + tasksData.length); + generateWorker.postMessage(tasksData); + } + } else { + // 没有可处理的队列，等待10秒后重试 + await new Promise(resolve => setTimeout(resolve, 10000)); + logger.debug('没有可处理的队列'); + } + + } catch (error) { + logger.error('批量处理任务失败:', error); + // 出错后等待5秒再重试 + await new Promise(resolve => setTimeout(resolve, 5000)); + } + } +})(); + +// 监听 generateWorker 的消息 +generateWorker.on('message', (message) => { + if (message.status === 'completed') { + logger.debug('等待任务处理完成'); + } +}); + +generateWorker.on('error', (error) => { + logger.error('生成 Worker 错误:', error); + // 可以考虑重启Worker +}); diff --git a/技术方案文档.md b/技术方案文档.md new file mode 100644 index 0000000..d4d47cf --- /dev/null +++ b/技术方案文档.md @@ -0,0 +1,1208 @@ +# 任务队列后端 - message-dispatcher 优先分发技术方案 + +## 1. 项目概述 + +### 1.1 背景 +当前任务队列后端系统通过 runninghub 等外部平台处理 comfyui 任务。为降低成本、提高响应速度，需要实现内部算力优先使用机制，将 runninghub 类型任务优先路由至内部 message-dispatcher 系统。 + +### 1.2 目标 +- 实现 comfyui 类型任务优先分发至内部 message-dispatcher +- 确保当内部算力不可用时自动降级至外部平台 +- 保持接口兼容性，不影响现有功能 +- 提供完善的健康检查和监控机制 + +## 2. 系统架构分析 + +### 2.1 任务队列后端架构 +``` +任务队列后端 (任务队列后端/) +├── index.js # HTTP 回调服务 (端口 8066) +├── outside/ +│ ├── generat.js # 任务分发逻辑 +│ └── outPlatforms/ +│ ├── outside.js # 平台管理 +│ └── runninghub.js # RunningHub 平台实现 +├── worker_threads/ +│ └── wait/ +│ ├── waiting.js # 任务队列轮询 +│ └── generatTask.js # 任务处理 Worker +└── redis/ # Redis 数据存储 +``` + +### 2.2 message-dispatcher 架构 +``` +message-dispatcher/ +├── src/ +│ ├── index.js # 主服务 (端口 4000) +│ ├── api/ +│ │ └── index.js # REST API 路由 +│ ├── bridge-manager/ # 桥接器管理 +│ └── websocket-server/ # WebSocket 服务 +``` + +### 2.3 关键接口分析 + +**任务队列后端 - 任务流程：** +1. 任务进入 Redis 等待队列 +2. `waiting.js` 轮询获取任务 +3. 调用 `generatTask.js` → `externalPostRequest()` +4. 通过 `outside[runninghub]` 发送至外部平台 + +**message-dispatcher - 任务接口：** +- `POST /api/task` - 提交任务 +- `GET /api/health` - 健康检查 +- `GET /api/overview` - 系统概览 + +## 3. 方案设计 + +### 3.1 总体架构 + +``` +任务队列后端 ──┐ + │ 优先 + ▼ + message-dispatcher (内部算力) + │ + └─ 失败/不可用时 + │ + ▼ + runninghub (外部平台) +``` + +### 3.2 核心模块设计 + +#### 3.2.1 新增模块：内部算力客户端 (InternalDispatcher) + +**文件位置：** `任务队列后端/outside/outPlatforms/messageDispatcher.js` + +**功能：** +- 封装 message-dispatcher API 调用 +- 提供与 runninghub 一致的接口方法 +- 仅修改请求地址和请求头，其他配置与 runninghub 完全一致 + +**接口方法：** +```javascript +getGenerateUrl() // 获取 API 地址 +getGenerateHeader(apikey) // 获取请求头（包含 JWT Token） +getGenerateBody(task) // 构造请求体（与 runninghub 完全一致） +getSuccessTasks(response) // 处理成功响应 +getTaskResult(response) // 处理结果回调 +getQueryUrl() // 获取回调地址 +``` + +#### 3.2.2 健康检查模块 + +**文件位置：** `任务队列后端/utils/healthCheck.js` + +**功能：** +- 定期检查 message-dispatcher 健康状态 +- 维护可用状态缓存 +- 提供可用性查询接口 +- 监控内部算力可用数量 + +**检查项：** +- HTTP 接口连通性 (`/api/health`) +- 可用桥接器数量 +- 在线实例数量（内部算力可用数） + +#### 3.2.3 路由决策模块 + +**修改位置：** `任务队列后端/outside/generat.js` 的 `externalPostRequest()` + +**决策逻辑：** +``` +获取内部算力可用数 (internalCapacity) +获取 runninghub 可容纳上限 (externalCapacity) +总可分发任务上限 = internalCapacity + externalCapacity + +if (待分发任务数 ≤ internalCapacity) { + 全部任务使用内部算力 +} else { + 前 internalCapacity 个任务使用内部算力 + 超出部分使用 runninghub（不超过 externalCapacity） +} +``` + +### 3.3 接口对齐方案 + +#### 3.3.1 配置对齐要求 + +**重要原则：** +- message-dispatcher 配置与 runninghub 保持完全一致 +- 仅修改以下两个参数： + 1. **请求地址**：从 runninghub URL 改为 message-dispatcher URL + 2. **请求头**：将 apiKey 字段值替换为 JWT Token +- 其他所有配置项、参数、字段名称均不做任何改动 + +#### 3.3.2 请求体标准化 + +**请求体必须与 runninghub 完全一致，包括：** + +| 字段名称 | 类型 | 说明 | 是否修改 | +|---------|------|------|---------| +| `workflow_id` | String | 工作流 ID | 否 | +| `node_info_list` | Array | 节点信息列表 | 否 | +| `apiKey` | String | API Key | 是（值改为 JWT Token） | +| `webhookUrl` | String | 回调地址 | 否 | +| 其他所有字段 | - | 保持原样 | 否 | + +**示例对比：** + +```javascript +// runninghub 请求体（原样） +{ + "workflow_id": "123", + "node_info_list": [...], + "apiKey": "runninghub-api-key-xxx", + "webhookUrl": "http://callback-url" +} + +// message-dispatcher 请求体（仅修改 apiKey 值） +{ + "workflow_id": "123", + "node_info_list": [...], + "apiKey": "eyJhbGciOiJIUzI1NiIs...", // JWT Token + "webhookUrl": "http://callback-url" +} +``` + +#### 3.3.3 响应格式转换 + +**message-dispatcher → 任务队列后端 格式映射：** + +```javascript +// message-dispatcher 响应 +{ + success: true, + data: { + requestId: "uuid", + taskId: "..." + } +} + +// 转换为 runninghub 兼容格式 +{ + msg: "success", + code: 0, + data: { + taskId: "requestId" + } +} +``` + +### 3.4 降级机制 + +#### 3.4.1 降级触发条件 +1. message-dispatcher 健康检查失败 +2. 任务提交请求超时 (30秒) +3. 任务提交返回错误 +4. 无可用桥接器/实例（内部算力可用数为0） + +#### 3.4.2 降级流程 +1. 记录降级日志 +2. 标记本次任务使用外部平台 +3. 更新健康检查状态 +4. 继续使用 runninghub 完成任务 + +### 3.5 JWT 认证机制 + +#### 3.5.1 JWT 生成与发送 + +**JWT 获取流程：** + +1. **初始登录**： + - 调用 message-dispatcher 登录接口 + - 接口：`POST /api/auth/login` + - 请求体：`{ username: "admin", password: "2233..2233" }` + - 获取 access_token + +2. **JWT 存储**： + - 存储在内存变量中 + - 同时存储过期时间（expires_at） + +3. **请求头设置**： + - 将 JWT Token 作为 apiKey 字段的值 + - 请求头格式与 runninghub 保持一致 + +#### 3.5.2 JWT 定期更新机制 + +**更新策略：** +- **更新间隔**：20小时（默认 JWT 有效期 24小时，提前4小时更新） +- **更新流程**： + 1. 检查当前 Token 是否在 4 小时内过期 + 2. 如果是，重新调用登录接口获取新 Token + 3. 更新内存中的 Token 和过期时间 + 4. 记录 Token 更新日志 + +**实现代码：** +```javascript +class JWTManager { + constructor() { + this.token = null; + this.expiresAt = null; + this.refreshInterval = 20 * 60 * 60 * 1000; // 20小时 + } + + async init() { + await this.refreshToken(); + setInterval(() => this.refreshToken(), this.refreshInterval); + } + + async refreshToken() { + try { + const response = await fetch(`${MESSAGE_DISPATCHER_URL}/api/auth/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + username: process.env.MD_USERNAME || 'admin', + password: process.env.MD_PASSWORD || '2233..2233' + }) + }); + + const data = await response.json(); + if (data.success && data.data?.access_token) { + this.token = data.data.access_token; + this.expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 假设24小时有效期 + console.log('[JWTManager] Token 已更新'); + } + } catch (error) { + console.error('[JWTManager] Token 更新失败:', error); + } + } + + getToken() { + return this.token; + } + + isTokenValid() { + return this.token && Date.now() < this.expiresAt; + } +} +``` + +### 3.6 任务分流逻辑 + +#### 3.6.1 核心计算公式 + +``` +内部算力可用数 = message-dispatcher 在线实例数 +runninghub 可容纳上限 = 配置的外部平台并发数 +总可分发任务上限 = 内部算力可用数 + runninghub 可容纳上限 +``` + +#### 3.6.2 具体分发策略 + +**场景 1：待分发任务数 ≤ 内部算力可用数** +``` +示例： +内部算力可用数 = 30 +待分发任务数 = 25 + +结果： +全部 25 个任务使用内部算力处理 +``` + +**场景 2：待分发任务数 > 内部算力可用数** +``` +示例： +内部算力可用数 = 30 +runninghub 可容纳上限 = 10 +待分发任务数 = 45 + +结果： +前 30 个任务 → 内部算力 +后 10 个任务 → runninghub（达到上限） +剩余 5 个任务 → 等待队列（等待资源释放） +``` + +#### 3.6.3 分发决策实现 + +**修改位置：** `worker_threads/wait/waiting.js` 和 `worker_threads/wait/generatTask.js` + +**实现逻辑：** +```javascript +async function distributeTasks(tasks) { + const internalCapacity = await getInternalCapacity(); + const externalCapacity = await getExternalCapacity(); + + const internalTasks = tasks.slice(0, internalCapacity); + const externalTasks = tasks.slice(internalCapacity, internalCapacity + externalCapacity); + + // 分发到内部算力 + for (const task of internalTasks) { + await sendToMessageDispatcher(task); + } + + // 分发到外部平台 + for (const task of externalTasks) { + await sendToRunningHub(task); + } + + // 剩余任务返回队列 + const remainingTasks = tasks.slice(internalCapacity + externalCapacity); + return remainingTasks; +} +``` + +### 3.7 配置设计 + +**新增环境变量 (.env)：** +```env +# Message Dispatcher 配置 +MESSAGE_DISPATCHER_URL=http://localhost:4000 +MESSAGE_DISPATCHER_ENABLED=true +MESSAGE_DISPATCHER_TIMEOUT=30000 +MESSAGE_DISPATCHER_HEALTH_CHECK_INTERVAL=10000 + +# Message Dispatcher 认证 +MD_USERNAME=admin +MD_PASSWORD=2233..2233 + +# 容量配置 +INTERNAL_CAPACITY_MAX=30 +EXTERNAL_CAPACITY_MAX=10 +``` + +**新增配置文件 (config/messageDispatcher.json)：** +```json +{ + "enabled": true, + "priority": true, + "healthCheck": { + "interval": 10000, + "timeout": 5000, + "failureThreshold": 3 + }, + "task": { + "timeout": 30000, + "retryCount": 1 + }, + "capacity": { + "internal": 30, + "external": 10 + }, + "jwt": { + "refreshInterval": 72000000 + } +} +``` + +## 4. 实现步骤 + +### 步骤 1: 创建 message-dispatcher 平台适配器 +- 创建 `outside/outPlatforms/messageDispatcher.js` +- 实现与 runninghub.js 相同的接口 +- 请求体与 runninghub 完全一致，仅修改 apiKey 值为 JWT Token + +### 步骤 2: 实现 JWT 认证模块 +- 创建 `utils/jwtManager.js` +- 实现 Token 获取、存储、定期更新逻辑 +- 集成到启动流程 + +### 步骤 3: 实现健康检查模块 +- 创建 `utils/healthCheck.js` +- 实现定期健康检查逻辑 +- 监控内部算力可用数量 + +### 步骤 4: 修改任务分发逻辑 +- 修改 `outside/generat.js` - 单任务分发决策 +- 修改 `worker_threads/wait/waiting.js` - 批量任务分流 +- 根据内部/外部容量进行任务分配 + +### 步骤 5: 更新平台管理 +- 修改 `outside/outPlatforms/outside.js` +- 注册 messageDispatcher 平台 + +### 步骤 6: 添加日志和监控 +- 新增分发决策日志 +- 新增容量使用统计日志 +- 新增降级统计日志 + +### 步骤 7: 测试验证 +- 单元测试 +- 集成测试 +- 降级场景测试 +- 容量边界测试 + +## 5. 验收标准 + +### 5.1 功能验收 +- [ ] 任务分流逻辑正确：≤内部容量全部走内部，超出部分走外部 +- [ ] message-dispatcher 不可用时自动降级至 runninghub +- [ ] 请求体与 runninghub 完全一致（仅 apiKey 值不同） +- [ ] JWT Token 自动更新机制正常工作 +- [ ] 任务成功执行并返回正确结果 +- [ ] 回调接口正常工作 +- [ ] 现有功能不受影响 + +### 5.2 接口兼容性验收 +- [ ] messageDispatcher.js 接口方法签名与 runninghub.js 完全一致 +- [ ] 请求体字段名称、类型、格式与 runninghub 完全一致 +- [ ] 响应格式与 runninghub 兼容 +- [ ] 不修改 runninghub.js 的任何代码 + +### 5.3 性能验收 +- [ ] 健康检查延迟 < 1s +- [ ] 任务分发决策时间 < 100ms +- [ ] 降级切换时间 < 1s +- [ ] 系统整体吞吐量不降低 + +### 5.4 可靠性验收 +- [ ] 连续 24 小时稳定运行 +- [ ] 降级机制在故障场景下正确触发 +- [ ] JWT Token 自动更新不中断服务 +- [ ] 日志完整可追溯 +- [ ] 错误处理完善 + +## 6. 风险与应对 + +| 风险 | 影响 | 概率 | 应对措施 | +|-----|------|------|---------| +| message-dispatcher 接口变更 | 高 | 中 | 版本兼容检查，完善监控 | +| JWT Token 过期 | 高 | 低 | 提前更新，多重校验 | +| 任务格式不匹配 | 高 | 低 | 充分测试，格式验证 | +| 降级失败 | 高 | 低 | 多级降级，超时保护 | +| 性能下降 | 中 | 低 | 性能测试，优化热点路径 | + +## 7. WebSocket 双向通信机制 + +### 7.1 架构设计 + +建立任务队列后端与 message-dispatcher 之间的持久化 WebSocket 连接，实现实时双向通信。 + +``` +任务队列后端 (WebSocket 客户端) + │ + │ 建立连接 + ▼ +message-dispatcher (WebSocket 服务端) +``` + +### 7.2 连接管理 + +#### 7.2.1 连接建立流程 +1. **任务队列后端启动时**： + - 初始化 WebSocket 客户端 + - 连接至 message-dispatcher 的 WebSocket 端点 + - 发送 `CONNECT` 消息进行身份认证 + - 接收 `CONNECT_ACK` 确认连接建立 + +2. **消息格式定义**： +```javascript +// 连接消息 +{ + type: 'CONNECT', + data: { + clientId: 'task-queue-backend', + timestamp: '2024-01-01T00:00:00.000Z' + } +} + +// 连接确认 +{ + type: 'CONNECT_ACK', + data: { + serverId: 'message-dispatcher', + timestamp: '2024-01-01T00:00:00.000Z' + } +} +``` + +#### 7.2.2 心跳保活机制 +- **心跳间隔**：30秒 +- **超时时间**：10秒 +- **重连策略**：指数退避（1s, 2s, 4s, 8s, 16s, 最大32s） + +```javascript +// 心跳消息 +{ + type: 'HEARTBEAT', + data: { + timestamp: '2024-01-01T00:00:00.000Z' + } +} + +// 心跳响应 +{ + type: 'HEARTBEAT_ACK', + data: { + timestamp: '2024-01-01T00:00:00.000Z' + } +} +``` + +### 7.3 算力状态上报 + +#### 7.3.1 上报内容 +message-dispatcher 定期向任务队列后端上报以下信息： + +```javascript +{ + type: 'CAPACITY_UPDATE', + data: { + timestamp: '2024-01-01T00:00:00.000Z', + bridges: [ + { + bridgeId: 'bridge-1', + connectedAt: '2024-01-01T00:00:00.000Z', + instances: [ + { + instanceId: 'instance-1', + status: 'online', // online | busy | offline + currentTask: null | { taskId: 'xxx', startTime: '...' } + } + ] + } + ], + summary: { + totalBridges: 2, + totalInstances: 8, + onlineInstances: 6, + busyInstances: 2, + availableCapacity: 4 // 可同时处理的任务数 + } + } +} +``` + +#### 7.3.2 上报触发条件 +1. **定期上报**：每 10 秒上报一次 +2. **事件触发**： + - 桥接器连接/断开 + - 实例状态变化（online → busy, busy → online, offline） + - 任务开始/完成 + +### 7.4 任务数量下发 + +任务队列后端向 message-dispatcher 下发任务分配信息： + +```javascript +{ + type: 'TASK_ALLOCATION', + data: { + timestamp: '2024-01-01T00:00:00.000Z', + allocatedTasks: [ + { + taskId: 'task-1', + targetInstance: 'instance-1' + } + ] + } +} +``` + +### 7.5 实例状态变化监测 + +#### 7.5.1 变化事件类型 +```javascript +// 实例上线 +{ + type: 'INSTANCE_ONLINE', + data: { + bridgeId: 'bridge-1', + instanceId: 'instance-1', + timestamp: '2024-01-01T00:00:00.000Z' + } +} + +// 实例下线 +{ + type: 'INSTANCE_OFFLINE', + data: { + bridgeId: 'bridge-1', + instanceId: 'instance-1', + timestamp: '2024-01-01T00:00:00.000Z', + reason: 'disconnected' + } +} + +// 实例繁忙 +{ + type: 'INSTANCE_BUSY', + data: { + bridgeId: 'bridge-1', + instanceId: 'instance-1', + taskId: 'task-1', + timestamp: '2024-01-01T00:00:00.000Z' + } +} + +// 实例空闲 +{ + type: 'INSTANCE_IDLE', + data: { + bridgeId: 'bridge-1', + instanceId: 'instance-1', + completedTaskId: 'task-1', + timestamp: '2024-01-01T00:00:00.000Z' + } +} +``` + +--- + +## 8. Redis 状态管理机制 + +### 8.1 Redis 数据结构设计 + +#### 8.1.1 连接状态存储 + +**Key 前缀：** `md:state:` + +| Key | 类型 | 说明 | TTL | +|-----|------|------|-----| +| `md:state:connection` | Hash | 连接状态信息 | 永久 | +| `md:state:capacity` | Hash | 算力容量信息 | 60s | +| `md:state:instances` | Hash | 实例详细状态 | 60s | +| `md:state:last_heartbeat` | String | 最后心跳时间 | 60s | + +**connection Hash 字段：** +```javascript +{ + status: 'connected', // connected | disconnected + connectedAt: '2024-01-01T00:00:00.000Z', + disconnectedAt: null, + disconnectReason: null, + reconnectAttempts: 0 +} +``` + +**capacity Hash 字段：** +```javascript +{ + totalBridges: 2, + totalInstances: 8, + onlineInstances: 6, + busyInstances: 2, + availableCapacity: 4, + updatedAt: '2024-01-01T00:00:00.000Z' +} +``` + +**instances Hash 结构：** +Key: `md:state:instances:{instanceId}` +```javascript +{ + instanceId: 'instance-1', + bridgeId: 'bridge-1', + status: 'online', + currentTaskId: null, + currentTaskStartTime: null, + lastUpdated: '2024-01-01T00:00:00.000Z' +} +``` + +### 8.2 健康检查机制 + +#### 8.2.1 检查维度 + +1. **WebSocket 连接检查** + - 检查连接状态是否为 `connected` + - 检查最后心跳时间是否在 60 秒内 + +2. **算力可用性检查** + - 检查在线实例数量 > 0 + - 检查可用容量 > 0 + +3. **综合健康评分** +``` +健康状态 = 连接状态 × 0.4 + 算力状态 × 0.6 +``` + +#### 8.2.2 健康检查流程 + +```javascript +async function checkHealth() { + const state = await getConnectionState(); + const capacity = await getCapacity(); + + const isConnected = state.status === 'connected'; + const hasHeartbeat = Date.now() - state.lastHeartbeat < 60000; + const hasCapacity = capacity.availableCapacity > 0; + + const isHealthy = isConnected && hasHeartbeat && hasCapacity; + + if (!isHealthy) { + await triggerDegradation(); + } + + return isHealthy; +} +``` + +### 8.3 自动降级触发 + +#### 8.3.1 降级触发条件 + +| 条件 | 说明 | 优先级 | +|-----|------|--------| +| WebSocket 断开 | 连接完全断开 | 1 | +| 心跳超时 (>60s) | 连接可能异常 | 2 | +| 无可用实例 | 在线实例数为0 | 3 | +| 可用容量为0 | 所有实例都忙 | 4 | + +#### 8.3.2 降级流程 + +1. **检测到异常** +2. **更新 Redis 状态**：设置 `status: 'degraded'` +3. **记录降级日志**：记录降级时间、原因、当前状态 +4. **通知路由模块**：触发路由策略切换 +5. **持续监控**：定期检查是否恢复 + +#### 8.3.3 自动恢复流程 + +1. **健康检查通过** +2. **等待稳定期**：连续 3 次健康检查通过（30秒） +3. **更新 Redis 状态**：设置 `status: 'healthy'` +4. **记录恢复日志** +5. **通知路由模块**：恢复优先使用内部算力 + +--- + +## 9. Message-Dispatcher 任务缓存与调度机制 + +### 9.1 缓存队列设计 + +#### 9.1.1 缓存队列数据结构 + +使用 Redis List 实现： + +| Key | 类型 | 说明 | +|-----|------|------| +| `md:cache:pending` | List | 待执行任务缓存 | +| `md:cache:processing` | Hash | 执行中任务 | +| `md:cache:completed` | List | 已完成任务（最近1000条） | + +**pending List 元素格式：** +```javascript +{ + taskId: 'xxx', + payload: {...}, + submittedAt: '2024-01-01T00:00:00.000Z', + retryCount: 0 +} +``` + +**processing Hash 字段：** +```javascript +{ + taskId: { + instanceId: 'instance-1', + bridgeId: 'bridge-1', + startTime: '2024-01-01T00:00:00.000Z', + payload: {...} + } +} +``` + +### 9.2 任务调度器 + +#### 9.2.1 调度器架构 + +``` +调度器主循环 + │ + ├─ 获取可用实例 + │ + ├─ 从缓存队列取出任务 + │ + ├─ 分配任务至实例 + │ + └─ 移动至 processing +``` + +#### 9.2.2 调度算法 + +```javascript +async function scheduleTasks() { + const availableInstances = await getAvailableInstances(); + const pendingTasks = await getPendingTasks(availableInstances.length); + + for (let i = 0; i < pendingTasks.length; i++) { + const task = pendingTasks[i]; + const instance = availableInstances[i]; + + await assignTask(task, instance); + } +} +``` + +#### 9.2.3 实例选择策略 + +1. **最少任务优先**：选择当前任务数最少的实例 +2. **同桥接优先**：优先选择同一桥接器下的实例 +3. **随机打散**：避免总是选择同一实例 + +```javascript +function selectOptimalInstance(availableInstances, task) { + // 按当前任务数排序 + const sorted = [...availableInstances].sort((a, b) => + a.currentTasks - b.currentTasks + ); + + // 如果有任务指定偏好实例 + if (task.preferredInstanceId) { + const preferred = sorted.find(i => i.id === task.preferredInstanceId); + if (preferred) return preferred; + } + + return sorted[0]; +} +``` + +### 9.3 任务完成监听 + +#### 9.3.1 完成事件处理 + +```javascript +async function handleTaskComplete(taskId, result) { + // 1. 从 processing 移除 + const processingInfo = await removeFromProcessing(taskId); + + // 2. 标记实例为空闲 + await markInstanceIdle(processingInfo.instanceId); + + // 3. 添加到 completed + await addToCompleted(taskId, result); + + // 4. 触发新一轮调度 + await scheduleTasks(); + + // 5. 调用回调（如果有） + if (processingInfo.webhookUrl) { + await invokeWebhook(processingInfo.webhookUrl, result); + } +} +``` + +#### 9.3.2 失败重试机制 + +```javascript +async function handleTaskFailure(taskId, error) { + const task = await getTask(taskId); + const maxRetries = 3; + + if (task.retryCount < maxRetries) { + task.retryCount++; + task.lastError = error.message; + await returnToPending(task); + } else { + await markAsFailed(taskId, error); + } +} +``` + +### 9.4 缓存队列管理 + +#### 9.4.1 队列扩容/缩容 + +当实例数量变化时： + +```javascript +async function adjustCacheQueue(instanceDelta) { + const currentPending = await getPendingCount(); + const currentCapacity = await getAvailableCapacity(); + + if (instanceDelta > 0) { + // 实例增加，立即调度 + await scheduleTasks(); + } else if (instanceDelta < 0) { + // 实例减少，检查是否需要回退任务 + const excess = currentPending - currentCapacity; + if (excess > 0) { + // 通知任务队列后端，这些任务需要降级 + await notifyTaskQueueForDegradation(excess); + } + } +} +``` + +#### 9.4.2 超时清理 + +```javascript +async function cleanStaleTasks() { + const now = Date.now(); + const staleThreshold = 30 * 60 * 1000; // 30分钟 + + const pendingTasks = await getAllPendingTasks(); + + for (const task of pendingTasks) { + if (now - task.submittedAt > staleThreshold) { + await removeFromPending(task.taskId); + await markAsExpired(task.taskId); + } + } +} +``` + +--- + +## 10. 任务队列逻辑健壮性测试方案 + +### 10.1 测试场景设计 + +#### 10.1.1 场景1：内部算力减少 + +**测试目的：** 验证当内部算力减少时，系统能否正确处理已下发任务 + +**测试步骤：** +1. 初始状态：4个在线实例，下发4个任务 +2. 操作：突然断开2个实例 +3. 预期结果： + - 未开始的任务重新回到缓存队列 + - 正在运行的任务继续执行（如果实例还在） + - 系统记录日志，不崩溃 + - 自动调度可用实例处理缓存任务 + +**测试用例代码：** +```javascript +async function testCapacityReduction() { + // 1. 准备环境 + await setupTestEnvironment({ onlineInstances: 4 }); + + // 2. 下发任务 + const taskIds = await submitTasks(4); + + // 3. 确认任务开始执行 + await waitForTasksRunning(taskIds); + + // 4. 断开2个实例 + await disconnectInstances(2); + + // 5. 验证 + const remainingTasks = await getRemainingTasks(); + assert(remainingTasks.length <= 2, '不应有超过2个任务继续执行'); + + const cacheQueue = await getCacheQueue(); + assert(cacheQueue.length >= 2, '应有至少2个任务回到缓存'); + + const systemHealth = await checkSystemHealth(); + assert(systemHealth.status === 'degraded', '系统应处于降级状态'); + assert(systemHealth.crashed === false, '系统不应崩溃'); +} +``` + +#### 10.1.2 场景2：任务数 > 可用实例数 + +**测试目的：** 验证当上一次下发任务数大于当前可用任务数时的系统行为 + +**测试步骤：** +1. 初始状态：4个在线实例 +2. 操作1：下发8个任务 +3. 预期1：4个开始执行，4个进入缓存 +4. 操作2：断开2个实例（仅剩2个） +5. 预期2： + - 缓存队列保持4个任务 + - 正在执行的任务继续 + - 完成后只调度2个新任务 + - 不发生任务丢失 + +**验证点：** +- [ ] 任务总数始终为8（执行中 + 缓存） +- [ ] 没有任务重复执行 +- [ ] 没有任务丢失 +- [ ] 最终所有任务都能完成（可能部分降级） + +#### 10.1.3 场景3：容量边界测试 + +**测试目的：** 验证任务分流逻辑在各种容量边界下的正确性 + +**测试用例：** + +| 用例 | 内部容量 | 外部容量 | 待分发任务数 | 预期结果 | +|-----|---------|---------|------------|---------| +| 1 | 30 | 10 | 25 | 全部25个走内部 | +| 2 | 30 | 10 | 30 | 全部30个走内部 | +| 3 | 30 | 10 | 35 | 30个内部，5个外部 | +| 4 | 30 | 10 | 40 | 30个内部，10个外部 | +| 5 | 30 | 10 | 45 | 30个内部，10个外部，5个等待 | +| 6 | 0 | 10 | 5 | 全部5个走外部 | +| 7 | 30 | 0 | 35 | 30个内部，5个等待 | + +### 10.2 潜在错误检查清单 + +#### 10.2.1 并发安全检查 + +| 检查项 | 风险 | 验证方法 | +|-------|------|---------| +| Redis 操作原子性 | 高 | 使用 MULTI/EXEC 事务 | +| 任务重复下发 | 高 | 任务 ID 幂等性检查 | +| 状态竞态条件 | 中 | 使用 Redis 锁保护关键操作 | +| 内存泄漏 | 中 | 长时间运行监控内存使用 | + +#### 10.2.2 边界条件检查 + +| 边界条件 | 测试方法 | 预期结果 | +|---------|---------|---------| +| 0 个实例 | 断开所有实例 | 全部任务降级，系统不崩溃 | +| 瞬间涌入1000任务 | 批量提交任务 | 队列可控，按容量分流 | +| 网络闪断 | 模拟网络抖动 | 自动重连，任务不丢失 | +| message-dispatcher 重启 | 重启服务 | 任务状态恢复，继续执行 | +| JWT Token 过期 | 模拟 Token 过期 | 自动获取新 Token，服务不中断 | + +### 10.3 错误处理与恢复机制建议 + +#### 10.3.1 错误分类与处理策略 + +| 错误类型 | 严重程度 | 处理策略 | +|---------|---------|---------| +| 网络超时 | 低 | 重试（最多3次） | +| 实例离线 | 中 | 任务回退至缓存，重新调度 | +| Redis 连接失败 | 高 | 立即降级，使用本地缓存 | +| JWT Token 失效 | 中 | 立即刷新 Token | +| 内存溢出 | 严重 | 紧急降级，停止接收新任务 | +| 未知异常 | 中 | 记录日志，降级处理 | + +#### 10.3.2 熔断机制 + +```javascript +class CircuitBreaker { + constructor(options) { + this.failureThreshold = options.failureThreshold || 5; + this.resetTimeout = options.resetTimeout || 30000; + this.failureCount = 0; + this.state = 'closed'; // closed, open, half-open + this.lastFailureTime = null; + } + + async execute(fn) { + if (this.state === 'open') { + if (Date.now() - this.lastFailureTime > this.resetTimeout) { + this.state = 'half-open'; + } else { + throw new Error('Circuit breaker is open'); + } + } + + try { + const result = await fn(); + this.onSuccess(); + return result; + } catch (error) { + this.onFailure(); + throw error; + } + } + + onSuccess() { + this.failureCount = 0; + this.state = 'closed'; + } + + onFailure() { + this.failureCount++; + this.lastFailureTime = Date.now(); + + if (this.failureCount >= this.failureThreshold) { + this.state = 'open'; + } + } +} +``` + +#### 10.3.3 状态快照与恢复 + +定期保存系统状态快照： + +```javascript +async function saveStateSnapshot() { + const snapshot = { + timestamp: new Date().toISOString(), + connectionState: await getConnectionState(), + capacity: await getCapacity(), + pendingTasks: await getPendingTasks(100), + processingTasks: await getProcessingTasks() + }; + + await redis.set('md:snapshot:latest', JSON.stringify(snapshot)); + await redis.lPush('md:snapshot:history', JSON.stringify(snapshot)); + await redis.lTrim('md:snapshot:history', 0, 99); // 保留最近100个 +} + +async function restoreFromSnapshot() { + const snapshotData = await redis.get('md:snapshot:latest'); + if (!snapshotData) return; + + const snapshot = JSON.parse(snapshotData); + await restoreConnectionState(snapshot.connectionState); + await logRecovery(snapshot.timestamp); +} +``` + +#### 10.3.4 监控告警建议 + +| 监控指标 | 告警阈值 | 告警级别 | +|---------|---------|---------| +| 降级率 | > 50% | 严重 | +| 任务超时率 | > 10% | 警告 | +| 缓存队列长度 | > 100 | 警告 | +| 系统崩溃次数 | > 0 | 严重 | +| JWT Token 更新失败 | > 3次/小时 | 警告 | +| 内部容量使用率 | > 90% | 警告 | + +--- + +## 11. 验证方法 + +### 11.1 接口兼容性验证 + +**验证清单：** +- [ ] messageDispatcher.js 与 runninghub.js 拥有完全相同的方法签名 +- [ ] getGenerateBody() 返回的请求体字段名称与 runninghub 完全一致 +- [ ] getGenerateBody() 返回的请求体字段类型与 runninghub 完全一致 +- [ ] 仅 apiKey 字段的值被修改为 JWT Token +- [ ] 响应格式与 runninghub 兼容 + +**验证命令：** +```javascript +// 对比两个平台适配器的接口 +const runninghubMethods = Object.keys(runninghub); +const messageDispatcherMethods = Object.keys(messageDispatcher); +assert.deepEqual(runninghubMethods, messageDispatcherMethods, '接口方法必须一致'); +``` + +### 11.2 任务分流验证 + +**验证场景：** +1. 构造不同数量的待分发任务 +2. 检查内部/外部任务分配比例 +3. 验证不超过各自容量上限 + +**验证代码：** +```javascript +async function verifyTaskDistribution() { + const testCases = [ + { internal: 30, external: 10, total: 25, expectedInternal: 25, expectedExternal: 0 }, + { internal: 30, external: 10, total: 35, expectedInternal: 30, expectedExternal: 5 }, + { internal: 30, external: 10, total: 45, expectedInternal: 30, expectedExternal: 10 } + ]; + + for (const testCase of testCases) { + const result = await distributeTasks(testCase); + assert(result.internalCount === testCase.expectedInternal); + assert(result.externalCount === testCase.expectedExternal); + } +} +``` + +### 11.3 JWT 认证验证 + +**验证清单：** +- [ ] 系统启动时成功获取 JWT Token +- [ ] Token 正确存储在内存中 +- [ ] 请求头中 apiKey 字段值为 JWT Token +- [ ] Token 在过期前自动更新 +- [ ] Token 更新不影响正在进行的任务 + +--- + +## 12. 后续优化方向 + +1. **智能负载均衡** - 根据内部算力负载动态调整分发策略 +2. **任务缓存** - 缓存常用工作流，减少重复提交 +3. **批量处理** - 支持任务批量提交，提高吞吐量 +4. **可视化监控** - 新增分发统计 dashboard +5. **A/B 测试** - 支持新旧方案并行，逐步切换 +6. **预测性调度** - 基于历史数据预测任务完成时间 +7. **多区域部署** - 支持跨区域算力调度 +8. **成本优化** - 综合考虑内部成本与外部成本