126 lines
3.7 KiB
JavaScript
126 lines
3.7 KiB
JavaScript
import { parentPort } from 'worker_threads';
|
||
import redis from '../../redis/index.js'
|
||
import initQueue from '../../redis/initQueue.js'
|
||
|
||
// 日志工具函数
|
||
const logger = {
|
||
info: (message) => {
|
||
const timestamp = new Date().toISOString();
|
||
console.log(`[${timestamp}] INFO: ${message}`);
|
||
},
|
||
error: (message, error) => {
|
||
const timestamp = new Date().toISOString();
|
||
console.error(`[${timestamp}] ERROR: ${message}`, error || '');
|
||
},
|
||
debug: (message) => {
|
||
const timestamp = new Date().toISOString();
|
||
console.debug(`[${timestamp}] DEBUG: ${message}`);
|
||
}
|
||
};
|
||
|
||
// 批量获取错误任务的信息
|
||
async function getTasks() {
|
||
try {
|
||
const taskIds = await redis.lRange(initQueue.errorList, 0, -1);
|
||
|
||
if (taskIds.length === 0) {
|
||
return true;
|
||
}
|
||
|
||
logger.debug('错误队列任务ID:', taskIds);
|
||
|
||
// 批量获取错误任务信息
|
||
const multi = redis.multi();
|
||
for (const taskId of taskIds) {
|
||
multi.hGetAll(`${initQueue.prefix}:task:${taskId}`);
|
||
}
|
||
|
||
const results = await multi.exec();
|
||
|
||
let processedCount = 0;
|
||
const taskCountMap = new Map();
|
||
|
||
// 处理结果
|
||
for (let i = 0; i < taskIds.length; i++) {
|
||
const taskId = taskIds[i];
|
||
const taskInfo = results[i];
|
||
|
||
if (taskInfo && taskInfo.taskId && taskInfo.resultData) {
|
||
try {
|
||
logger.debug('错误队列任务数据:', taskInfo);
|
||
|
||
// 直接打包错误信息和任务ID，发送给主线程
|
||
const resultWithTaskId = {
|
||
taskId: taskInfo.taskId,
|
||
result: taskInfo.resultData
|
||
};
|
||
|
||
parentPort.postMessage({
|
||
type: 'error',
|
||
backendId: taskInfo.backendId,
|
||
message: JSON.stringify(resultWithTaskId)
|
||
});
|
||
processedCount++;
|
||
|
||
// 统计需要减少计数的任务
|
||
const key = `${taskInfo.AIGC}:${taskInfo.platform}`;
|
||
if(taskCountMap.has(key)){
|
||
taskCountMap.set(key, taskCountMap.get(key) + 1);
|
||
} else {
|
||
taskCountMap.set(key, 1);
|
||
}
|
||
} catch (parseError) {
|
||
logger.error(`解析错误任务数据失败: ${taskInfo.resultData}`, parseError);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 删除已处理的错误任务
|
||
if (processedCount > 0) {
|
||
const deleteMulti = redis.multi();
|
||
for (const taskId of taskIds) {
|
||
deleteMulti.lRem(initQueue.errorList, 1, taskId);
|
||
// 同时删除tasks中的任务信息（从哈希存储中删除），使用项目前缀
|
||
deleteMulti.del(`${initQueue.prefix}:task:${taskId}`);
|
||
}
|
||
await deleteMulti.exec();
|
||
|
||
await initQueue.reduceEQtaskALL(processedCount);
|
||
logger.info(`处理了 ${processedCount} 个错误任务`);
|
||
|
||
// 减少等待队列的计数（错误任务来自等待队列）
|
||
if (taskCountMap.size > 0) {
|
||
await initQueue.reducePlatformsWait(taskCountMap);
|
||
}
|
||
}
|
||
|
||
return true;
|
||
} catch (error) {
|
||
logger.error('获取错误任务失败:', error);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
// 持续执行批量处理
|
||
(async () => {
|
||
while (true) {
|
||
try {
|
||
const errorTasksCount = await initQueue.getEQtaskALL();
|
||
|
||
// 判断是否有可处理的错误任务
|
||
if (errorTasksCount > 0) {
|
||
logger.info('错误队列有可处理任务，数量:', errorTasksCount);
|
||
await getTasks();
|
||
} else {
|
||
// 没有可处理的错误任务，等待15秒后重试
|
||
await new Promise(resolve => setTimeout(resolve, 15000));
|
||
logger.debug('错误队列无任务可处理');
|
||
}
|
||
} catch (error) {
|
||
logger.error('持续处理错误任务失败:', error);
|
||
// 出错后等待5秒再重试
|
||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||
}
|
||
}
|
||
})()
|