|
@@ -2,17 +2,13 @@
|
|
|
|
|
|
|
|
import dayjs from "dayjs";
|
|
import dayjs from "dayjs";
|
|
|
import doneRateService from "../../src/services/doneRateService"; // 导入 DoneRateService
|
|
import doneRateService from "../../src/services/doneRateService"; // 导入 DoneRateService
|
|
|
-import artService from "../../src/services/artService"; // 👈 导入 ArtService
|
|
|
|
|
import { clickhouseService } from "../../src/services/clients";
|
|
import { clickhouseService } from "../../src/services/clients";
|
|
|
import mongoose, { Connection } from "mongoose"; // 导入 mongoose 和 Connection 用于处理远程连接
|
|
import mongoose, { Connection } from "mongoose"; // 导入 mongoose 和 Connection 用于处理远程连接
|
|
|
-import Art, { IArt } from "../../src/models/artModel"; // 👈 导入 Art 模型和 IArt 接口
|
|
|
|
|
|
|
+import TotalDoneRate from "../../src/models/totalDoneRateModel"; // 导入 TotalDoneRate 模型 (已包含 totalTipCount)
|
|
|
|
|
|
|
|
// ClickHouse 表名
|
|
// ClickHouse 表名
|
|
|
const CLICKHOUSE_EVENTS_TABLE = "events"; // 确保与 ClickHouseService 中的表名一致
|
|
const CLICKHOUSE_EVENTS_TABLE = "events"; // 确保与 ClickHouseService 中的表名一致
|
|
|
|
|
|
|
|
-// 远程数据库连接 URL
|
|
|
|
|
-const REMOTE_MONGO_URI = "mongodb://coloring:coloring123.@hk.jccytech.cn:7881/coloring_ol?authSource=admin";
|
|
|
|
|
-
|
|
|
|
|
/**
|
|
/**
|
|
|
* ClickHouse 查询结果接口:每日每个作品的独立开始用户数
|
|
* ClickHouse 查询结果接口:每日每个作品的独立开始用户数
|
|
|
*/
|
|
*/
|
|
@@ -29,10 +25,18 @@ interface ClickHouseDoneCountResult {
|
|
|
unique_dones: number; // 独立完成用户数
|
|
unique_dones: number; // 独立完成用户数
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+/**
|
|
|
|
|
+ * ClickHouse 查询结果接口:每日每个作品的使用道具数 (对应 tipCount)
|
|
|
|
|
+ */
|
|
|
|
|
+interface ClickHouseTipCountResult {
|
|
|
|
|
+ res: string; // 作品 ID
|
|
|
|
|
+ tip_count: number; // 道具使用次数
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
/**
|
|
/**
|
|
|
* 每日统计昨天的作品完成率。
|
|
* 每日统计昨天的作品完成率。
|
|
|
- * 统计逻辑从 ClickHouse 中提取数据,得到每个作品的完成情况,并更新到 doneRateModel。
|
|
|
|
|
- * 随后,根据这些日统计数据,累加更新本地和远程的 Art 表的总统计字段。
|
|
|
|
|
|
|
+ * 统计逻辑从 ClickHouse 中提取数据,得到每个作品的完成情况、道具使用情况,并更新到 doneRateModel。
|
|
|
|
|
+ * 随后,根据这些日统计数据,累加更新本地的 totalDoneRate 表 (包括 totalTipCount)。
|
|
|
* @returns Promise<string> - 返回统计结果的摘要信息。
|
|
* @returns Promise<string> - 返回统计结果的摘要信息。
|
|
|
*/
|
|
*/
|
|
|
async function run(): Promise<string> {
|
|
async function run(): Promise<string> {
|
|
@@ -50,13 +54,10 @@ async function run(): Promise<string> {
|
|
|
|
|
|
|
|
console.log(`[DoneRate Cron] Processing data for date: ${yesterdayYYYYMMDD}`);
|
|
console.log(`[DoneRate Cron] Processing data for date: ${yesterdayYYYYMMDD}`);
|
|
|
|
|
|
|
|
- let remoteConn: Connection | null = null;
|
|
|
|
|
- let updatedRemoteArtworksCount = 0;
|
|
|
|
|
-
|
|
|
|
|
try {
|
|
try {
|
|
|
- // --- 1. 从 ClickHouse 中提取数据 ---
|
|
|
|
|
|
|
+ // --- 1. 从 ClickHouse 中提取数据 (Start, Done, Tip Counts) ---
|
|
|
|
|
|
|
|
- // 查询昨天每个作品的独立开始用户数
|
|
|
|
|
|
|
+ // 1.1 查询昨天每个作品的独立开始用户数
|
|
|
const startCountsQuery = `
|
|
const startCountsQuery = `
|
|
|
SELECT
|
|
SELECT
|
|
|
res,
|
|
res,
|
|
@@ -79,7 +80,7 @@ async function run(): Promise<string> {
|
|
|
});
|
|
});
|
|
|
console.log(`[DoneRate Cron] Retrieved ${startResults.length} unique start counts from ClickHouse.`);
|
|
console.log(`[DoneRate Cron] Retrieved ${startResults.length} unique start counts from ClickHouse.`);
|
|
|
|
|
|
|
|
- // 查询昨天每个作品的独立完成用户数
|
|
|
|
|
|
|
+ // 1.2 查询昨天每个作品的独立完成用户数
|
|
|
const doneCountsQuery = `
|
|
const doneCountsQuery = `
|
|
|
SELECT
|
|
SELECT
|
|
|
res,
|
|
res,
|
|
@@ -102,31 +103,44 @@ async function run(): Promise<string> {
|
|
|
});
|
|
});
|
|
|
console.log(`[DoneRate Cron] Retrieved ${doneResults.length} unique done counts from ClickHouse.`);
|
|
console.log(`[DoneRate Cron] Retrieved ${doneResults.length} unique done counts from ClickHouse.`);
|
|
|
|
|
|
|
|
- // --- 2. 合并数据并更新 DoneRate 模型 ---
|
|
|
|
|
- let updatedRecordsCount = 0; // for DoneRate
|
|
|
|
|
- let createdRecordsCount = 0; // for DoneRate
|
|
|
|
|
-
|
|
|
|
|
- // 遍历所有有开始事件的作品ID
|
|
|
|
|
- for (const [resIdStr, startCount] of artworkStartCounts.entries()) {
|
|
|
|
|
- const doneCount = artworkDoneCounts.get(resIdStr) || 0;
|
|
|
|
|
- const resObjectId = new mongoose.Types.ObjectId(resIdStr);
|
|
|
|
|
-
|
|
|
|
|
- // 使用 DoneRateService 来创建或更新记录
|
|
|
|
|
- const doneRateDoc = await doneRateService.createOrUpdateDoneRate(yesterdayYYYYMMDD, resObjectId, startCount, doneCount);
|
|
|
|
|
- if (doneRateDoc.createdAt.getTime() === doneRateDoc.updatedAt.getTime()) {
|
|
|
|
|
- createdRecordsCount++;
|
|
|
|
|
|
|
+ // 1.3 查询昨天每个作品的使用道具数
|
|
|
|
|
+ const tipCountsQuery = `
|
|
|
|
|
+ SELECT
|
|
|
|
|
+ res,
|
|
|
|
|
+ count() AS tip_count
|
|
|
|
|
+ FROM ${CLICKHOUSE_EVENTS_TABLE}
|
|
|
|
|
+ WHERE event = 'color_tip'
|
|
|
|
|
+ AND time >= toDateTime('${yesterdayStartString}')
|
|
|
|
|
+ AND time < toDateTime('${yesterdayEndString}')
|
|
|
|
|
+ GROUP BY res
|
|
|
|
|
+ HAVING res IS NOT NULL
|
|
|
|
|
+ `;
|
|
|
|
|
+ const tipResults = await clickhouseService.queryEvents<ClickHouseTipCountResult>(tipCountsQuery);
|
|
|
|
|
+ const artworkTipCounts = new Map<string, number>();
|
|
|
|
|
+ tipResults.forEach((row) => {
|
|
|
|
|
+ if (row.res && mongoose.Types.ObjectId.isValid(row.res)) {
|
|
|
|
|
+ artworkTipCounts.set(row.res, row.tip_count);
|
|
|
} else {
|
|
} else {
|
|
|
- updatedRecordsCount++;
|
|
|
|
|
|
|
+ console.warn(`[DoneRate Cron] Invalid artwork ID found in tip_counts result: ${row.res}`);
|
|
|
}
|
|
}
|
|
|
- artworkDoneCounts.delete(resIdStr); // 已经处理过的作品ID从 doneCounts 中移除
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ });
|
|
|
|
|
+ console.log(`[DoneRate Cron] Retrieved ${tipResults.length} unique tip counts from ClickHouse.`);
|
|
|
|
|
+
|
|
|
|
|
+ // --- 2. 合并数据并更新 DoneRate 模型 (每日记录) ---
|
|
|
|
|
+ let updatedRecordsCount = 0;
|
|
|
|
|
+ let createdRecordsCount = 0;
|
|
|
|
|
+
|
|
|
|
|
+ // 获取所有需要处理的作品 ID 集合 (Start + Done + Tip)
|
|
|
|
|
+ const allResIds = new Set([...artworkStartCounts.keys(), ...artworkDoneCounts.keys(), ...artworkTipCounts.keys()]);
|
|
|
|
|
|
|
|
- // 处理只有完成事件但没有开始事件的作品 (通常不应发生,但以防万一)
|
|
|
|
|
- for (const [resIdStr, doneCount] of artworkDoneCounts.entries()) {
|
|
|
|
|
- const startCount = 0; // 没有开始事件,所以开始次数为0
|
|
|
|
|
|
|
+ for (const resIdStr of allResIds.values()) {
|
|
|
|
|
+ const startCount = artworkStartCounts.get(resIdStr) || 0;
|
|
|
|
|
+ const doneCount = artworkDoneCounts.get(resIdStr) || 0;
|
|
|
|
|
+ const tipCount = artworkTipCounts.get(resIdStr) || 0;
|
|
|
const resObjectId = new mongoose.Types.ObjectId(resIdStr);
|
|
const resObjectId = new mongoose.Types.ObjectId(resIdStr);
|
|
|
|
|
|
|
|
- const doneRateDoc = await doneRateService.createOrUpdateDoneRate(yesterdayYYYYMMDD, resObjectId, startCount, doneCount);
|
|
|
|
|
|
|
+ // 使用 DoneRateService 来创建或更新记录,并传入 tipCount
|
|
|
|
|
+ const doneRateDoc = await doneRateService.createOrUpdateDoneRate(yesterdayYYYYMMDD, resObjectId, startCount, doneCount, tipCount);
|
|
|
if (doneRateDoc.createdAt.getTime() === doneRateDoc.updatedAt.getTime()) {
|
|
if (doneRateDoc.createdAt.getTime() === doneRateDoc.updatedAt.getTime()) {
|
|
|
createdRecordsCount++;
|
|
createdRecordsCount++;
|
|
|
} else {
|
|
} else {
|
|
@@ -136,97 +150,85 @@ async function run(): Promise<string> {
|
|
|
|
|
|
|
|
const totalProcessedArtworks = createdRecordsCount + updatedRecordsCount;
|
|
const totalProcessedArtworks = createdRecordsCount + updatedRecordsCount;
|
|
|
|
|
|
|
|
- console.log(`[DoneRate Cron] DoneRate model update completed. Total artworks processed: ${totalProcessedArtworks}. Created: ${createdRecordsCount}, Updated: ${updatedRecordsCount}.`);
|
|
|
|
|
-
|
|
|
|
|
- // --- 3. 获取昨天的所有 DoneRate 记录,并更新本地和远程的 Art 表 ---
|
|
|
|
|
|
|
+ console.log(`[DoneRate Cron] DoneRate model (Daily) update completed. Total artworks processed: ${totalProcessedArtworks}. Created: ${createdRecordsCount}, Updated: ${updatedRecordsCount}.`);
|
|
|
|
|
|
|
|
- // 建立远程数据库连接和模型
|
|
|
|
|
- remoteConn = await mongoose.createConnection(REMOTE_MONGO_URI);
|
|
|
|
|
- const RemoteArt = remoteConn.model<IArt>("Art", Art.schema);
|
|
|
|
|
- console.log(`[DoneRate Cron] Connected to remote database.`);
|
|
|
|
|
|
|
+ // --- 3. 获取昨天的 DoneRate 记录,并更新本地的 TotalDoneRate 表 (累计记录) ---
|
|
|
|
|
|
|
|
- let updatedLocalArtworksCount = 0; // for Art model
|
|
|
|
|
|
|
+ let updatedTotalDoneRateCount = 0;
|
|
|
|
|
+ // 仅获取昨天更新或创建的记录,以保证数据源为最新
|
|
|
const yesterdayDoneRates = await doneRateService.getDoneRatesByDate(yesterdayYYYYMMDD);
|
|
const yesterdayDoneRates = await doneRateService.getDoneRatesByDate(yesterdayYYYYMMDD);
|
|
|
- console.log(`[DoneRate Cron] Found ${yesterdayDoneRates.length} DoneRate records for yesterday to update Art table.`);
|
|
|
|
|
|
|
|
|
|
- // ============= 新增日志和进度追踪逻辑 =============
|
|
|
|
|
- const totalDoneRates = yesterdayDoneRates.length;
|
|
|
|
|
- let remoteUpdateStartTime = new Date().getTime();
|
|
|
|
|
- console.log(`[DoneRate Cron] 开始更新远程 Art 表。总计 ${totalDoneRates} 条记录。`);
|
|
|
|
|
- // ===================================================
|
|
|
|
|
|
|
+ const totalDoneRatesToUpdate = yesterdayDoneRates.length;
|
|
|
|
|
+ let updateStartTime = new Date().getTime();
|
|
|
|
|
+ console.log(`[DoneRate Cron] 开始更新本地 TotalDoneRate 表。总计 ${totalDoneRatesToUpdate} 条记录。`);
|
|
|
|
|
|
|
|
- for (let i = 0; i < totalDoneRates; i++) {
|
|
|
|
|
|
|
+ for (let i = 0; i < totalDoneRatesToUpdate; i++) {
|
|
|
const doneRateDoc = yesterdayDoneRates[i];
|
|
const doneRateDoc = yesterdayDoneRates[i];
|
|
|
try {
|
|
try {
|
|
|
const artworkId = doneRateDoc.res; // 获取作品 ObjectId
|
|
const artworkId = doneRateDoc.res; // 获取作品 ObjectId
|
|
|
- const currentArt = await artService.getArtById(artworkId.toString());
|
|
|
|
|
-
|
|
|
|
|
- if (currentArt) {
|
|
|
|
|
- // 累加总开始数和总完成数
|
|
|
|
|
- const newTotalStartCount = (currentArt.totalStartCount || 0) + doneRateDoc.startCount;
|
|
|
|
|
- const newTotalDoneCount = (currentArt.totalDoneCount || 0) + doneRateDoc.doneCount;
|
|
|
|
|
-
|
|
|
|
|
- // 重新计算总完成率
|
|
|
|
|
- const newCompletionRate = newTotalStartCount > 0 ? (newTotalDoneCount / newTotalStartCount) * 100 : 0;
|
|
|
|
|
-
|
|
|
|
|
- // 更新本地 Art 文档
|
|
|
|
|
- await artService.updateArt(artworkId.toString(), {
|
|
|
|
|
- totalStartCount: newTotalStartCount,
|
|
|
|
|
- totalDoneCount: newTotalDoneCount,
|
|
|
|
|
- completionRate: newCompletionRate,
|
|
|
|
|
- });
|
|
|
|
|
- updatedLocalArtworksCount++;
|
|
|
|
|
-
|
|
|
|
|
- // 同步更新远程 Art 文档
|
|
|
|
|
- const remoteUpdateResult = await RemoteArt.findByIdAndUpdate(
|
|
|
|
|
- artworkId,
|
|
|
|
|
- {
|
|
|
|
|
- $set: {
|
|
|
|
|
- totalStartCount: newTotalStartCount,
|
|
|
|
|
- totalDoneCount: newTotalDoneCount,
|
|
|
|
|
- completionRate: newCompletionRate,
|
|
|
|
|
- },
|
|
|
|
|
|
|
+
|
|
|
|
|
+ // 查找现有的 TotalDoneRate 文档
|
|
|
|
|
+ const existingTotal = await TotalDoneRate.findById(artworkId).lean().exec();
|
|
|
|
|
+
|
|
|
|
|
+ // 1. 初始化昨天的计数
|
|
|
|
|
+ const dailyStartCount = doneRateDoc.startCount;
|
|
|
|
|
+ const dailyDoneCount = doneRateDoc.doneCount;
|
|
|
|
|
+ const dailyTipCount = doneRateDoc.tipCount; // 使用昨天的道具使用次数
|
|
|
|
|
+
|
|
|
|
|
+ // 2. 计算累计总数
|
|
|
|
|
+ let newTotalStartCount = dailyStartCount;
|
|
|
|
|
+ let newTotalDoneCount = dailyDoneCount;
|
|
|
|
|
+ let newTotalTipCount = dailyTipCount; // 累计总道具使用次数
|
|
|
|
|
+
|
|
|
|
|
+ if (existingTotal) {
|
|
|
|
|
+ // 如果已存在,则累加旧的总数
|
|
|
|
|
+ // 注意:假设此 cron job 每天只运行一次,否则需要更复杂的幂等逻辑
|
|
|
|
|
+ newTotalStartCount += existingTotal.totalStartCount || 0;
|
|
|
|
|
+ newTotalDoneCount += existingTotal.totalDoneCount || 0;
|
|
|
|
|
+ newTotalTipCount += existingTotal.totalTipCount || 0; // 累加 totalTipCount
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // 3. 计算总完成率
|
|
|
|
|
+ const newCompletionRate = newTotalStartCount > 0 ? (newTotalDoneCount / newTotalStartCount) * 100 : 0;
|
|
|
|
|
+
|
|
|
|
|
+ // 4. 使用 findByIdAndUpdate 进行原子操作
|
|
|
|
|
+ const updatedDoc = await TotalDoneRate.findByIdAndUpdate(
|
|
|
|
|
+ artworkId,
|
|
|
|
|
+ {
|
|
|
|
|
+ $set: {
|
|
|
|
|
+ totalStartCount: newTotalStartCount,
|
|
|
|
|
+ totalDoneCount: newTotalDoneCount,
|
|
|
|
|
+ totalTipCount: newTotalTipCount, // 写入新的累计道具使用次数
|
|
|
|
|
+ completionRate: newCompletionRate,
|
|
|
},
|
|
},
|
|
|
- { new: true } // 返回更新后的文档
|
|
|
|
|
- );
|
|
|
|
|
-
|
|
|
|
|
- if (remoteUpdateResult) {
|
|
|
|
|
- updatedRemoteArtworksCount++;
|
|
|
|
|
- } else {
|
|
|
|
|
- console.warn(`[DoneRate Cron] Remote Art document with ID ${artworkId} not found. Skipping remote update.`);
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- // ============= 新增进度日志 =============
|
|
|
|
|
- if ((i + 1) % 50 === 0 || i === totalDoneRates - 1) {
|
|
|
|
|
- const elapsed = new Date().getTime() - remoteUpdateStartTime;
|
|
|
|
|
- console.log(`[DoneRate Cron] 进度: 已更新 ${i + 1}/${totalDoneRates} 条远程 Art 记录, 当前耗时: ${elapsed}ms`);
|
|
|
|
|
- }
|
|
|
|
|
- // ==========================================
|
|
|
|
|
- } else {
|
|
|
|
|
- console.warn(`[DoneRate Cron] Local Art document with ID ${artworkId} not found for DoneRate record (date: ${doneRateDoc.date}). Skipping Art update.`);
|
|
|
|
|
|
|
+ },
|
|
|
|
|
+ { new: true, upsert: true } // upsert: true 表示如果文档不存在则创建
|
|
|
|
|
+ );
|
|
|
|
|
+
|
|
|
|
|
+ if (updatedDoc) {
|
|
|
|
|
+ updatedTotalDoneRateCount++;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // 进度日志
|
|
|
|
|
+ if ((i + 1) % 50 === 0 || i === totalDoneRatesToUpdate - 1) {
|
|
|
|
|
+ const elapsed = new Date().getTime() - updateStartTime;
|
|
|
|
|
+ console.log(`[DoneRate Cron] 进度: 已更新 ${i + 1}/${totalDoneRatesToUpdate} 条本地 TotalDoneRate 记录, 当前耗时: ${elapsed}ms`);
|
|
|
}
|
|
}
|
|
|
- } catch (artUpdateError) {
|
|
|
|
|
- console.error(`[DoneRate Cron] Error updating Art document for artwork ID ${doneRateDoc.res}:`, artUpdateError);
|
|
|
|
|
|
|
+ } catch (totalUpdateError) {
|
|
|
|
|
+ console.error(`[DoneRate Cron] Error updating TotalDoneRate document for artwork ID ${doneRateDoc.res}:`, totalUpdateError);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- // ============= 新增总结日志 =============
|
|
|
|
|
- const remoteUpdateTimeTaken = new Date().getTime() - remoteUpdateStartTime;
|
|
|
|
|
- console.log(`[DoneRate Cron] 远程 Art 表更新完成。总计更新 ${updatedRemoteArtworksCount} 条记录,总耗时 ${remoteUpdateTimeTaken}ms。`);
|
|
|
|
|
- // ========================================
|
|
|
|
|
|
|
+ const updateTimeTaken = new Date().getTime() - updateStartTime;
|
|
|
|
|
+ console.log(`[DoneRate Cron] 本地 TotalDoneRate 表更新完成。总计更新 ${updatedTotalDoneRateCount} 条记录,总耗时 ${updateTimeTaken}ms。`);
|
|
|
|
|
|
|
|
- const summary = `[DoneRate Cron] Daily done-rate calculation for ${yesterdayYYYYMMDD} completed. Total DoneRate processed: ${totalProcessedArtworks}. Created DoneRate: ${createdRecordsCount}, Updated DoneRate: ${updatedRecordsCount}. Updated Local Art records: ${updatedLocalArtworksCount}. Updated Remote Art records: ${updatedRemoteArtworksCount}.`;
|
|
|
|
|
|
|
+ const summary = `[DoneRate Cron] Daily done-rate calculation for ${yesterdayYYYYMMDD} completed. Total DoneRate processed: ${totalProcessedArtworks}. Created DoneRate: ${createdRecordsCount}, Updated DoneRate: ${updatedRecordsCount}. Updated TotalDoneRate records: ${updatedTotalDoneRateCount}.`;
|
|
|
console.log(summary);
|
|
console.log(summary);
|
|
|
return summary;
|
|
return summary;
|
|
|
} catch (error) {
|
|
} catch (error) {
|
|
|
console.error(`[DoneRate Cron] Error during done-rate calculation for ${yesterdayYYYYMMDD}:`, error);
|
|
console.error(`[DoneRate Cron] Error during done-rate calculation for ${yesterdayYYYYMMDD}:`, error);
|
|
|
throw new Error("Failed to calculate daily done-rates."); // 抛出错误以通知 cron 调度器
|
|
throw new Error("Failed to calculate daily done-rates."); // 抛出错误以通知 cron 调度器
|
|
|
} finally {
|
|
} finally {
|
|
|
- // 确保在任何情况下都关闭远程连接
|
|
|
|
|
- if (remoteConn) {
|
|
|
|
|
- await remoteConn.close();
|
|
|
|
|
- console.log("[DoneRate Cron] Disconnected from remote database.");
|
|
|
|
|
- }
|
|
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|