"use strict";
// oms/scripts/ingestHistoricalData.ts
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
// Load environment variables for database credentials (MONGO_URI, CLICKHOUSE_*)
// Log file directory is now passed as a function argument.
const dotenv = __importStar(require("dotenv"));
dotenv.config();
const mongoose_1 = __importDefault(require("mongoose")); // Mongoose for OMS MongoDB models
const dayjs_1 = __importDefault(require("dayjs")); // For date manipulation
const duration_1 = __importDefault(require("dayjs/plugin/duration")); // dayjs plugin for duration
const uuid_1 = require("uuid"); // For generating unique IDs where needed
const userModel_1 = require("../models/userModel"); // OMS User Model
const clickhouseService_1 = require("../services/clickhouseService"); // OMS ClickHouse Service
// Node.js built-in modules for file processing
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const zlib = __importStar(require("zlib"));
const readline = __importStar(require("readline"));
dayjs_1.default.extend(duration_1.default);
// --- Persistent Configuration (can still come from .env) ---
const OMS_MONGO_URI = process.env.MONGO_URI || "mongodb://oms:oms123.@localhost:27717/omsdb?authSource=admin";
const CLICKHOUSE_HOST = process.env.CLICKHOUSE_HOST || "http://localhost:8123";
const CLICKHOUSE_DATABASE = process.env.CLICKHOUSE_DATABASE || "omsdb"; // Aligned with docker-compose.yml
const CLICKHOUSE_USER = process.env.CLICKHOUSE_USER || "ckuser";
const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD || "ckpassword";
const CLICKHOUSE_EVENTS_TABLE = "events"; // Aligned with clickhouseService.ts table name
// --- Batching Configuration ---
const CLICKHOUSE_BATCH_SIZE = 5000; // ClickHouse 批量插入大小
const MONGO_BATCH_SIZE = 1000; // MongoDB 批量写入大小
// List of event types to process
const ALLOWED_EVENT_TYPES = ["visit", "show_deeplink_dialog", "share", "save", "revenue", "rate", "favorite", "color_tip", "color_start", "color_done", "color_data", "ad_color_tip", "ad_color_float"];
// Define an array of valid IUser keys for copying from event log to User model
const USER_FIELDS_TO_UPDATE = [
    "network",
    "campaign",
    "adgroup",
    "creative",
    "prod",
    "libraryName", // maps to library_name
    "cc",
    "lang",
    "manufacturer",
    "deviceModel", // maps to model
    "deviceInfo", // maps to device
    "hardware",
    "deviceMem",
    "apiLevel", // maps to android_api
    "versionName", // maps to version_name or library_version
    "versionCode", // maps to version_code
    "fmToken", // map to token
    "project", // Add project field for updating, map to project_id
];
// --- Initialize Services ---
let clickhouseService;
async function initializeServices() {
    try {
        // Connect to OMS MongoDB (using Mongoose)
        await mongoose_1.default.connect(OMS_MONGO_URI);
        console.log(`Connected to OMS MongoDB: ${OMS_MONGO_URI}`);
        // Initialize ClickHouse service with credentials
        clickhouseService = new clickhouseService_1.ClickhouseService(CLICKHOUSE_HOST, CLICKHOUSE_DATABASE, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD);
        // Ensure ClickHouse table exists
        await clickhouseService.ensureTable(CLICKHOUSE_EVENTS_TABLE);
        console.log(`ClickHouse Service initialized for ${CLICKHOUSE_DATABASE} at ${CLICKHOUSE_HOST}`);
    }
    catch (error) {
        console.error("Failed to initialize services:", error);
        process.exit(1); // Exit if essential services cannot connect
    }
}
// --- Helper function to flush ClickHouse buffer ---
async function flushClickHouseBuffer(buffer, tableName, processedCount) {
    if (buffer.length === 0)
        return;
    try {
        // clickhouseService.insertEvent now accepts IEventLog | IEventLog[]
        await clickhouseService.insertEvent(tableName, buffer);
        console.log(`Flushed ${buffer.length} events to ClickHouse. Total processed: ${processedCount}`);
        buffer.length = 0; // Clear buffer
    }
    catch (error) {
        console.error(`Error flushing ClickHouse buffer:`, error);
        // Depending on error handling strategy, might rethrow or log and continue
    }
}
// --- Historical Data Ingestion Logic ---
// Now accepts only the log files directory as argument
async function ingestHistoricalData(logFilesDir) {
    const overallStartTime = (0, dayjs_1.default)();
    console.log(`Starting historical ingestion from local files in: ${logFilesDir}`);
    let totalProcessedEvents = 0;
    let totalUpsertedUsers = 0;
    // Buffers for batching
    const clickhouseEventsBuffer = [];
    const mongoUserWriteOperations = []; // Array of Mongoose bulkWrite operations
    // Get all files from the directory and filter for .log.gz files
    let allFiles = [];
    try {
        allFiles = fs
            .readdirSync(logFilesDir)
            .filter((file) => file.endsWith(".log.gz"))
            .sort(); // Sort files to ensure chronological processing
    }
    catch (readDirError) {
        console.error(`Error reading directory '${logFilesDir}':`, readDirError);
        // If the directory can't be read, we can't proceed.
        return;
    }
    const totalFiles = allFiles.length;
    if (totalFiles === 0) {
        console.log("No .log.gz files found in the specified directory. Exiting.");
        return;
    }
    console.log(`Found ${totalFiles} .log.gz files to process.`);
    for (let i = 0; i < totalFiles; i++) {
        const expectedFilename = allFiles[i];
        const filePath = path.join(logFilesDir, expectedFilename);
        const fileStartTime = (0, dayjs_1.default)();
        console.log(`\n--- Processing file ${i + 1}/${totalFiles}: ${expectedFilename} ---`);
        let processedEventsCount = 0;
        let upsertedUsersCount = 0;
        const fileStream = fs.createReadStream(filePath);
        const gunzip = zlib.createGunzip();
        const rl = readline.createInterface({
            input: fileStream.pipe(gunzip),
            crlfDelay: Infinity, // Handle Windows/Unix line endings
        });
        try {
            // Cache for firstLoginAt lookup for the current batch
            const firstLoginAtCache = new Map();
            const currentBatchUids = new Set();
            const eventsInCurrentBatch = []; // Temporary store for raw eventLog in a batch
            for await (const line of rl) {
                if (!line.trim())
                    continue; // Skip empty lines
                let eventLog;
                try {
                    eventLog = JSON.parse(line);
                }
                catch (parseError) {
                    console.error(`Error parsing JSON from line in '${expectedFilename}': ${line}. Error: ${parseError}`);
                    continue; // Skip malformed JSON lines
                }
                // Filter by project_id
                const projectId = eventLog.project_id;
                if (projectId !== 1 && projectId !== 6) {
                    continue; // Skip if not project 1 or 6
                }
                // Determine event type field name based on project_id
                const eventType = projectId === 1 ? eventLog.type : eventLog.name;
                // Filter by allowed event types
                if (!ALLOWED_EVENT_TYPES.includes(eventType)) {
                    continue; // Skip if not an allowed event type
                }
                // Skip events with invalid duration
                if (eventLog.duration > 100000 || eventLog.duration < 0) {
                    console.warn(`Skipping event with invalid duration: ${eventLog.duration}. Event: ${JSON.stringify(eventLog)}`);
                    continue;
                }
                const uid = projectId === 1 ? eventLog.uid : eventLog.user_id;
                if (!uid) {
                    console.warn(`Skipping event with missing UID in '${expectedFilename}': ${JSON.stringify(eventLog)}`);
                    continue;
                }
                // --- Accumulate for User Data (MongoDB) Batch ---
                currentBatchUids.add(uid);
                eventsInCurrentBatch.push(eventLog); // Store raw eventLog temporarily
                // Process a batch when Mongo batch size is reached
                if (eventsInCurrentBatch.length >= MONGO_BATCH_SIZE) {
                    // 1. Fetch existing firstLoginAt for all UIDs in the current batch
                    const existingUsers = await userModel_1.User.find({ uid: { $in: Array.from(currentBatchUids) } }, { uid: 1, firstLoginAt: 1 });
                    existingUsers.forEach((user) => {
                        firstLoginAtCache.set(user.uid, user.firstLoginAt);
                    });
                    for (const batchedEventLog of eventsInCurrentBatch) {
                        const batchUid = batchedEventLog.project_id === 1 ? batchedEventLog.uid : batchedEventLog.user_id;
                        const batchProjectId = batchedEventLog.project_id;
                        let batchLastActiveAtDateObj; // Declare here
                        if (batchedEventLog.t) {
                            batchLastActiveAtDateObj = (0, dayjs_1.default)(batchedEventLog.t).toDate();
                        }
                        else if (batchedEventLog.create_at) {
                            batchLastActiveAtDateObj = (0, dayjs_1.default)(batchedEventLog.create_at).toDate();
                        }
                        else {
                            batchLastActiveAtDateObj = new Date();
                        }
                        let batchFirstLoginAt;
                        if (batchedEventLog.days !== undefined && batchedEventLog.days !== null) {
                            // Changed to use batchLastActiveAtDateObj directly
                            batchFirstLoginAt = (0, dayjs_1.default)(batchLastActiveAtDateObj).subtract(batchedEventLog.days, "day").toDate();
                        }
                        const batchUserUpdate = { project: batchProjectId };
                        batchUserUpdate.lastActiveAt = batchLastActiveAtDateObj;
                        if (batchFirstLoginAt) {
                            const cachedFirstLoginAt = firstLoginAtCache.get(batchUid);
                            if (!cachedFirstLoginAt || !cachedFirstLoginAt.getTime() || (0, dayjs_1.default)(batchFirstLoginAt).isBefore(cachedFirstLoginAt)) {
                                batchUserUpdate.firstLoginAt = batchFirstLoginAt;
                            }
                        }
                        for (const field of USER_FIELDS_TO_UPDATE) {
                            let sourceFieldName;
                            if (field === "libraryName")
                                sourceFieldName = "library_name";
                            else if (field === "deviceModel")
                                sourceFieldName = "model";
                            else if (field === "deviceInfo")
                                sourceFieldName = "device";
                            else if (field === "apiLevel")
                                sourceFieldName = "android_api";
                            else if (field === "versionName")
                                sourceFieldName = batchProjectId === 1 ? "version_name" : "library_version";
                            else if (field === "versionCode")
                                sourceFieldName = "version_code";
                            else if (field === "deviceMem")
                                sourceFieldName = "deviceMem";
                            else if (field === "project")
                                continue;
                            else if (field === "fmToken")
                                sourceFieldName = "token";
                            else
                                sourceFieldName = field;
                            if (field === "uid")
                                continue;
                            if (sourceFieldName && batchedEventLog[sourceFieldName] !== undefined && batchedEventLog[sourceFieldName] !== null) {
                                if (field === "deviceMem" && typeof batchedEventLog[sourceFieldName] === "number") {
                                    batchUserUpdate.deviceMem = batchedEventLog[sourceFieldName];
                                }
                                else {
                                    batchUserUpdate[field] = batchedEventLog[sourceFieldName];
                                }
                            }
                        }
                        mongoUserWriteOperations.push({
                            updateOne: {
                                filter: { uid: batchUid },
                                update: {
                                    $set: batchUserUpdate,
                                    $setOnInsert: { uid: batchUid, createdAt: new Date() },
                                },
                                upsert: true,
                            },
                        });
                    }
                    if (mongoUserWriteOperations.length > 0) {
                        try {
                            const bulkResult = await userModel_1.User.bulkWrite(mongoUserWriteOperations);
                            upsertedUsersCount += (bulkResult.upsertedCount || 0) + (bulkResult.modifiedCount || 0);
                            console.log(`MongoDB batch written. Upserted/Modified: ${bulkResult.upsertedCount + bulkResult.modifiedCount}`);
                        }
                        catch (bulkError) {
                            console.error(`Error in MongoDB bulkWrite for file '${expectedFilename}':`, bulkError);
                        }
                        mongoUserWriteOperations.length = 0; // Clear buffer
                    }
                    eventsInCurrentBatch.length = 0; // Clear raw events buffer
                    currentBatchUids.clear(); // Clear UIDs for next batch
                    firstLoginAtCache.clear(); // Clear cache for next batch
                }
                // --- Prepare Event Data for ClickHouse Batch ---
                // Redeclare lastActiveAtDateObj for ClickHouse event processing if not already in scope.
                // It's safer to recalculate or ensure it's in scope, here we use `let` to declare it
                // in the scope of the `for await` loop.
                let lastActiveAtDateObj;
                if (eventLog.t) {
                    lastActiveAtDateObj = (0, dayjs_1.default)(eventLog.t).toDate();
                }
                else if (eventLog.create_at) {
                    lastActiveAtDateObj = (0, dayjs_1.default)(eventLog.create_at).toDate();
                }
                else {
                    lastActiveAtDateObj = new Date();
                }
                const clickhouseEvent = {
                    log_id: eventLog._id ? eventLog._id.toString() : (0, uuid_1.v4)(),
                    uid: uid,
                    project: projectId,
                    os: eventLog.library_name || null,
                    version: projectId === 1 ? eventLog.version_name : eventLog.library_version || null,
                    event: eventType,
                    time: lastActiveAtDateObj, // Use the explicitly formatted string for ClickHouse
                    res: projectId === 1 ? eventLog.res : eventLog.sku_id || null,
                    from: projectId === 1 ? eventLog.from : eventLog.tab_source || null,
                    position: projectId === 1 ? eventLog.position : eventLog.click_position || null,
                    duration: eventLog.duration || null, // Handles Nullable(UInt32) if present
                    ad_type: eventLog.ad_type || null,
                    ad_src: eventLog.ad_src || null,
                    revenue: projectId === 1 ? eventLog.rev : eventLog.ad_revenue || null,
                    cc: eventLog.cc || null,
                    raw_json_data: JSON.stringify(eventLog),
                };
                clickhouseEventsBuffer.push(clickhouseEvent);
                processedEventsCount++;
                // Flush ClickHouse buffer if batch size reached
                if (clickhouseEventsBuffer.length >= CLICKHOUSE_BATCH_SIZE) {
                    await flushClickHouseBuffer(clickhouseEventsBuffer, CLICKHOUSE_EVENTS_TABLE, processedEventsCount);
                }
            } // End of for await (const line of rl) loop
            // --- Flush any remaining buffers after file processing ---
            // Flush remaining MongoDB operations
            // Process remaining user data (MongoDB) that didn't form a full batch
            if (eventsInCurrentBatch.length > 0) {
                // If UIDs exist and cache is empty (meaning no full batch was processed), fetch existing users
                if (currentBatchUids.size > 0 && firstLoginAtCache.size === 0) {
                    const existingUsers = await userModel_1.User.find({ uid: { $in: Array.from(currentBatchUids) } }, { uid: 1, firstLoginAt: 1 });
                    existingUsers.forEach((user) => {
                        firstLoginAtCache.set(user.uid, user.firstLoginAt);
                    });
                }
                for (const batchedEventLog of eventsInCurrentBatch) {
                    const batchUid = batchedEventLog.project_id === 1 ? batchedEventLog.uid : batchedEventLog.user_id;
                    const batchProjectId = batchedEventLog.project_id;
                    let batchLastActiveAtDateObj;
                    if (batchedEventLog.t) {
                        batchLastActiveAtDateObj = (0, dayjs_1.default)(batchedEventLog.t).toDate();
                    }
                    else if (batchedEventLog.create_at) {
                        batchLastActiveAtDateObj = (0, dayjs_1.default)(batchedEventLog.create_at).toDate();
                    }
                    else {
                        batchLastActiveAtDateObj = new Date();
                    }
                    let batchFirstLoginAt;
                    if (batchedEventLog.days !== undefined && batchedEventLog.days !== null) {
                        // Changed to use batchLastActiveAtDateObj directly
                        batchFirstLoginAt = (0, dayjs_1.default)(batchLastActiveAtDateObj).subtract(batchedEventLog.days, "day").toDate();
                    }
                    const batchUserUpdate = { project: batchProjectId };
                    batchUserUpdate.lastActiveAt = batchLastActiveAtDateObj;
                    if (batchFirstLoginAt) {
                        const cachedFirstLoginAt = firstLoginAtCache.get(batchUid);
                        // Ensure cachedFirstLoginAt is a valid Date before comparison
                        if (!cachedFirstLoginAt || !cachedFirstLoginAt.getTime() || (0, dayjs_1.default)(batchFirstLoginAt).isBefore(cachedFirstLoginAt)) {
                            batchUserUpdate.firstLoginAt = batchFirstLoginAt;
                        }
                    }
                    for (const field of USER_FIELDS_TO_UPDATE) {
                        let sourceFieldName;
                        if (field === "libraryName")
                            sourceFieldName = "library_name";
                        else if (field === "deviceModel")
                            sourceFieldName = "model";
                        else if (field === "deviceInfo")
                            sourceFieldName = "device";
                        else if (field === "apiLevel")
                            sourceFieldName = "android_api";
                        else if (field === "versionName")
                            sourceFieldName = batchProjectId === 1 ? "version_name" : "library_version";
                        else if (field === "versionCode")
                            sourceFieldName = "version_code";
                        else if (field === "deviceMem")
                            sourceFieldName = "deviceMem";
                        else if (field === "project")
                            continue;
                        else if (field === "fmToken")
                            sourceFieldName = "token";
                        else
                            sourceFieldName = field;
                        if (field === "uid")
                            continue;
                        if (sourceFieldName && batchedEventLog[sourceFieldName] !== undefined && batchedEventLog[sourceFieldName] !== null) {
                            if (field === "deviceMem" && typeof batchedEventLog[sourceFieldName] === "number") {
                                batchUserUpdate.deviceMem = batchedEventLog[sourceFieldName];
                            }
                            else {
                                batchUserUpdate[field] = batchedEventLog[sourceFieldName];
                            }
                        }
                    }
                    mongoUserWriteOperations.push({
                        updateOne: {
                            filter: { uid: batchUid },
                            update: {
                                $set: batchUserUpdate,
                                $setOnInsert: { uid: batchUid, createdAt: new Date() },
                            },
                            upsert: true,
                        },
                    });
                }
            }
            if (mongoUserWriteOperations.length > 0) {
                try {
                    const bulkResult = await userModel_1.User.bulkWrite(mongoUserWriteOperations);
                    upsertedUsersCount += (bulkResult.upsertedCount || 0) + (bulkResult.modifiedCount || 0);
                    // console.log(`MongoDB final batch written. Upserted/Modified: ${bulkResult.upsertedCount + bulkResult.modifiedCount}`);
                }
                catch (bulkError) {
                    console.error(`Error in final MongoDB bulkWrite for file '${expectedFilename}':`, bulkError);
                }
                mongoUserWriteOperations.length = 0; // Clear buffer
            }
            // Flush remaining ClickHouse events
            await flushClickHouseBuffer(clickhouseEventsBuffer, CLICKHOUSE_EVENTS_TABLE, processedEventsCount);
            totalProcessedEvents += processedEventsCount;
            totalUpsertedUsers += upsertedUsersCount;
            const fileDuration = dayjs_1.default.duration((0, dayjs_1.default)().diff(fileStartTime)).asSeconds();
            console.log(`File '${expectedFilename}' processed: ${processedEventsCount} events, ${upsertedUsersCount} users upserted. Time taken: ${fileDuration.toFixed(2)} seconds.`);
        }
        catch (error) {
            console.error(`Error processing file '${expectedFilename}':`, error);
            // If an error occurs during file processing, we still want to move to the next file
        }
    } // End of for...of allFiles loop
    const overallDuration = dayjs_1.default.duration((0, dayjs_1.default)().diff(overallStartTime)).asSeconds();
    console.log(`\n--- Historical data ingestion complete ---`);
    console.log(`Total processed events: ${totalProcessedEvents}`);
    console.log(`Total upserted users: ${totalUpsertedUsers}`);
    console.log(`Total time taken: ${overallDuration.toFixed(2)} seconds.`);
}
// --- Main execution ---
async function main() {
    await initializeServices(); // Initialize OMS DB and ClickHouse
    // --- Hardcoded parameters for one-time ingestion ---
    const logFilesDirectory = process.argv[2] || path.join(__dirname, "../../../../../logs/"); // Fallback directory
    await ingestHistoricalData(logFilesDirectory);
    // Ensure mongoose connection is properly closed only if it was connected
    if (mongoose_1.default.connection.readyState === 1)
        await mongoose_1.default.disconnect();
    console.log("Database connections closed.");
    process.exit(0);
}
main().catch(console.error);
