"use strict";
// oms/scripts/ingestHistoricalData.ts
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
// Load environment variables from .env (or .env.ingest if specified in script command)
const dotenv = __importStar(require("dotenv"));
dotenv.config();
const mongodb_1 = require("mongodb"); // Native MongoDB driver for external connection
const mongoose_1 = __importDefault(require("mongoose")); // Mongoose for OMS MongoDB models
const dayjs_1 = __importDefault(require("dayjs")); // For date manipulation
const duration_1 = __importDefault(require("dayjs/plugin/duration")); // dayjs plugin for duration
const isSameOrBefore_1 = __importDefault(require("dayjs/plugin/isSameOrBefore")); // 👈 NEW: dayjs plugin for isSameOrBefore
const userModel_1 = require("../models/userModel"); // OMS User Model
const clickhouseService_1 = require("../services/clickhouseService"); // OMS ClickHouse Service
const uuid_1 = require("uuid");
dayjs_1.default.extend(duration_1.default);
dayjs_1.default.extend(isSameOrBefore_1.default); // Extend dayjs with the isSameOrBefore plugin
// --- Configuration ---
const CLOGS_MONGO_URI = process.env.CLOGS_MONGO_URI || "mongodb://localhost:27017/clogs";
const CLOGS_DATABASE_NAME = process.env.CLOGS_DATABASE_NAME || "clogs"; // Database containing daily collections
const OMS_MONGO_URI = process.env.MONGO_URI || "mongodb://oms:oms123.@localhost:27717/omsdb";
const CLICKHOUSE_HOST = process.env.CLICKHOUSE_HOST || "http://localhost:8123";
const CLICKHOUSE_DATABASE = process.env.CLICKHOUSE_DATABASE || "omsdb";
const CLICKHOUSE_USER = process.env.CLICKHOUSE_USER || "ckuser";
const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD || "ckpassword";
const CLICKHOUSE_EVENTS_TABLE = "events";
// Specify the date range for ingestion (inclusive)
// Format: YYYYMMDD
const START_DATE_STR = process.env.START_DATE || "20250820";
const END_DATE_STR = process.env.END_DATE || "20250820";
// List of event types to process
const ALLOWED_EVENT_TYPES = [
    "visit",
    "show_deeplink_dialog",
    "share",
    "save",
    "revenue",
    "rate",
    "favorite",
    "color_tip",
    "color_start",
    "color_done",
    "color_data",
    "ad_color_tip",
    "ad_color_float",
];
// Define an array of valid IUser keys for copying from event log to User model
const USER_FIELDS_TO_UPDATE = [
    "network",
    "campaign",
    "adgroup",
    "creative",
    "prod",
    "libraryName", // maps to library_name
    "cc",
    "lang",
    "manufacturer",
    "deviceModel", // maps to model
    "deviceInfo", // maps to device
    "hardware",
    "deviceMem",
    "apiLevel", // maps to android_api
    "versionName", // maps to version_name or library_version
    "versionCode", // maps to version_code
    "fmToken", // map to token
    // tags and loginAt are handled specially
    "project" // Add project field for updating, map to project_id
];
// --- Initialize Services ---
let clogsClient;
let clickhouseService;
async function initializeServices() {
    try {
        // Connect to external clogs MongoDB
        clogsClient = new mongodb_1.MongoClient(CLOGS_MONGO_URI);
        await clogsClient.connect();
        console.log(`Connected to CLOGS MongoDB: ${CLOGS_MONGO_URI}`);
        // Connect to OMS MongoDB (using Mongoose)
        await mongoose_1.default.connect(OMS_MONGO_URI);
        console.log(`Connected to OMS MongoDB: ${OMS_MONGO_URI}`);
        // Initialize ClickHouse service with credentials
        clickhouseService = new clickhouseService_1.ClickhouseService(CLICKHOUSE_HOST, CLICKHOUSE_DATABASE, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD);
        // Ensure ClickHouse table exists
        await clickhouseService.ensureTable(CLICKHOUSE_EVENTS_TABLE);
        console.log(`ClickHouse Service initialized for ${CLICKHOUSE_DATABASE} at ${CLICKHOUSE_HOST}`);
    }
    catch (error) {
        console.error("Failed to initialize services:", error);
        process.exit(1); // Exit if essential services cannot connect
    }
}
// --- Historical Data Ingestion Logic ---
async function ingestHistoricalData() {
    console.log(`ingestHistoricalData from ${START_DATE_STR} to ${END_DATE_STR}`);
    let currentDate = (0, dayjs_1.default)(START_DATE_STR, "YYYYMMDD");
    const endDate = (0, dayjs_1.default)(END_DATE_STR, "YYYYMMDD");
    while (currentDate.isSameOrBefore(endDate, "day")) {
        const collectionName = currentDate.format("YYYYMMDD");
        console.log(`\n--- Processing collection: ${collectionName} ---`);
        const clogsDb = clogsClient.db(CLOGS_DATABASE_NAME);
        const collections = await clogsDb
            .listCollections({ name: collectionName })
            .toArray();
        const collectionExists = collections.length > 0;
        if (!collectionExists) {
            console.warn(`Collection '${collectionName}' does not exist. Skipping.`);
            currentDate = currentDate.add(1, "day");
            continue;
        }
        const collection = clogsDb.collection(collectionName);
        let processedEventsCount = 0;
        let upsertedUsersCount = 0;
        try {
            const cursor = collection.find({});
            for await (const eventLog of cursor) {
                // Filter by project_id
                const projectId = eventLog.project_id;
                if (projectId !== 1 && projectId !== 6) {
                    continue; // Skip if not project 1 or 6
                }
                // Determine event type field name based on project_id
                const eventType = projectId === 1 ? eventLog.type : eventLog.name;
                // Filter by allowed event types
                if (!ALLOWED_EVENT_TYPES.includes(eventType)) {
                    continue; // Skip if not an allowed event type
                }
                const uid = projectId === 1 ? eventLog.uid : eventLog.user_id;
                if (!uid) {
                    console.warn(`Skipping event with missing UID: ${JSON.stringify(eventLog)}`);
                    continue;
                }
                // --- 1. Process User Data (MongoDB) ---
                const lastActiveAtDateObj = eventLog.t
                    ? (0, dayjs_1.default)(eventLog.t).toDate()
                    : eventLog.create_at
                        ? (0, dayjs_1.default)(eventLog.create_at).toDate()
                        : new Date();
                // Format lastActiveAt for ClickHouse specifically
                // Explicitly format to 'YYYY-MM-DD HH:mm:ss' which ClickHouse DateTime type expects
                const lastActiveAtClickHouse = (0, dayjs_1.default)(lastActiveAtDateObj).format("YYYY-MM-DD HH:mm:ss");
                let firstLoginAt;
                if (eventLog.days !== undefined &&
                    eventLog.days !== null &&
                    lastActiveAtDateObj) {
                    // days 字段是天数，倒推 firstLoginAt
                    firstLoginAt = (0, dayjs_1.default)(lastActiveAtDateObj)
                        .subtract(eventLog.days, "day")
                        .toDate();
                }
                // Initialize userUpdate without 'uid' here, as it's in the query filter
                const userUpdate = { project: projectId }; // project is okay here
                userUpdate.lastActiveAt = lastActiveAtDateObj; // Still use Date object for MongoDB
                // Logic for firstLoginAt: only update if current event's calculated firstLoginAt is earlier or if it's new
                if (firstLoginAt) {
                    const existingUser = await userModel_1.User.findOne({ uid: uid }, { firstLoginAt: 1 });
                    if (!existingUser ||
                        !existingUser.firstLoginAt ||
                        (0, dayjs_1.default)(firstLoginAt).isBefore(existingUser.firstLoginAt)) {
                        userUpdate.firstLoginAt = firstLoginAt;
                    }
                }
                // Copy other relevant user profile fields from event log
                for (const field of USER_FIELDS_TO_UPDATE) {
                    // Map source field names to target model names
                    let sourceFieldName;
                    if (field === 'libraryName')
                        sourceFieldName = 'library_name';
                    else if (field === 'deviceModel')
                        sourceFieldName = 'model';
                    else if (field === 'deviceInfo')
                        sourceFieldName = 'device';
                    else if (field === 'apiLevel')
                        sourceFieldName = 'android_api';
                    else if (field === 'versionName')
                        sourceFieldName = projectId === 1 ? 'version_name' : 'library_version';
                    else if (field === 'versionCode')
                        sourceFieldName = 'version_code';
                    else if (field === 'deviceMem')
                        sourceFieldName = 'deviceMem';
                    else if (field === 'project')
                        continue; // project is already handled and should not be re-copied from loop
                    else
                        sourceFieldName = field; // For fields with consistent names
                    // Skip uid and project as they are handled explicitly or are part of the query
                    if (field === 'uid')
                        continue;
                    if (sourceFieldName && eventLog[sourceFieldName] !== undefined && eventLog[sourceFieldName] !== null) {
                        userUpdate[field] = eventLog[sourceFieldName];
                    }
                }
                // Perform upsert
                // Use updateOne for more precise control and to avoid the findOneAndUpdate 'uid' conflict
                const result = await userModel_1.User.updateOne({ uid: uid }, {
                    $set: userUpdate,
                    $setOnInsert: { uid: uid, createdAt: new Date() },
                }, { upsert: true });
                if (result.upsertedCount > 0 || result.modifiedCount > 0) {
                    upsertedUsersCount++;
                }
                // --- 2. Process Event Data (ClickHouse) ---
                const clickhouseEvent = {
                    log_id: eventLog._id ? eventLog._id.toString() : (0, uuid_1.v4)(),
                    uid: uid,
                    project: projectId,
                    os: eventLog.library_name || null,
                    version: projectId === 1
                        ? eventLog.version_name
                        : eventLog.library_version || null,
                    event: eventType,
                    time: lastActiveAtClickHouse, // <--- Use the explicitly formatted string for ClickHouse
                    res: projectId === 1 ? eventLog.res : eventLog.sku_id || null,
                    from: projectId === 1 ? eventLog.from : eventLog.tab_source || null,
                    position: projectId === 1
                        ? eventLog.position
                        : eventLog.click_position || null,
                    duration: eventLog.duration || null, // Handles Nullable(UInt32) if present
                    ad_type: eventLog.ad_type || null,
                    ad_src: eventLog.ad_src || null,
                    revenue: projectId === 1 ? eventLog.rev : eventLog.ad_revenue || null, // project_id=1 uses 'rev', project_id=6 uses 'ad_revenue'
                    cc: eventLog.cc || null,
                    raw_json_data: JSON.stringify(eventLog),
                };
                await clickhouseService.insertEvent(CLICKHOUSE_EVENTS_TABLE, clickhouseEvent);
                processedEventsCount++;
            }
            console.log(`Processed ${processedEventsCount} events and upserted ${upsertedUsersCount} users for collection '${collectionName}'.`);
        }
        catch (error) {
            console.error(`Error processing collection '${collectionName}':`, error);
        }
        currentDate = currentDate.add(1, "day");
    }
    console.log("\n--- Historical data ingestion complete ---");
}
// --- Main execution ---
async function main() {
    await initializeServices(); // Initialize OMS DB and ClickHouse
    await ingestHistoricalData(); // Run historical ingestion
    // Close connections gracefully for historical script
    if (clogsClient)
        await clogsClient.close();
    // Ensure mongoose connection is properly closed only if it was connected
    if (mongoose_1.default.connection.readyState === 1)
        await mongoose_1.default.disconnect();
    console.log("Database connections closed.");
    process.exit(0);
}
main().catch(console.error);
