// oms/scripts/ingestHistoricalData.ts

// Load environment variables from .env (or .env.ingest if specified in script command)
import * as dotenv from "dotenv";
dotenv.config();

import { MongoClient, Db } from "mongodb"; // Native MongoDB driver for external connection
import mongoose from "mongoose"; // Mongoose for OMS MongoDB models
import dayjs from "dayjs"; // For date manipulation
import duration from "dayjs/plugin/duration"; // dayjs plugin for duration
import isSameOrBefore from "dayjs/plugin/isSameOrBefore"; // 👈 NEW: dayjs plugin for isSameOrBefore
import { User, IUser } from "../models/userModel"; // OMS User Model
import { ClickhouseService, IEventLog } from "../services/clickhouseService"; // OMS ClickHouse Service

import { v4 as uuidv4 } from "uuid";

dayjs.extend(duration);
dayjs.extend(isSameOrBefore); // Extend dayjs with the isSameOrBefore plugin

// --- Configuration ---
const CLOGS_MONGO_URI =
  process.env.CLOGS_MONGO_URI || "mongodb://localhost:27017/clogs";
const CLOGS_DATABASE_NAME = process.env.CLOGS_DATABASE_NAME || "clogs"; // Database containing daily collections

const OMS_MONGO_URI = process.env.MONGO_URI || "mongodb://oms:oms123.@localhost:27717/omsdb";
const CLICKHOUSE_HOST = process.env.CLICKHOUSE_HOST || "http://localhost:8123";
const CLICKHOUSE_DATABASE = process.env.CLICKHOUSE_DATABASE || "omsdb";
const CLICKHOUSE_USER = process.env.CLICKHOUSE_USER || "ckuser";
const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD || "ckpassword";
const CLICKHOUSE_EVENTS_TABLE = "events";

// Specify the date range for ingestion (inclusive)
// Format: YYYYMMDD
const START_DATE_STR = process.env.START_DATE || "20250820";
const END_DATE_STR = process.env.END_DATE || "20250820";

// List of event types to process
const ALLOWED_EVENT_TYPES = [
  "visit",
  "show_deeplink_dialog",
  "share",
  "save",
  "revenue",
  "rate",
  "favorite",
  "color_tip",
  "color_start",
  "color_done",
  "color_data",
  "ad_color_tip",
  "ad_color_float",
];

// Define an array of valid IUser keys for copying from event log to User model
const USER_FIELDS_TO_UPDATE: (keyof IUser)[] = [
  "network",
  "campaign",
  "adgroup",
  "creative",
  "prod",
  "libraryName", // maps to library_name
  "cc",
  "lang",
  "manufacturer",
  "deviceModel", // maps to model
  "deviceInfo", // maps to device
  "hardware",
  "deviceMem",
  "apiLevel", // maps to android_api
  "versionName", // maps to version_name or library_version
  "versionCode", // maps to version_code
  "fmToken",  // map to token
  // tags and loginAt are handled specially
  "project" // Add project field for updating, map to project_id
];

// --- Initialize Services ---
let clogsClient: MongoClient;
let clickhouseService: ClickhouseService;

async function initializeServices() {
  try {
    // Connect to external clogs MongoDB
    clogsClient = new MongoClient(CLOGS_MONGO_URI);
    await clogsClient.connect();
    console.log(`Connected to CLOGS MongoDB: ${CLOGS_MONGO_URI}`);

    // Connect to OMS MongoDB (using Mongoose)
    await mongoose.connect(OMS_MONGO_URI);
    console.log(`Connected to OMS MongoDB: ${OMS_MONGO_URI}`);

    // Initialize ClickHouse service with credentials
    clickhouseService = new ClickhouseService(
      CLICKHOUSE_HOST,
      CLICKHOUSE_DATABASE,
      CLICKHOUSE_USER,
      CLICKHOUSE_PASSWORD
    );
    // Ensure ClickHouse table exists
    await clickhouseService.ensureTable(CLICKHOUSE_EVENTS_TABLE); 
    console.log(
      `ClickHouse Service initialized for ${CLICKHOUSE_DATABASE} at ${CLICKHOUSE_HOST}`
    );
  } catch (error) {
    console.error("Failed to initialize services:", error);
    process.exit(1); // Exit if essential services cannot connect
  }
}

// --- Historical Data Ingestion Logic ---
async function ingestHistoricalData() {
  
  console.log(`ingestHistoricalData from ${START_DATE_STR} to ${END_DATE_STR}`);

  let currentDate = dayjs(START_DATE_STR, "YYYYMMDD");
  const endDate = dayjs(END_DATE_STR, "YYYYMMDD");

  while (currentDate.isSameOrBefore(endDate, "day")) {
    const collectionName = currentDate.format("YYYYMMDD");
    console.log(`\n--- Processing collection: ${collectionName} ---`);

    const clogsDb = clogsClient.db(CLOGS_DATABASE_NAME);
    const collections = await clogsDb
      .listCollections({ name: collectionName })
      .toArray();
    const collectionExists = collections.length > 0;

    if (!collectionExists) {
      console.warn(`Collection '${collectionName}' does not exist. Skipping.`);
      currentDate = currentDate.add(1, "day");
      continue;
    }

    const collection = clogsDb.collection(collectionName);
    let processedEventsCount = 0;
    let upsertedUsersCount = 0;

    try {
      const cursor = collection.find({});
      for await (const eventLog of cursor) {
        // Filter by project_id
        const projectId = eventLog.project_id;
        if (projectId !== 1 && projectId !== 6) {
          continue; // Skip if not project 1 or 6
        }

        // Determine event type field name based on project_id
        const eventType = projectId === 1 ? eventLog.type : eventLog.name;

        // Filter by allowed event types
        if (!ALLOWED_EVENT_TYPES.includes(eventType)) {
          continue; // Skip if not an allowed event type
        }

        const uid = projectId === 1 ? eventLog.uid : eventLog.user_id;
        if (!uid) {
          console.warn(
            `Skipping event with missing UID: ${JSON.stringify(eventLog)}`
          );
          continue;
        }

        // --- 1. Process User Data (MongoDB) ---
        const lastActiveAtDateObj = eventLog.t
          ? dayjs(eventLog.t).toDate()
          : eventLog.create_at
          ? dayjs(eventLog.create_at).toDate()
          : new Date();

        // Format lastActiveAt for ClickHouse specifically
        // Explicitly format to 'YYYY-MM-DD HH:mm:ss' which ClickHouse DateTime type expects
        const lastActiveAtClickHouse = dayjs(lastActiveAtDateObj).format(
          "YYYY-MM-DD HH:mm:ss"
        );

        let firstLoginAt: Date | undefined;

        if (
          eventLog.days !== undefined &&
          eventLog.days !== null &&
          lastActiveAtDateObj
        ) {
          // days 字段是天数，倒推 firstLoginAt
          firstLoginAt = dayjs(lastActiveAtDateObj)
            .subtract(eventLog.days, "day")
            .toDate();
        }

        // Initialize userUpdate without 'uid' here, as it's in the query filter
        const userUpdate: Partial<IUser> = { project: projectId }; // project is okay here
        userUpdate.lastActiveAt = lastActiveAtDateObj; // Still use Date object for MongoDB

        // Logic for firstLoginAt: only update if current event's calculated firstLoginAt is earlier or if it's new
        if (firstLoginAt) {
          const existingUser = await User.findOne(
            { uid: uid },
            { firstLoginAt: 1 }
          );
          if (
            !existingUser ||
            !existingUser.firstLoginAt ||
            dayjs(firstLoginAt).isBefore(existingUser.firstLoginAt)
          ) {
            userUpdate.firstLoginAt = firstLoginAt;
          }
        }
 
        // Copy other relevant user profile fields from event log
        for (const field of USER_FIELDS_TO_UPDATE) {
          // Map source field names to target model names
          let sourceFieldName: string | undefined;
            if (field === 'libraryName') sourceFieldName = 'library_name';
            else if (field === 'deviceModel') sourceFieldName = 'model';
            else if (field === 'deviceInfo') sourceFieldName = 'device';
            else if (field === 'apiLevel') sourceFieldName = 'android_api';
            else if (field === 'versionName') sourceFieldName = projectId === 1 ? 'version_name' : 'library_version';
            else if (field === 'versionCode') sourceFieldName = 'version_code';
            else if (field === 'deviceMem') sourceFieldName = 'deviceMem';
            else if (field === 'project') continue; // project is already handled and should not be re-copied from loop
          else sourceFieldName = field; // For fields with consistent names

            // Skip uid and project as they are handled explicitly or are part of the query
            if (field === 'uid') continue;


            if (sourceFieldName && eventLog[sourceFieldName] !== undefined && eventLog[sourceFieldName] !== null) {
              userUpdate[field] = eventLog[sourceFieldName];
            }
        }


        // Perform upsert
        // Use updateOne for more precise control and to avoid the findOneAndUpdate 'uid' conflict
        const result = await User.updateOne(
          { uid: uid },
          {
            $set: userUpdate,
            $setOnInsert: { uid: uid, createdAt: new Date() },
          },
          { upsert: true }
        );
        if (result.upsertedCount > 0 || result.modifiedCount > 0) {
          upsertedUsersCount++;
        }

        // --- 2. Process Event Data (ClickHouse) ---
        const clickhouseEvent: IEventLog = {
          log_id: eventLog._id ? eventLog._id.toString() : uuidv4(),
          uid: uid,
          project: projectId,
          os: eventLog.library_name || null,
          version:
            projectId === 1
              ? eventLog.version_name
              : eventLog.library_version || null,
          event: eventType,
          time: lastActiveAtClickHouse as any, // <--- Use the explicitly formatted string for ClickHouse
          res: projectId === 1 ? eventLog.res : eventLog.sku_id || null,
          from: projectId === 1 ? eventLog.from : eventLog.tab_source || null,
          position:
            projectId === 1
              ? eventLog.position
              : eventLog.click_position || null,
          duration: eventLog.duration || null, // Handles Nullable(UInt32) if present
          ad_type: eventLog.ad_type || null,
          ad_src: eventLog.ad_src || null,
          revenue: projectId === 1 ? eventLog.rev : eventLog.ad_revenue || null, // project_id=1 uses 'rev', project_id=6 uses 'ad_revenue'
          cc: eventLog.cc || null,
          raw_json_data: JSON.stringify(eventLog),
        };

        await clickhouseService.insertEvent(
          CLICKHOUSE_EVENTS_TABLE,
          clickhouseEvent
        );
        processedEventsCount++;
      }
      console.log(
        `Processed ${processedEventsCount} events and upserted ${upsertedUsersCount} users for collection '${collectionName}'.`
      );
    } catch (error) {
      console.error(`Error processing collection '${collectionName}':`, error);
    }

    currentDate = currentDate.add(1, "day");
  }
  console.log("\n--- Historical data ingestion complete ---");
}

// --- Main execution ---
async function main() {
  await initializeServices(); // Initialize OMS DB and ClickHouse
  await ingestHistoricalData(); // Run historical ingestion
  // Close connections gracefully for historical script
  if (clogsClient) await clogsClient.close();
  // Ensure mongoose connection is properly closed only if it was connected
  if (mongoose.connection.readyState === 1) await mongoose.disconnect();
  console.log("Database connections closed.");
  process.exit(0);
}

main().catch(console.error);
