monitor/service/monitor/check-api.js

/// 每个小时检查下api接口及数据一致性
const axios = require('axios');

const { sendEmail } = require('./email')
const { sendSms } = require('./sms')
const SmsTemplate = require('./sms-templates');

const onlineHost = 'app2.pcoloring.com';
const offlineHost = 'color.jccytech.cn';
const artLatestUri = 'napi/number/v9/list/art/content?column=latest';
const jigsawLatestUri = 'napi/jigsaw/mobi/list/latest';
const artPuzzleLatestUri = 'napi/puzzle/mobi/list/latest';

async function checkArtLatest() {
  let onlineUrl = `https://${onlineHost}/${artLatestUri}`;
  let offlineUrl = `https://${offlineHost}/${artLatestUri}`;
  let onlineData, offlineData;
  try {
    let resp = await axios.get(onlineUrl);
    onlineData = resp.data;
  } catch (e) {
    console.error('Art线上latest数据请求出错, 请及时检查:', [e.message]);
    return {
      result: false,
      errcode: 100,
      title: 'Art线上latest数据请求出错, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: [e.message],
    }
  }

  try {
    let resp = await axios.get(offlineUrl);
    offlineData = resp.data;
  } catch (e) {
    console.error('Art 线下latest数据请求出错, 请及时检查:', [e.message]);
    return {
      result: false,
      errcode: 100,
      title: 'Art 线下latest数据请求出错, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: [e.message],
    }
  }

  // 检查数据是否一致： 看第一条即可
  if (!onlineData.data || onlineData.data.length <= 0) {
    return {
      result: false,
      errcode: 100,
      title: 'Art 线上latest数据有误, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: ['latest数据为空'],
    }
  }
  if (!offlineData.data || offlineData.data.length <= 0) {
    return {
      result: false,
      errcode: 100,
      title: 'Art 线下latest数据有误, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: ['latest数据为空'],
    }
  }

  if (onlineData.data[0]._id != offlineData.data[0]._id) {
    return {
      result: false,
      errcode: 100,
      title: 'Art latest 线上线下数据不一致, 请及时检查',
      sms: SmsTemplate.SMS_SYNC_ERROR,
      data: [onlineUrl, offlineUrl],
    }
  }

  return {
    result: true,
    errcode: 0,
    title: 'Art 服务恢复正常运行',
    sms: SmsTemplate.SMS_RESUME,
    data: ['Art 服务一切正常']
  }
}

async function checkJigsawLatest() {
  let onlineUrl = `https://${onlineHost}/${jigsawLatestUri}`;
  let offlineUrl = `https://${offlineHost}/${jigsawLatestUri}`;
  let onlineData, offlineData;
  try {
    let resp = await axios.get(onlineUrl);
    onlineData = resp.data;
  } catch (e) {
    console.error('Jigsaw 线上latest数据请求出错, 请及时检查:', e.message);
    return {
      result: false,
      errcode: 200,
      title: 'Jigsaw 线上latest数据请求出错, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: [e.message],
    }
  }

  try {
    let resp = await axios.get(offlineUrl);
    offlineData = resp.data;
  } catch (e) {
    console.error('Jigsaw 线下latest数据请求出错, 请及时检查:', e.message);
    return {
      result: false,
      errcode: 200,
      title: 'Jigsaw 线下latest数据请求出错, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: [e.message],
    }
  }

  // 检查数据是否一致： 看第一条即可
  if (!onlineData.data || onlineData.data.length <= 0) {
    return {
      result: false,
      errcode: 200,
      title: 'Jigsaw 线上latest数据有误, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: ['latest数据为空'],
    }
  }
  if (!offlineData.data || offlineData.data.length <= 0) {
    return {
      result: false,
      errcode: 200,
      title: 'Jigsaw 线下latest数据有误, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: ['latest数据为空'],
    }
  }

  if (onlineData.data[0]._id != offlineData.data[0]._id) {
    return {
      result: false,
      errcode: 200,
      title: 'Jigsaw latest 线上线下数据不一致, 请及时检查',
      sms: SmsTemplate.SMS_SYNC_ERROR,
      data: [onlineUrl, offlineUrl],
    }
  }

  return {
    result: true,
    errcode: 0,
    title: 'Jigsaw 服务恢复正常运行',
    sms: SmsTemplate.SMS_RESUME,
    data: ['Jigsaw 服务一切正常']
  }
}

async function checkArtPuzzleLatest() {
  let onlineUrl = `https://${onlineHost}/${artPuzzleLatestUri}`;
  let offlineUrl = `https://${offlineHost}/${artPuzzleLatestUri}`;
  let onlineData, offlineData;
  try {
    let resp = await axios.get(onlineUrl);
    onlineData = resp.data;
  } catch (e) {
    console.error('ArtPuzzle 线上latest数据请求出错, 请及时检查:', e.message);
    return {
      result: false,
      errcode: 300,
      title: 'ArtPuzzle 线上latest数据请求出错, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: [e.message],
    }
  }

  try {
    let resp = await axios.get(offlineUrl);
    offlineData = resp.data;
  } catch (e) {
    console.error('ArtPuzzle 线下latest数据请求出错, 请及时检查:', e.message);
    return {
      result: false,
      errcode: 300,
      title: 'ArtPuzzle 线下latest数据请求出错, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: [e.message],
    }
  }

  if (!onlineData.data || onlineData.data.length <= 0) {
    return {
      result: false,
      errcode: 300,
      title: 'ArtPuzzle 线上latest数据有误, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: ['latest数据为空'],
    }
  }
  if (!offlineData.data || offlineData.data.length <= 0) {
    return {
      result: false,
      errcode: 300,
      title: 'ArtPuzzle 线下latest数据有误, 请及时检查',
      sms: SmsTemplate.SMS_API_ERROR,
      data: ['latest数据为空'],
    }
  }

  // 检查数据是否一致： 看第一条即可
  if (onlineData.data[0]._id != offlineData.data[0]._id) {
    return {
      result: false,
      errcode: 300,
      title: 'ArtPuzzle latest 线上线下数据不一致, 请及时检查',
      sms: SmsTemplate.SMS_SYNC_ERROR,
      data: [onlineUrl, offlineUrl],
    }
  }

  return {
    result: true,
    errcode: 0,
    title: 'Jigsaw 服务恢复正常运行',
    sms: SmsTemplate.SMS_RESUME,
    data: ['Jigsaw 服务一切正常']
  }
}

async function run(lastErrCode = 0) {
  console.log("check api...", new Date());
  try {
    let result = null;
    // 如果出现异常，连续检测3次才发送通知，减少误报，因为出现不少次因为网络原因一时半会请求不到数据就报警的情况
    for (let i = 0; i < 3; i++) {
      result = await checkArtLatest();
      console.log(result);
      if (!result.result) {
        console.warn("检查Art服务出现异常，一分钟后重试");
        await delay(60 * 1000);
      } else {
        console.warn("Art服务一切正常");
        break;
      }
    }
    if (!result.result) {
      console.error("连续3次检查Art服务异常，发送通知");
      sendEmail(result.title, result.data);
      sendSms(result.sms);
      console.log("5分钟后再试");
      setTimeout(run, 5 * 60 * 1000, result.errcode);
      return;
    } else {
      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
      if (lastErrCode == 100) {
        sendEmail(result.title, result.data);
        sendSms(result.sms);
      }
    }

    for (let i = 0; i < 3; i++) {
      result = await checkJigsawLatest();
      console.log(result);
      if (!result.result) {
        console.warn("检查Jigsaw服务出现异常，一分钟后重试");
        await delay(60 * 1000);
      } else {
        console.warn("Jigsaw服务一切正常");
        break;
      }
    }
    if (!result.result) {
      console.error("连续3次检查Jigsaw服务异常，发送通知");
      sendEmail(result.title, result.data);
      sendSms(result.sms);
      console.log("5分钟后再试");
      setTimeout(run, 5 * 60 * 1000, result.errcode);
      return;
    } else {
      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
      if (lastErrCode == 200) {
        sendEmail(result.title, result.data);
        sendSms(result.sms);
      }
    }

    for (let i = 0; i < 3; i++) {
      result = await checkArtPuzzleLatest();
      console.log(result);
      if (!result.result) {
        console.warn("检查ArtPuzzle服务出现异常，一分钟后重试");
        await delay(60 * 1000);
      } else {
        console.warn("ArtPuzzle服务一切正常");
        break;
      }
    }
    if (!result.result) {
      console.error("连续3次检查ArtPuzzle服务异常，发送通知");
      sendEmail(result.title, result.data);
      sendSms(result.sms);
      console.log("5分钟后再试");
      setTimeout(run, 5 * 60 * 1000, result.errcode);
      return;
    } else {
      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
      if (lastErrCode == 300) {
        sendEmail(result.title, result.data);
        sendSms(result.sms);
      }
    }

  } catch (err) {
    console.error(err.stack);
  }
}


function delay(ms) {
  return new Promise(done => setTimeout(done, ms));
}

module.exports = { run }


if (require.main == module) {
  run();
}