优化一轮

service/monitor/check-api.js

@@ -7,7 +7,7 @@ const SmsTemplate = require('./sms-templates');
 
 const onlineHost = 'app2.pcoloring.com';
 const offlineHost = 'color.jccytech.cn';
-const artLatestUri = 'napi/number/v8/list/art/content?column=latest';
+const artLatestUri = 'napi/number/v9/list/art/content?column=latest';
 const jigsawLatestUri = 'napi/jigsaw/mobi/list/latest';
 const artPuzzleLatestUri = 'napi/puzzle/mobi/list/latest';
 
@@ -19,41 +19,67 @@ async function checkArtLatest() {
     let resp = await axios.get(onlineUrl);
     onlineData = resp.data;
   } catch (e) {
-    console.error('Art线上latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('Art线上latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    console.error('Art线上latest数据请求出错, 请及时检查:', [e.message]);
+    return {
+      result: false,
+      errcode: 100,
+      title: 'Art线上latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
   try {
     let resp = await axios.get(offlineUrl);
     offlineData = resp.data;
   } catch (e) {
-    console.error('Art 线下latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('Art 线下latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    console.error('Art 线下latest数据请求出错, 请及时检查:', [e.message]);
+    return {
+      result: false,
+      errcode: 100,
+      title: 'Art 线下latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
   // 检查数据是否一致： 看第一条即可
   if (!onlineData.data || onlineData.data.length <= 0) {
-    sendEmail('Art 线上latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 100,
+      title: 'Art 线上latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
   if (!offlineData.data || offlineData.data.length <= 0) {
-    sendEmail('Art 线下latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 100,
+      title: 'Art 线下latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
 
   if (onlineData.data[0]._id != offlineData.data[0]._id) {
-    sendEmail('Art latest 线上线下数据不一致, 请及时检查', `${onlineUrl}<br>${offlineUrl}`);
-    sendSms(SmsTemplate.SMS_SYNC_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 100,
+      title: 'Art latest 线上线下数据不一致, 请及时检查',
+      sms: SmsTemplate.SMS_SYNC_ERROR,
+      data: [onlineUrl, offlineUrl],
+    }
   }
 
-  return true;
+  return {
+    result: true,
+    errcode: 0,
+    title: 'Art 服务恢复正常运行',
+    sms: SmsTemplate.SMS_RESUME,
+    data: ['Art 服务一切正常']
+  }
 }
 
 async function checkJigsawLatest() {
@@ -65,9 +91,13 @@ async function checkJigsawLatest() {
     onlineData = resp.data;
   } catch (e) {
     console.error('Jigsaw 线上latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('Jigsaw 线上latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 200,
+      title: 'Jigsaw 线上latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
   try {
@@ -75,30 +105,52 @@ async function checkJigsawLatest() {
     offlineData = resp.data;
   } catch (e) {
     console.error('Jigsaw 线下latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('Jigsaw 线下latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 200,
+      title: 'Jigsaw 线下latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
   // 检查数据是否一致： 看第一条即可
   if (!onlineData.data || onlineData.data.length <= 0) {
-    sendEmail('Jigsaw 线上latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 200,
+      title: 'Jigsaw 线上latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
   if (!offlineData.data || offlineData.data.length <= 0) {
-    sendEmail('Jigsaw 线下latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 200,
+      title: 'Jigsaw 线下latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
 
   if (onlineData.data[0]._id != offlineData.data[0]._id) {
-    sendEmail('Jigsaw latest 线上线下数据不一致, 请及时检查', `${onlineUrl}<br>${offlineUrl}`);
-    sendSms(SmsTemplate.SMS_SYNC_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 200,
+      title: 'Jigsaw latest 线上线下数据不一致, 请及时检查',
+      sms: SmsTemplate.SMS_SYNC_ERROR,
+      data: [onlineUrl, offlineUrl],
+    }
   }
 
-  return true;
+  return {
+    result: true,
+    errcode: 0,
+    title: 'Jigsaw 服务恢复正常运行',
+    sms: SmsTemplate.SMS_RESUME,
+    data: ['Jigsaw 服务一切正常']
+  }
 }
 
 async function checkArtPuzzleLatest() {
@@ -110,9 +162,13 @@ async function checkArtPuzzleLatest() {
     onlineData = resp.data;
   } catch (e) {
     console.error('ArtPuzzle 线上latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('ArtPuzzle 线上latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 300,
+      title: 'ArtPuzzle 线上latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
   try {
@@ -120,61 +176,147 @@ async function checkArtPuzzleLatest() {
     offlineData = resp.data;
   } catch (e) {
     console.error('ArtPuzzle 线下latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('ArtPuzzle 线下latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 300,
+      title: 'ArtPuzzle 线下latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
   if (!onlineData.data || onlineData.data.length <= 0) {
-    sendEmail('ArtPuzzle 线上latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 300,
+      title: 'ArtPuzzle 线上latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
   if (!offlineData.data || offlineData.data.length <= 0) {
-    sendEmail('ArtPuzzle 线下latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 300,
+      title: 'ArtPuzzle 线下latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
 
   // 检查数据是否一致： 看第一条即可
   if (onlineData.data[0]._id != offlineData.data[0]._id) {
-    sendEmail('ArtPuzzle latest 线上线下数据不一致, 请及时检查', `\r\n${onlineUrl}<br>${offlineUrl}`);
-    sendSms(SmsTemplate.SMS_SYNC_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 300,
+      title: 'ArtPuzzle latest 线上线下数据不一致, 请及时检查',
+      sms: SmsTemplate.SMS_SYNC_ERROR,
+      data: [onlineUrl, offlineUrl],
+    }
   }
 
-  return true;
+  return {
+    result: true,
+    errcode: 0,
+    title: 'Jigsaw 服务恢复正常运行',
+    sms: SmsTemplate.SMS_RESUME,
+    data: ['Jigsaw 服务一切正常']
+  }
 }
 
-async function run() {
+async function run(lastErrCode = 0) {
   console.log("check api...", new Date());
   try {
-    if (await checkArtLatest()) {
-      console.log("check art latest OK! ");
-    } else {
-      console.error("check art latest error! ");
+    let result = null;
+    // 如果出现异常，连续检测3次才发送通知，减少误报，因为出现不少次因为网络原因一时半会请求不到数据就报警的情况
+    for (let i = 0; i < 3; i++) {
+      result = await checkArtLatest();
+      console.log(result);
+      if (!result.result) {
+        console.warn("检查Art服务出现异常，一分钟后重试");
+        await delay(60 * 1000);
+      } else {
+        console.warn("Art服务一切正常");
+        break;
+      }
+    }
+    if (!result.result) {
+      console.error("连续3次检查Art服务异常，发送通知");
+      sendEmail(result.title, result.data);
+      sendSms(result.sms);
+      console.log("5分钟后再试");
+      setTimeout(run, 5 * 60 * 1000, result.errcode);
       return;
+    } else {
+      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
+      if (lastErrCode == 100) {
+        sendEmail(result.title, result.data);
+        sendSms(result.sms);
+      }
     }
 
-    if (await checkJigsawLatest()) {
-      console.log("check jigsaw latest OK! ");
-    } else {
-      console.error("check jigsaw latest error! ");
+    for (let i = 0; i < 3; i++) {
+      result = await checkJigsawLatest();
+      console.log(result);
+      if (!result.result) {
+        console.warn("检查Jigsaw服务出现异常，一分钟后重试");
+        await delay(60 * 1000);
+      } else {
+        console.warn("Jigsaw服务一切正常");
+        break;
+      }
+    }
+    if (!result.result) {
+      console.error("连续3次检查Jigsaw服务异常，发送通知");
+      sendEmail(result.title, result.data);
+      sendSms(result.sms);
+      console.log("5分钟后再试");
+      setTimeout(run, 5 * 60 * 1000, result.errcode);
       return;
+    } else {
+      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
+      if (lastErrCode == 200) {
+        sendEmail(result.title, result.data);
+        sendSms(result.sms);
+      }
     }
 
-    if (await checkArtPuzzleLatest()) {
-      console.log("check art puzzle latest OK! ");
-    } else {
-      console.error("check art puzzle latest error! ");
+    for (let i = 0; i < 3; i++) {
+      result = await checkArtPuzzleLatest();
+      console.log(result);
+      if (!result.result) {
+        console.warn("检查ArtPuzzle服务出现异常，一分钟后重试");
+        await delay(60 * 1000);
+      } else {
+        console.warn("ArtPuzzle服务一切正常");
+        break;
+      }
+    }
+    if (!result.result) {
+      console.error("连续3次检查ArtPuzzle服务异常，发送通知");
+      sendEmail(result.title, result.data);
+      sendSms(result.sms);
+      console.log("5分钟后再试");
+      setTimeout(run, 5 * 60 * 1000, result.errcode);
       return;
+    } else {
+      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
+      if (lastErrCode == 300) {
+        sendEmail(result.title, result.data);
+        sendSms(result.sms);
+      }
     }
+
   } catch (err) {
     console.error(err.stack);
   }
 }
 
 
+function delay(ms) {
+  return new Promise(done => setTimeout(done, ms));
+}
+
 module.exports = { run }
 
 

service/monitor/check-new.js

@@ -20,28 +20,46 @@ async function checkArtLatest() {
     data = resp.data;
   } catch (e) {
     console.error('Art线上latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('Art线上latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 100,
+      title: 'Art线上latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
 
   if (!data.data || data.data.length <= 0) {
-    sendEmail('Art 线上latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 100,
+      title: 'Art 线上latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
 
   // 检查今日是否有新的内容
   let today = new Date(new Date().setHours(0, 0, 0, 0));
   let onshelfTime = new Date(data.data[0].onshelfTime);
   if (datefns.isBefore(onshelfTime, today)) {
-    sendEmail('art 未及时上新，请检查', `${artLatestUrl}`);
-    sendSms(SmsTemplate.SMS_NO_NEW_CONTENT);
-    return false;
+    return {
+      result: false,
+      errcode: 100,
+      title: 'art 未及时上新，请检查',
+      sms: SmsTemplate.SMS_NO_NEW_CONTENT,
+      data: [artLatestUrl],
+    }
   }
 
-  return true;
+  return {
+    result: true,
+    errcode: 0,
+    title: 'Art 服务恢复正常运行',
+    sms: SmsTemplate.SMS_RESUME,
+    data: ['Art 服务一切正常']
+  }
 }
 
 async function checkHolaLatest() {
@@ -50,29 +68,47 @@ async function checkHolaLatest() {
     let resp = await axios.get(holaLatestUrl);
     data = resp.data;
   } catch (e) {
-    console.error('Art线上latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('Art线上latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    console.error('Hola线上latest数据请求出错, 请及时检查:', e.message);
+    return {
+      result: false,
+      errcode: 101,
+      title: 'Hola线上latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
 
   if (!data.data || data.data.length <= 0) {
-    sendEmail('Art 线上latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 101,
+      title: 'Hola 线上latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
 
   // 检查今日是否有新的内容
   let today = new Date(new Date().setHours(0, 0, 0, 0));
   let onshelfTime = new Date(data.data[0].onshelfTime);
   if (datefns.isBefore(onshelfTime, today)) {
-    sendEmail('hola 未及时上新，请检查', `${holaLatestUrl}`);
-    sendSms(SmsTemplate.SMS_NO_NEW_CONTENT);
-    return false;
+    return {
+      result: false,
+      errcode: 101,
+      title: 'hola 未及时上新，请检查',
+      sms: SmsTemplate.SMS_NO_NEW_CONTENT,
+      data: [holaLatestUrl],
+    }
   }
 
-  return true;
+  return {
+    result: true,
+    errcode: 0,
+    title: 'Hola 服务恢复正常运行',
+    sms: SmsTemplate.SMS_RESUME,
+    data: ['Hola 服务一切正常']
+  }
 }
 
 async function checkCasualLatest() {
@@ -81,29 +117,47 @@ async function checkCasualLatest() {
     let resp = await axios.get(casualLatestUrl);
     data = resp.data;
   } catch (e) {
-    console.error('Art线上latest数据请求出错, 请及时检查:', e.message);
-    sendEmail('Art线上latest数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    console.error('Casual线上latest数据请求出错, 请及时检查:', e.message);
+    return {
+      result: false,
+      errcode: 102,
+      title: 'Casual线上latest数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
 
   if (!data.data || data.data.length <= 0) {
-    sendEmail('Art 线上latest数据有误, 请及时检查', 'latest数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 102,
+      title: 'Casual 线上latest数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['latest数据为空'],
+    }
   }
 
   // 检查今日是否有新的内容
   let today = new Date(new Date().setHours(0, 0, 0, 0));
   let onshelfTime = new Date(data.data[0].onshelfTime);
   if (datefns.isBefore(onshelfTime, today)) {
-    sendEmail('casual 未及时上新，请检查', `${casualLatestUrl}`);
-    sendSms(SmsTemplate.SMS_NO_NEW_CONTENT);
-    return false;
+    return {
+      result: false,
+      errcode: 102,
+      title: 'casual 未及时上新，请检查',
+      sms: SmsTemplate.SMS_NO_NEW_CONTENT,
+      data: [casualLatestUrl],
+    }
   }
 
-  return true;
+  return {
+    result: true,
+    errcode: 0,
+    title: 'Casual 服务恢复正常运行',
+    sms: SmsTemplate.SMS_RESUME,
+    data: ['Casual 服务一切正常']
+  }
 }
 
 async function checkArtDaily() {
@@ -113,67 +167,172 @@ async function checkArtDaily() {
     data = resp.data;
   } catch (e) {
     console.error('Art线上daily数据请求出错, 请及时检查:', e.message);
-    sendEmail('Art线上daily数据请求出错, 请及时检查', e.message);
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 400,
+      title: 'Art线上daily数据请求出错, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: [e.message],
+    }
   }
 
 
   if (!data.data || data.data.length <= 0) {
-    sendEmail('Art 线上daily数据有误, 请及时检查', 'daily数据为空');
-    sendSms(SmsTemplate.SMS_API_ERROR);
-    return false;
+    return {
+      result: false,
+      errcode: 400,
+      title: 'Art 线上daily数据有误, 请及时检查',
+      sms: SmsTemplate.SMS_API_ERROR,
+      data: ['daily数据为空'],
+    }
   }
 
   // 检查今日是否有新的内容
   let today = new Date(new Date().setHours(0, 0, 0, 0));
   let dailyDate = new Date(data.data[0].dailyDate);
   if (datefns.isBefore(dailyDate, today)) {
-    sendEmail('art daily 未及时上新，请检查', `${artDailyUrl}`);
-    sendSms(SmsTemplate.SMS_NO_NEW_CONTENT);
-    return false;
+    return {
+      result: false,
+      errcode: 400,
+      title: 'Art daily 未及时上新，请检查',
+      sms: SmsTemplate.SMS_NO_NEW_CONTENT,
+      data: [artDailyUrl],
+    }
   }
 
-  return true;
+  return {
+    result: true,
+    errcode: 0,
+    title: 'Art Daialy 服务恢复正常运行',
+    sms: SmsTemplate.SMS_RESUME,
+    data: ['Art Daialy 服务一切正常']
+  }
 }
 
 
-async function run() {
+async function run(lastErrCode = 0) {
   console.log("check new...", new Date());
   try {
-    if (await checkArtLatest()) {
-      console.log("check art latest new OK! ");
-    } else {
-      console.error("check art latest new error! ");
+    let result = null;
+    // 如果出现异常，连续检测3次才发送通知，减少误报，因为出现不少次因为网络原因一时半会请求不到数据就报警的情况
+    for (let i = 0; i < 3; i++) {
+      result = await checkArtLatest();
+      console.log(result);
+      if (!result.result) {
+        console.warn("检查Art服务出现异常，一分钟后重试");
+        await delay(60 * 1000);
+      } else {
+        console.warn("Art服务一切正常");
+        break;
+      }
+    }
+    if (!result.result) {
+      console.error("连续3次检查Art服务异常，发送通知");
+      sendEmail(result.title, result.data);
+      sendSms(result.sms);
+      console.log("5分钟后再试");
+      setTimeout(run, 5 * 60 * 1000, result.errcode);
       return;
+    } else {
+      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
+      if (lastErrCode == 100) {
+        sendEmail(result.title, result.data);
+        sendSms(result.sms);
+      }
     }
 
-    if (await checkHolaLatest()) {
-      console.log("check hola latest new OK! ");
-    } else {
-      console.error("check hola latest new error! ");
+    result = null;
+    for (let i = 0; i < 3; i++) {
+      result = await checkHolaLatest();
+      console.log(result);
+      if (!result.result) {
+        console.warn("检查Hola服务出现异常，一分钟后重试");
+        await delay(60 * 1000);
+      } else {
+        console.warn("Hola服务一切正常");
+        break;
+      }
+    }
+    if (!result.result) {
+      console.error("连续3次检查Hola服务异常，发送通知");
+      sendEmail(result.title, result.data);
+      sendSms(result.sms);
+      console.log("5分钟后再试");
+      setTimeout(run, 5 * 60 * 1000, result.errcode);
       return;
+    } else {
+      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
+      if (lastErrCode == 101) {
+        sendEmail(result.title, result.data);
+        sendSms(result.sms);
+      }
     }
 
-    if (await checkCasualLatest()) {
-      console.log("check casual latest new OK! ");
-    } else {
-      console.error("check casual latest new error! ");
+    result = null;
+    for (let i = 0; i < 3; i++) {
+      result = await checkCasualLatest();
+      console.log(result);
+      if (!result.result) {
+        console.warn("检查Casual服务出现异常，一分钟后重试");
+        await delay(60 * 1000);
+      } else {
+        console.warn("Casual服务一切正常");
+        break;
+      }
+    }
+    if (!result.result) {
+      console.error("连续3次检查Casual服务异常，发送通知");
+      sendEmail(result.title, result.data);
+      sendSms(result.sms);
+      console.log("5分钟后再试");
+      setTimeout(run, 5 * 60 * 1000, result.errcode);
       return;
+    } else {
+      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
+      if (lastErrCode == 102) {
+        sendEmail(result.title, result.data);
+        sendSms(result.sms);
+      }
     }
 
-    if (await checkArtDaily()) {
-      console.log("check art daily new OK! ");
-    } else {
-      console.error("check art daily new error! ");
+    result = null;
+    for (let i = 0; i < 3; i++) {
+      result = await checkArtDaily();
+      console.log(result);
+      if (!result.result) {
+        console.warn("检查Art Daily服务出现异常，一分钟后重试");
+        await delay(60 * 1000);
+      } else {
+        console.warn("Art Daily服务一切正常");
+        break;
+      }
+    }
+    if (!result.result) {
+      console.error("连续3次检查Art Daily服务异常，发送通知");
+      sendEmail(result.title, result.data);
+      sendSms(result.sms);
+      console.log("5分钟后再试");
+      setTimeout(run, 5 * 60 * 1000, result.errcode);
       return;
+    } else {
+      // 如果上次时出错的情况， 那么属于服务恢复，发送恢复通知
+      if (lastErrCode == 400) {
+        sendEmail(result.title, result.data);
+        sendSms(result.sms);
+      }
     }
+
   } catch (err) {
     console.error(err.stack);
   }
 }
 
 
+function delay(ms) {
+  return new Promise(done => setTimeout(done, ms));
+}
+
+
 module.exports = { run }
 
 

service/monitor/email.js

@@ -5,14 +5,14 @@ const utils = require('../../libs/utils');
 const ejs = require('ejs');
 
 
-async function sendEmail(title, data) {
-  await reportEmail(title, data);
+async function sendEmail(title, datalist) {
+  await reportEmail(title, datalist);
 }
 
-async function reportEmail(title, data) {
+async function reportEmail(title, datalist) {
   let html = await renderFile(__dirname + '/monitor.ejs', {
     title,
-    data,
+    datalist,
     now: new Date(),
     format: (str, fmt) => {
       fmt = 'yyyy/M/d HH:mm'
@@ -45,5 +45,5 @@ module.exports = { sendEmail }
 
 
 if (require.main == module) {
-  sendEmail("Monitor Heartbeat", "heartbeat");
+  sendEmail("Monitor Heartbeat", ["heartbeat"]);
 }

service/monitor/heartbeat.js

@@ -7,7 +7,7 @@ const SmsTemplate = require('./sms-templates');
 async function run() {
   console.log("heartbeat...", new Date());
   try {
-    sendEmail("Monitor Hearbeat", "heartbeat");
+    sendEmail("Monitor Hearbeat", ["heartbeat"]);
     sendSms(SmsTemplate.SMS_HEARTBEAT);
   } catch (err) {
     console.error(err.stack);

service/monitor/monitor.ejs

@@ -4,7 +4,9 @@
 <head>
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-  <title><%= title %></title>
+  <title>
+    <%= title %>
+  </title>
   <style>
 
   </style>
@@ -14,9 +16,15 @@
   style="background-color: #f6f6f6; font-family: sans-serif; -webkit-font-smoothing: antialiased; font-size: 14px; line-height: 1.4; margin: 0; padding: 0; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%;">
 
 
-  <p>
-    <%= format(now)%>: <%= data %>
-  </p>
+  <div>
+    <%= format(now)%>
+  </div>
+
+  <% datalist.forEach(data=> { %>
+    <div>
+      <%= data %>
+    </div>
+    <% }) %>
 
 
 </body>

service/monitor/sms-templates.js

@@ -3,4 +3,5 @@ module.exports = {
   SMS_NO_NEW_CONTENT: '2341062',  // 没有新的发布内容
   SMS_SYNC_ERROR: '2341060', // 线上数据不同步
   SMS_HEARTBEAT: '2341055',  // monitor心跳提醒
+  SMS_RESUME: '2394348',  // 服务恢复正常
 }