add sync

bin/coloring-sync

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+const appConfig = require('../config/app');
+const syncService = require('../sync/sync-service');
+
+if (appConfig.sync == 'slave') {
+  syncService.start().catch(console.error);
+}

config/app/index.js

@@ -16,6 +16,8 @@ let configs = {
   host: 'http://art.pcoloring.com',
   resHost: 'http://pcoloring.com',
   STATIC_DIR: path.resolve(os.homedir(), 'www/artsite'),
+  sync: 'slave',  // master or slave
+  syncUrl: 'mongodb://coloring:coloring123.@hk.jccytech.cn:7881?authSource=admin',
 }
 
 

config/app/production.js

@@ -9,4 +9,5 @@ module.exports = {
   },
   mongodbUrl: 'mongodb://coloring:coloring123.@localhost:62701/artsite?authSource=admin',
   host: 'http://art.pcoloring.com',
+  resHost: 'http://pcoloring.com',
 }

package-lock.json

@@ -20,6 +20,7 @@
         "express-session": "^1.18.1",
         "moment": "^2.30.1",
         "mongoose": "^8.9.5",
+        "node-cron": "^3.0.3",
         "node-fetch": "^2.7.0",
         "sharp": "^0.33.5"
       }
@@ -1287,6 +1288,17 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/node-cron": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz",
+      "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==",
+      "dependencies": {
+        "uuid": "8.3.2"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
     "node_modules/node-fetch": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
@@ -1763,6 +1775,14 @@
         "node": ">= 0.4.0"
       }
     },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",

package.json

@@ -21,6 +21,7 @@
     "express-session": "^1.18.1",
     "moment": "^2.30.1",
     "mongoose": "^8.9.5",
+    "node-cron": "^3.0.3",
     "node-fetch": "^2.7.0",
     "sharp": "^0.33.5"
   }

service/cron-jobs/ai-seo.js

@@ -0,0 +1,22 @@
+const { format, subMinutes } = require('date-fns');
+const models = require('../../models');
+const utils = require('../../libs/utils');
+
+async function run() {
+  let now = new Date();
+
+  // 删选出所有已经ready并且还没有title的图
+  let query = { status: { $gte: 7000 }, $or: [{ title: { $exists: false } }, { title: null }] };
+  let docs = await models.Art.find(query);
+
+
+
+}
+
+
+
+module.exports = { run }
+
+
+if (require.main == module) {
+}

service/cron-jobs/index.js

@@ -0,0 +1,20 @@
+const cron = require('node-cron');
+
+const settings = [
+  ['ai-seo', '* * * * *', require('./ai-seo')],  // 每个小时跑一遍，ai自动生成标题和文案
+]
+
+
+
+settings.forEach(setting => {
+  let [name, schedule, job] = setting;
+  if (!job.run) throw new Error(`Job [${name}] has no run() function`);
+  console.log(`[${name}] run@'${schedule}' installed!`);
+  cron.schedule(schedule, () => {
+    let date = new Date().toLocaleString();
+    console.log(`[${name}]@'${schedule}' running @ ${date}`);
+    if (job.run) {
+      job.run().then(console.log).catch(console.error);
+    }
+  })
+})

sync/schema-sync-seq.js

@@ -0,0 +1,8 @@
+const { Schema } = require("mongoose");
+
+// slave 记录同步event sequence
+let syncSeqSchema = new Schema({
+  seq: { type: Number, required: true, desc: '同步seq，只有一条记录' }
+});
+
+module.exports = syncSeqSchema;

sync/sync-conn.js

@@ -0,0 +1,7 @@
+const mongoose = require('mongoose');
+const appConfig = require('../config/app');
+
+const syncConn = mongoose.createConnection(appConfig.mongodbUrl);
+mongoose.Promise = require('bluebird');
+
+module.exports = syncConn;

sync/sync-seq.js

@@ -0,0 +1,7 @@
+const syncConn = require('./sync-conn');
+const syncSeqSchema = require('./schema-sync-seq');
+
+
+const SyncSeq = syncConn.model('SyncSeq', syncSeqSchema);
+
+module.exports = SyncSeq;

sync/sync-service.js

@@ -0,0 +1,110 @@
+const utils = require('../libs/utils');
+const path = require('path');
+const MongoClient = require('mongoose').mongo.MongoClient;
+const appConfig = require('../config/app');
+const SyncSeq = require('./sync-seq');
+const { format } = require('date-fns');
+
+const dbs = ['coloring_ol'];  // 需要同步的数据库, 主数据库（即syncevent表所在的库）放在第一个
+
+/**
+ * start sync from remote
+ */
+async function start() {
+  let delay = 30 * 1000; // sync freq
+  let localDBHash = {};
+  let remoteDBHash = {};
+  const localClient = await new MongoClient(appConfig.mongodbUrl).connect();
+  const remoteClient = await new MongoClient(appConfig.syncUrl).connect();
+  for (let db of dbs) {
+    localDBHash[db] = localClient.db(db);
+    remoteDBHash[db] = remoteClient.db(db);
+  }
+  console.log('Connect to remote mongodb ' + appConfig.syncUrl + ' success!');
+
+  // 读取远程syncevent表
+  const synceventTB = remoteDBHash[dbs[0]].collection('syncevents');
+  // get current local slave sync seq (use mongoose)
+  let seq = -1;
+  let seqDoc = await SyncSeq.findOne();
+  if (seqDoc) seq = seqDoc.seq;
+  else seqDoc = new SyncSeq({});
+
+  let count, localtb, remotetb, localdoc, remotedoc;
+  setTimeout(async function run() {
+    try {
+      let eventDocs = await synceventTB.find({ _id: { $gt: seq } }).limit(200).toArray() || [];
+      count = 0;
+      for (let i = 0; i < eventDocs.length; i++) {
+        let eventDoc = eventDocs[i];
+        console.log(eventDoc);
+        if (eventDoc.tb == 'identitycounters') continue;  // skip
+        // 只同步特定的几张表： arts， daily-arts， users， albums， translates
+        if (eventDoc.tb != 'arts'
+          && eventDoc.tb != 'daily-arts'
+          && eventDoc.tb != 'users'
+          && eventDoc.tb != 'albums'
+          && eventDoc.tb != 'translates') {
+          continue;
+        }
+
+        if (!eventDoc.db) eventDoc.db = dbs[0];
+        // 对应的表
+        remotetb = remoteDBHash[eventDoc.db] ? remoteDBHash[eventDoc.db].collection(eventDoc.tb) : null;
+        localtb = localDBHash[eventDoc.db] ? localDBHash[eventDoc.db].collection(eventDoc.tb) : null;
+        if (!remotetb || !localtb) {
+          console.warn('unkown sync database: ' + eventDoc.db + ', will skip');
+          continue;
+        }
+
+        if (eventDoc.op == 'remove') {
+          await localtb.deleteOne({ _id: eventDoc.rid });
+          console.log("sync remove : " + eventDoc.tb + " " + eventDoc.rid);
+        } else if (eventDoc.op == 'save') {
+          remotedoc = await remotetb.findOne({ _id: eventDoc.rid });
+          localdoc = await localtb.findOne({ _id: eventDoc.rid });
+          if (!remotedoc) {
+            console.log("remote doc not found, may be deleted, skip: " + eventDoc.tb + " " + eventDoc.rid);
+          } else if (!localdoc) {
+            console.log("sync add :" + eventDoc.tb + " " + eventDoc.rid);
+            try {
+              await localtb.insertOne(remotedoc);
+            } catch (e) {
+              console.error(e);
+            }
+          } else {
+            console.log("sync update :" + eventDoc.tb + " " + eventDoc.rid);
+            // 删除desc和title字段，避免覆盖
+            delete remotedoc.desc;
+            await localtb.replaceOne({ _id: eventDoc.rid }, remotedoc);
+          }
+        }
+
+        seq = eventDoc._id;
+        seqDoc.seq = seq;
+        await seqDoc.save();
+        count++;
+      }
+
+      console.log(`${format(new Date(), 'yyyy-MM-dd HH:mm')} sync event length: ${eventDocs.length}, precessed: ${count}`);
+
+    } catch (err) {
+      console.error(err);
+    }
+
+    if (count > 0) setTimeout(run, 0);
+    else setTimeout(run, delay);
+
+  }, 0);
+
+}
+
+
+module.exports = {
+  start
+}
+
+
+if (require.main == module) {
+  start().catch(console.error);
+}