add sitemap cron job

routes/index.js

@@ -186,6 +186,11 @@ router.get(/^\/(en|zh|es|pt|ja)$/, function (req, res, next) {  // 限制严格
 });
 
 
+// 分类页不带tag，redirect 到 latest
+router.get('/:lang/category', (req, res, next) => {
+  return res.redirect(`${req.originalUrl}/latest`);
+});
+
 // 分类页路由
 router.get('/:lang/category/:tag?', function (req, res, next) {
   (async function () {

service/cron-jobs/index.js

@@ -4,6 +4,7 @@ const settings = [
   ['sync', '0 8 * * *', require('../../sync/sync-service')],  // 每天早上8点同步一次
   ['fetch-meta', '10 8 * * *', require('./fetch-meta')],  // 每天早上8点10分跑一次，跟在sync同步之后，ai自动生成标题和文案
   ['open-art', '0 10 * * *', require('./open-art')],  // 每天早上10点跑一次，开放部分内容
+  ['sitemap', '12 0 * * *', require('./open-art')],  // 每天0点12分生成一次sitemap
 ]
 
 

service/cron-jobs/sitemap.js

@@ -0,0 +1,249 @@
+/**
+ * 每天运行一次，生成站点地图
+ */
+
+const fs = require('fs');
+const datefns = require('date-fns');
+const models = require('../../models');
+const categories = require('../../config/category');
+const tags = require('../../config/tag');
+const date = datefns.format(Date.now(), 'yyyy-MM-dd');
+
+const sitemapPath = __dirname + '/../../dist/sitemap.xml';
+const sitemapTempPath = __dirname + '/../../dist/sitemap.xml.temp';
+
+// 生成站点地图
+async function generateSitemap() {
+  let commonXml = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    // '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+    '<urlset',
+    '  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"',
+    '  xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd http://www.w3.org/1999/xhtml http://www.w3.org/2002/08/xhtml/xhtml1-strict.xsd"',
+    '  xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"',
+    '  xmlns:xhtml="http://www.w3.org/1999/xhtml"',
+    '>',
+    '  <url>',
+    '    <loc>https://art.pcoloring.com</loc>',
+    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en" />',
+    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es" />',
+    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt" />',
+    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja" />',
+    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh" />',
+    `    <lastmod>${date}</lastmod>`,
+    '  </url>',
+    '  <url>',
+    '    <loc>https://art.pcoloring.com/en/category</loc>',
+    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/category" />',
+    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/category" />',
+    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/category" />',
+    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/category" />',
+    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/category" />',
+    `    <lastmod>${date}</lastmod>`,
+    '  </url>',
+    '  <url>',
+    '    <loc>https://art.pcoloring.com/en/tag</loc>',
+    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/tag" />',
+    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/tag" />',
+    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/tag" />',
+    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/tag" />',
+    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/tag" />',
+    `    <lastmod>${date}</lastmod>`,
+    '  </url>',
+    '  <url>',
+    '    <loc>https://art.pcoloring.com/en/albums</loc>',
+    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/albums" />',
+    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/albums" />',
+    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/albums" />',
+    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/albums" />',
+    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/albums" />',
+    `    <lastmod>${date}</lastmod>`,
+    '  </url>',
+    '  <url>',
+    '    <loc>https://art.pcoloring.com/en/special</loc>',
+    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/special" />',
+    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/special" />',
+    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/special" />',
+    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/special" />',
+    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/special" />',
+    `    <lastmod>${date}</lastmod>`,
+    '  </url>',
+    '  <url>',
+    '    <loc>https://art.pcoloring.com/en/designers</loc>',
+    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/designers" />',
+    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/designers" />',
+    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/designers" />',
+    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/designers" />',
+    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/designers" />',
+    `    <lastmod>${date}</lastmod>`,
+    '  </url>',
+    '  <url>',
+    '    <loc>https://art.pcoloring.com/en/info</loc>',
+    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/info" />',
+    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/info" />',
+    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/info" />',
+    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/info" />',
+    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/info" />',
+    `    <lastmod>${date}</lastmod>`,
+    '  </url>',
+  ];
+
+  // categories
+  let categoriesXml = [];
+  categories.forEach(e => {
+    categoriesXml = categoriesXml.concat([
+      '  <url>',
+      `    <loc>https://art.pcoloring.com/en/category/${e.id}</loc>`,
+      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/category/${e.id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/category/${e.id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/category/${e.id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/category/${e.id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/category/${e.id}" />`,
+      `    <lastmod>${date}</lastmod>`,
+      '  </url>',
+    ]
+    );
+  });
+
+  console.log(categoriesXml);
+
+  // tags
+  let tagsXml = [];
+  tags.forEach(e => {
+    tagsXml = tagsXml.concat([
+      '  <url>',
+      `    <loc>https://art.pcoloring.com/en/tag/${e.tag}</loc>`,
+      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/tag/${e.tag}" />`,
+      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/tag/${e.tag}" />`,
+      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/tag/${e.tag}" />`,
+      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/tag/${e.tag}" />`,
+      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/tag/${e.tag}" />`,
+      `    <lastmod>${date}</lastmod>`,
+      '  </url>',
+    ]);
+  });
+  console.log(tagsXml);
+
+  // albums
+  let albums = await models.ArtAlbum
+    .find({ pid: 'art', enabled: true })
+    .sort({ order: 'asc' })
+    .select('_id')
+    .lean()
+    .exec();
+  let albumsXml = [];
+  albums.forEach(e => {
+    albumsXml = albumsXml.concat([
+      '  <url>',
+      `    <loc>https://art.pcoloring.com/en/album/${e._id}</loc>`,
+      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/album/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/album/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/album/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/album/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/album/${e._id}" />`,
+      `    <lastmod>${date}</lastmod>`,
+      '  </url>',
+    ])
+  });
+  console.log(albumsXml);
+
+  // designers
+  let designers = await models.Art.aggregate([
+    // 首先，过滤出 status = 9000 的文档
+    { $match: { open: true, status: 9000 } },
+
+    // 首先，根据 user 字段进行分组，并计算每个 user 出现的次数
+    { $group: { _id: '$user', count: { $sum: 1 } } },
+
+    // 然后，按照 count 字段进行降序排列
+    { $sort: { count: -1 } },
+
+    // 接着，与 users 集合进行连接，以获取用户的详细信息
+    {
+      $lookup: {
+        from: 'users', // 要连接的集合名称
+        localField: '_id', // 本地字段（即上一步分组后的 _id 字段）
+        foreignField: '_id', // 要连接的集合中的字段
+        as: 'userDetails' // 连接后结果存储在新字段 userDetails 中
+      }
+    },
+
+    // 展开 userDetails 数组，以便将用户信息提升到顶层
+    { $unwind: '$userDetails' },
+
+    // 调整输出格式，只保留需要的字段
+    { $project: { _id: 1, user: '$_id', count: 1, username: '$userDetails.username', name: '$userDetails.name' } },
+
+    // 限制返回的记录数量
+    { $limit: 40 }
+  ]);
+  let designersXml = [];
+  designers.forEach(e => {
+    designersXml = designersXml.concat([
+      '  <url>',
+      `    <loc>https://art.pcoloring.com/en/designer/${e._id}</loc>`,
+      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/designer/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/designer/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/designer/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/designer/${e._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/designer/${e._id}" />`,
+      `    <lastmod>${date}</lastmod>`,
+      '  </url>',
+    ])
+  });
+  console.log(designersXml);
+
+  // details
+  let detailsXml = [];
+  const cursor = await models.Art
+    .find({ open: true, status: 9000 })
+    .select('name title')
+    .sort({ publishTime: -1 }).cursor();  // 使用cursor，避免一次查询太多记录内存受不了
+  // 遍历游标中的每个文档
+  for await (const doc of cursor) {
+    let uriTitle = doc.name;
+    if (doc.title) {
+      try {
+        let titleJson = JSON.parse(doc.title);
+        if (titleJson.en) uriTitle = titleJson.en;  // URL 里还是尽量用英文title
+      } catch (e) {
+        console.error(e.message);
+      }
+    }
+    let utf8name = encodeURIComponent(uriTitle.replace(/[\s_]+/g, '-')).toLowerCase();
+    detailsXml = detailsXml.concat([
+      '  <url>',
+      `    <loc>https://art.pcoloring.com/en/coloring-page/${utf8name}-${doc._id}</loc>`,
+      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/coloring-page/${utf8name}-${doc._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/coloring-page/${utf8name}-${doc._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/coloring-page/${utf8name}-${doc._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/coloring-page/${utf8name}-${doc._id}" />`,
+      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/coloring-page/${utf8name}-${doc._id}" />`,
+      `    <lastmod>${date}</lastmod>`,
+      '  </url>',
+    ]);
+    console.log(`process detail ${doc._id} url done`);
+  }
+
+  let endXml = ['</urlset>'];
+
+  let sitemapXml = [...commonXml, ...categoriesXml, ...tagsXml, ...albumsXml, ...designersXml, ...detailsXml, ...endXml];
+
+  let sitemapXmlStr = sitemapXml.join('\n');
+
+  fs.writeFileSync(sitemapTempPath, sitemapXmlStr);  // 先写入临时文件
+  fs.renameSync(sitemapTempPath, sitemapPath); // 再重命名
+
+  console.log('生成sitemap成功');
+}
+
+function run() {
+  generateSitemap();
+}
+
+module.exports = { run }
+
+
+if (require.main == module) {
+  run();
+}