artsite/service/cron-jobs/sitemap.js

/**
 * 每天运行一次，生成站点地图
 */

const fs = require('fs');
const datefns = require('date-fns');
const models = require('../../models');
const categories = require('../../config/category');
const { tags } = require('../../config/tag');
const date = datefns.format(Date.now(), 'yyyy-MM-dd');

const sitemapPath = __dirname + '/../../dist/sitemap.xml';
const sitemapTempPath = __dirname + '/../../dist/sitemap.xml.temp';

// 生成站点地图
async function generateSitemap() {
  let commonXml = [
    '<?xml version="1.0" encoding="UTF-8"?>',
    // '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
    '<urlset',
    '  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"',
    '  xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd http://www.w3.org/1999/xhtml http://www.w3.org/2002/08/xhtml/xhtml1-strict.xsd"',
    '  xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"',
    '  xmlns:xhtml="http://www.w3.org/1999/xhtml"',
    '>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja" />',
    `    <lastmod>${date}</lastmod>`,
    '    <changefreq>daily</changefreq>',
    '    <priority>1.0</priority>',
    '  </url>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en/gallery</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/gallery" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/gallery" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/gallery" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/gallery" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/gallery" />',
    `    <lastmod>${date}</lastmod>`,
    '    <changefreq>weekly</changefreq>',
    '    <priority>0.8</priority>',
    '  </url>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en/videos</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/videos" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/videos" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/videos" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/videos" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/videos" />',
    `    <lastmod>${date}</lastmod>`,
    '    <changefreq>weekly</changefreq>',
    '    <priority>0.8</priority>',
    '  </url>',
    // '  <url>',
    // '    <loc>https://art.pcoloring.com/en/category</loc>',
    // `    <lastmod>${date}</lastmod>`,
    // '    <changefreq>daily</changefreq>',
    // '    <priority>0.8</priority>',
    // '  </url>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en/tag</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/tag" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/tag" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/tag" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/tag" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/tag" />',
    `    <lastmod>${date}</lastmod>`,
    '    <changefreq>daily</changefreq>',
    '    <priority>0.8</priority>',
    '  </url>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en/albums</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/albums" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/albums" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/albums" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/albums" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/albums" />',
    `    <lastmod>${date}</lastmod>`,
    '    <changefreq>weekly</changefreq>',
    '    <priority>0.8</priority>',
    '  </url>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en/special</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/special" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/special" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/special" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/special" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/special" />',
    `    <lastmod>${date}</lastmod>`,
    '    <changefreq>daily</changefreq>',
    '    <priority>0.8</priority>',
    '  </url>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en/artists</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/artists" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/artists" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/artists" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/artists" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/artists" />',
    `    <lastmod>${date}</lastmod>`,
    '    <changefreq>monthly</changefreq>',
    '    <priority>0.8</priority>',
    '  </url>',
    '  <url>',
    '    <loc>https://art.pcoloring.com/en/info</loc>',
    '    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/info" />',
    '    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/info" />',
    '    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/info" />',
    '    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/info" />',
    '    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/info" />',
    `    <lastmod>${date}</lastmod>`,
    '  </url>',
  ];

  // categories
  // let categoriesXml = [];
  // categories.forEach(e => {
  //   categoriesXml = categoriesXml.concat([
  //     '  <url>',
  //     `    <loc>https://art.pcoloring.com/en/category/${e.id}</loc>`,
  //     `    <lastmod>${date}</lastmod>`,
  //     '  </url>',
  //   ]
  //   );
  // });

  // console.log(categoriesXml);

  // tags
  let tagsXml = [];
  let newtags = tags.filter(e => e.count && e.count > 0);
  newtags.forEach(e => {
    tagsXml = tagsXml.concat([
      '  <url>',
      `    <loc>https://art.pcoloring.com/en/tag/${e.tag}</loc>`,
      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/tag/${e.tag}" />`,
      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/tag/${e.tag}" />`,
      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/tag/${e.tag}" />`,
      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/tag/${e.tag}" />`,
      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/tag/${e.tag}" />`,
      `    <lastmod>${date}</lastmod>`,
      '  </url>',
    ]);
  });
  console.log(tagsXml);

  // albums
  let albums = await models.ArtAlbum
    .find({ pid: 'art', enabled: true })
    .sort({ order: 'asc' })
    .select('_id')
    .lean()
    .exec();
  let albumsXml = [];
  albums.forEach(e => {
    albumsXml = albumsXml.concat([
      '  <url>',
      `    <loc>https://art.pcoloring.com/en/coloring-page-album/${e._id}</loc>`,
      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/coloring-page-album/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/coloring-page-album/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/coloring-page-album/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/coloring-page-album/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/coloring-page-album/${e._id}" />`,
      `    <lastmod>${date}</lastmod>`,
      '  </url>',
    ])
  });
  console.log(albumsXml);

  // designers
  let designers = await models.Art.aggregate([
    // 首先，过滤出 status = 9000 的文档
    { $match: { open: true, status: 9000 } },

    // 首先，根据 user 字段进行分组，并计算每个 user 出现的次数
    { $group: { _id: '$user', count: { $sum: 1 } } },

    // 然后，按照 count 字段进行降序排列
    { $sort: { count: -1 } },

    // 接着，与 users 集合进行连接，以获取用户的详细信息
    {
      $lookup: {
        from: 'users', // 要连接的集合名称
        localField: '_id', // 本地字段（即上一步分组后的 _id 字段）
        foreignField: '_id', // 要连接的集合中的字段
        as: 'userDetails' // 连接后结果存储在新字段 userDetails 中
      }
    },

    // 展开 userDetails 数组，以便将用户信息提升到顶层
    { $unwind: '$userDetails' },

    // 调整输出格式，只保留需要的字段
    { $project: { _id: 1, user: '$_id', count: 1, username: '$userDetails.username', name: '$userDetails.name' } },

    // 限制返回的记录数量
    { $limit: 40 }
  ]);
  let designersXml = [];
  designers.forEach(e => {
    designersXml = designersXml.concat([
      '  <url>',
      `    <loc>https://art.pcoloring.com/en/artist/${e._id}</loc>`,
      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/artist/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/artist/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/artist/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/artist/${e._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/artist/${e._id}" />`,
      `    <lastmod>${date}</lastmod>`,
      '  </url>',
    ])
  });
  console.log(designersXml);

  // details
  let detailsXml = [];
  const cursor = await models.Art
    .find({ open: true, status: 9000 })
    .select('name title')
    .sort({ publishTime: -1 }).cursor();  // 使用cursor，避免一次查询太多记录内存受不了
  // 遍历游标中的每个文档
  for await (const doc of cursor) {
    let uriTitle = doc.name;
    if (doc.title) {
      try {
        let titleJson = JSON.parse(doc.title);
        if (titleJson.en) uriTitle = titleJson.en;  // URL 里还是尽量用英文title
      } catch (e) {
        console.error(e.message);
      }
    }
    let utf8name = encodeURIComponent(uriTitle.replace(/[\s_]+/g, '-')).toLowerCase();
    detailsXml = detailsXml.concat([
      '  <url>',
      `    <loc>https://art.pcoloring.com/en/coloring-page/${utf8name}-${doc._id}</loc>`,
      `    <xhtml:link rel="alternate" hreflang="en" href="https://art.pcoloring.com/en/coloring-page/${utf8name}-${doc._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="zh" href="https://art.pcoloring.com/zh/coloring-page/${utf8name}-${doc._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="es" href="https://art.pcoloring.com/es/coloring-page/${utf8name}-${doc._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="pt" href="https://art.pcoloring.com/pt/coloring-page/${utf8name}-${doc._id}" />`,
      `    <xhtml:link rel="alternate" hreflang="ja" href="https://art.pcoloring.com/ja/coloring-page/${utf8name}-${doc._id}" />`,
      `    <lastmod>${date}</lastmod>`,
      '  </url>',
    ]);
    console.log(`process detail ${doc._id} url done`);
  }

  let endXml = ['</urlset>'];

  let sitemapXml = [...commonXml, ...tagsXml, ...albumsXml, ...designersXml, ...detailsXml, ...endXml];

  let sitemapXmlStr = sitemapXml.join('\n');

  fs.writeFileSync(sitemapTempPath, sitemapXmlStr);  // 先写入临时文件
  fs.renameSync(sitemapTempPath, sitemapPath); // 再重命名

  console.log('生成sitemap成功');
}

function run() {
  generateSitemap();
}

module.exports = { run }


if (require.main == module) {
  run();
}