Generate sitemap server-side

web/next-sitemap.js

@@ -1,43 +1,15 @@
-const https = require('https')
-
 /** @type {import('next-sitemap').IConfig} */
 
 module.exports = {
   siteUrl: process.env.SITE_URL || 'https://manifold.markets',
   changefreq: 'hourly',
   priority: 0.7, // Set high priority by default
-  additionalPaths,
-  exclude: ['/admin'],
+  exclude: ['/admin', '/server-sitemap.xml'],
   generateRobotsTxt: true,
+  robotsTxtOptions: {
+    additionalSitemaps: [
+      'https://manifold.markets/server-sitemap.xml', // <==== Add here
+    ],
+  },
   // Other options: https://github.com/iamvishnusankar/next-sitemap#configuration-options
 }
-
-// See https://github.com/iamvishnusankar/next-sitemap#additional-paths-function
-async function additionalPaths(config) {
-  // Fetching data from https://docs.manifold.markets/api
-  const response = await fetch(`${config.siteUrl}/api/v0/markets`)
-
-  const liteMarkets = await response
-  // See https://www.sitemaps.org/protocol.html
-  return liteMarkets.map((liteMarket) => ({
-    loc: liteMarket.url,
-    changefreq: 'hourly',
-    priority: 0.2, // Individual markets aren't that important
-    // TODO: Add `lastmod` aka last modified time
-  }))
-}
-
-// Polyfill for fetch: get the JSON contents of a URL
-async function fetch(url) {
-  return new Promise((resolve, reject) => {
-    https.get(url, (res) => {
-      let data = ''
-      res.on('data', (chunk) => {
-        data += chunk
-      })
-      res.on('end', () => {
-        resolve(JSON.parse(data))
-      })
-    })
-  })
-}

web/pages/server-sitemap.xml.tsx

@@ -0,0 +1,21 @@
+import { GetServerSideProps } from 'next'
+import { getServerSideSitemap } from 'next-sitemap'
+import { DOMAIN } from '../../common/envs/constants'
+
+export const getServerSideProps: GetServerSideProps = async (ctx) => {
+  // Fetching data from https://docs.manifold.markets/api
+  const response = await fetch(`https://${DOMAIN}/api/v0/markets`)
+
+  const liteMarkets = await response.json()
+  const fields = liteMarkets.map((liteMarket: any) => ({
+    // See https://www.sitemaps.org/protocol.html
+    loc: liteMarket.url,
+    changefreq: 'hourly',
+    priority: 0.2, // Individual markets aren't that important
+    // TODO: Add `lastmod` aka last modified time
+  }))
+  return getServerSideSitemap(ctx, fields)
+}
+
+// Default export to prevent next.js errors
+export default function Sitemap() {}

web/public/robots.txt

@@ -0,0 +1,10 @@
+# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://manifold.markets
+
+# Sitemaps
+Sitemap: https://manifold.markets/sitemap.xml
+Sitemap: https://manifold.markets/server-sitemap.xml

web/public/sitemap-0.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">
+<url><loc>https://manifold.markets</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/about</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/account</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/add-funds</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/analytics</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/create</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/embed/analytics</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/folds</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/home</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/landing-page</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/leaderboards</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/make-predictions</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/markets</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/profile</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/simulator</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+<url><loc>https://manifold.markets/trades</loc><changefreq>hourly</changefreq><priority>0.7</priority><lastmod>2022-03-24T16:51:19.526Z</lastmod></url>
+</urlset>

web/public/sitemap.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+<sitemap><loc>https://manifold.markets/sitemap-0.xml</loc></sitemap>
+</sitemapindex>