From b6281b0b56349e5ec4a877bd25787238d1faa0dd Mon Sep 17 00:00:00 2001 From: Austin Chen Date: Thu, 24 Mar 2022 09:52:13 -0700 Subject: [PATCH] Generate sitemap server-side --- web/next-sitemap.js | 40 +++++--------------------------- web/pages/server-sitemap.xml.tsx | 21 +++++++++++++++++ web/public/robots.txt | 10 ++++++++ web/public/sitemap-0.xml | 19 +++++++++++++++ web/public/sitemap.xml | 4 ++++ 5 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 web/pages/server-sitemap.xml.tsx create mode 100644 web/public/robots.txt create mode 100644 web/public/sitemap-0.xml create mode 100644 web/public/sitemap.xml diff --git a/web/next-sitemap.js b/web/next-sitemap.js index 3298f52b..cd6c9c35 100644 --- a/web/next-sitemap.js +++ b/web/next-sitemap.js @@ -1,43 +1,15 @@ -const https = require('https') - /** @type {import('next-sitemap').IConfig} */ module.exports = { siteUrl: process.env.SITE_URL || 'https://manifold.markets', changefreq: 'hourly', priority: 0.7, // Set high priority by default - additionalPaths, - exclude: ['/admin'], + exclude: ['/admin', '/server-sitemap.xml'], generateRobotsTxt: true, + robotsTxtOptions: { + additionalSitemaps: [ + 'https://manifold.markets/server-sitemap.xml', // <==== Add here + ], + }, // Other options: https://github.com/iamvishnusankar/next-sitemap#configuration-options } - -// See https://github.com/iamvishnusankar/next-sitemap#additional-paths-function -async function additionalPaths(config) { - // Fetching data from https://docs.manifold.markets/api - const response = await fetch(`${config.siteUrl}/api/v0/markets`) - - const liteMarkets = await response - // See https://www.sitemaps.org/protocol.html - return liteMarkets.map((liteMarket) => ({ - loc: liteMarket.url, - changefreq: 'hourly', - priority: 0.2, // Individual markets aren't that important - // TODO: Add `lastmod` aka last modified time - })) -} - -// Polyfill for fetch: get the JSON contents of a URL -async function fetch(url) { - return new Promise((resolve, reject) => { - https.get(url, (res) => { - let data = '' - res.on('data', (chunk) => { - data += chunk - }) - res.on('end', () => { - resolve(JSON.parse(data)) - }) - }) - }) -} diff --git a/web/pages/server-sitemap.xml.tsx b/web/pages/server-sitemap.xml.tsx new file mode 100644 index 00000000..8625860c --- /dev/null +++ b/web/pages/server-sitemap.xml.tsx @@ -0,0 +1,21 @@ +import { GetServerSideProps } from 'next' +import { getServerSideSitemap } from 'next-sitemap' +import { DOMAIN } from '../../common/envs/constants' + +export const getServerSideProps: GetServerSideProps = async (ctx) => { + // Fetching data from https://docs.manifold.markets/api + const response = await fetch(`https://${DOMAIN}/api/v0/markets`) + + const liteMarkets = await response.json() + const fields = liteMarkets.map((liteMarket: any) => ({ + // See https://www.sitemaps.org/protocol.html + loc: liteMarket.url, + changefreq: 'hourly', + priority: 0.2, // Individual markets aren't that important + // TODO: Add `lastmod` aka last modified time + })) + return getServerSideSitemap(ctx, fields) +} + +// Default export to prevent next.js errors +export default function Sitemap() {} diff --git a/web/public/robots.txt b/web/public/robots.txt new file mode 100644 index 00000000..014904fd --- /dev/null +++ b/web/public/robots.txt @@ -0,0 +1,10 @@ +# * +User-agent: * +Allow: / + +# Host +Host: https://manifold.markets + +# Sitemaps +Sitemap: https://manifold.markets/sitemap.xml +Sitemap: https://manifold.markets/server-sitemap.xml diff --git a/web/public/sitemap-0.xml b/web/public/sitemap-0.xml new file mode 100644 index 00000000..9db55606 --- /dev/null +++ b/web/public/sitemap-0.xml @@ -0,0 +1,19 @@ + + +https://manifold.marketshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/abouthourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/accounthourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/add-fundshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/analyticshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/createhourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/embed/analyticshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/foldshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/homehourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/landing-pagehourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/leaderboardshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/make-predictionshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/marketshourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/profilehourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/simulatorhourly0.72022-03-24T16:51:19.526Z +https://manifold.markets/tradeshourly0.72022-03-24T16:51:19.526Z + \ No newline at end of file diff --git a/web/public/sitemap.xml b/web/public/sitemap.xml new file mode 100644 index 00000000..050639f2 --- /dev/null +++ b/web/public/sitemap.xml @@ -0,0 +1,4 @@ + + +https://manifold.markets/sitemap-0.xml + \ No newline at end of file