diff --git a/web/next-sitemap.js b/web/next-sitemap.js
index 3298f52b..cd6c9c35 100644
--- a/web/next-sitemap.js
+++ b/web/next-sitemap.js
@@ -1,43 +1,15 @@
-const https = require('https')
-
/** @type {import('next-sitemap').IConfig} */
module.exports = {
siteUrl: process.env.SITE_URL || 'https://manifold.markets',
changefreq: 'hourly',
priority: 0.7, // Set high priority by default
- additionalPaths,
- exclude: ['/admin'],
+ exclude: ['/admin', '/server-sitemap.xml'],
generateRobotsTxt: true,
+ robotsTxtOptions: {
+ additionalSitemaps: [
+ 'https://manifold.markets/server-sitemap.xml', // <==== Add here
+ ],
+ },
// Other options: https://github.com/iamvishnusankar/next-sitemap#configuration-options
}
-
-// See https://github.com/iamvishnusankar/next-sitemap#additional-paths-function
-async function additionalPaths(config) {
- // Fetching data from https://docs.manifold.markets/api
- const response = await fetch(`${config.siteUrl}/api/v0/markets`)
-
- const liteMarkets = await response
- // See https://www.sitemaps.org/protocol.html
- return liteMarkets.map((liteMarket) => ({
- loc: liteMarket.url,
- changefreq: 'hourly',
- priority: 0.2, // Individual markets aren't that important
- // TODO: Add `lastmod` aka last modified time
- }))
-}
-
-// Polyfill for fetch: get the JSON contents of a URL
-async function fetch(url) {
- return new Promise((resolve, reject) => {
- https.get(url, (res) => {
- let data = ''
- res.on('data', (chunk) => {
- data += chunk
- })
- res.on('end', () => {
- resolve(JSON.parse(data))
- })
- })
- })
-}
diff --git a/web/pages/server-sitemap.xml.tsx b/web/pages/server-sitemap.xml.tsx
new file mode 100644
index 00000000..8625860c
--- /dev/null
+++ b/web/pages/server-sitemap.xml.tsx
@@ -0,0 +1,21 @@
+import { GetServerSideProps } from 'next'
+import { getServerSideSitemap } from 'next-sitemap'
+import { DOMAIN } from '../../common/envs/constants'
+
+export const getServerSideProps: GetServerSideProps = async (ctx) => {
+ // Fetching data from https://docs.manifold.markets/api
+ const response = await fetch(`https://${DOMAIN}/api/v0/markets`)
+
+ const liteMarkets = await response.json()
+ const fields = liteMarkets.map((liteMarket: any) => ({
+ // See https://www.sitemaps.org/protocol.html
+ loc: liteMarket.url,
+ changefreq: 'hourly',
+ priority: 0.2, // Individual markets aren't that important
+ // TODO: Add `lastmod` aka last modified time
+ }))
+ return getServerSideSitemap(ctx, fields)
+}
+
+// Default export to prevent next.js errors
+export default function Sitemap() {}
diff --git a/web/public/robots.txt b/web/public/robots.txt
new file mode 100644
index 00000000..014904fd
--- /dev/null
+++ b/web/public/robots.txt
@@ -0,0 +1,10 @@
+# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://manifold.markets
+
+# Sitemaps
+Sitemap: https://manifold.markets/sitemap.xml
+Sitemap: https://manifold.markets/server-sitemap.xml
diff --git a/web/public/sitemap-0.xml b/web/public/sitemap-0.xml
new file mode 100644
index 00000000..9db55606
--- /dev/null
+++ b/web/public/sitemap-0.xml
@@ -0,0 +1,19 @@
+
+
+https://manifold.marketshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/abouthourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/accounthourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/add-fundshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/analyticshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/createhourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/embed/analyticshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/foldshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/homehourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/landing-pagehourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/leaderboardshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/make-predictionshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/marketshourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/profilehourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/simulatorhourly0.72022-03-24T16:51:19.526Z
+https://manifold.markets/tradeshourly0.72022-03-24T16:51:19.526Z
+
\ No newline at end of file
diff --git a/web/public/sitemap.xml b/web/public/sitemap.xml
new file mode 100644
index 00000000..050639f2
--- /dev/null
+++ b/web/public/sitemap.xml
@@ -0,0 +1,4 @@
+
+
+https://manifold.markets/sitemap-0.xml
+
\ No newline at end of file