Add paging to /markets API endpoint (#468)

* Add really simple paging to markets endpoint

* Document changes to markets endpoint

* n -> limit

docs/docs/api.md

@@ -24,8 +24,9 @@ APIs that require authentication accept an `Authorization` header in one of two
   identity. This is what our web client uses. It will probably be annoying for
   you to generate and we will not document it further here.
 
-API requests that accept parameters should have a body with a JSON object with
+API requests that accept parameters should either have the parameters in the
-one property per parameter.
+query string if they are GET requests, or have a body with a JSON object with
+one property per parameter if they are POST requests.
 
 API responses should always either have a body with a JSON result object (if
 the response was a 200) or with a JSON object representing an error (if the
@@ -35,13 +36,21 @@ response was a 4xx or 5xx.)
 
 ### `GET /v0/markets`
 
-Lists all markets.
+Lists all markets, ordered by creation date descending.
+
+Parameters:
+
+- `limit`: Optional. How many markets to return. The maximum and the default is 1000.
+- `before`: Optional. The ID of the market before which the list will start. For
+  example, if you ask for the most recent 10 markets, and then perform a second
+  query for 10 more markets with `before=[the id of the 10th market]`, you will
+  get markets 11 through 20.
 
 Requires no authorization.
 
 - Example request
   ```
-  http://manifold.markets/api/v0/markets
+  http://manifold.markets/api/v0/markets?limit=1
   ```
 - Example response
   ```json
@@ -445,6 +454,7 @@ $ curl https://manifold.markets/api/v0/market -X POST -H 'Content-Type: applicat
 
 ## Changelog
 
+- 2022-06-08: Add paging to markets endpoint
 - 2022-06-05: Add new authorized write endpoints
 - 2022-02-28: Add `resolutionTime` to markets, change `closeTime` definition
 - 2022-02-19: Removed user IDs from bets

web/lib/firebase/contracts.ts

@@ -12,6 +12,7 @@ import {
   getDoc,
   updateDoc,
   limit,
+  startAfter,
 } from 'firebase/firestore'
 import { range, sortBy, sum } from 'lodash'
 
@@ -145,8 +146,15 @@ export async function listTaggedContractsCaseInsensitive(
   return snapshot.docs.map((doc) => doc.data() as Contract)
 }
 
-export async function listAllContracts(): Promise<Contract[]> {
+export async function listAllContracts(
-  const q = query(contractCollection, orderBy('createdTime', 'desc'))
+  n: number,
+  before?: string
+): Promise<Contract[]> {
+  let q = query(contractCollection, orderBy('createdTime', 'desc'), limit(n))
+  if (before != null) {
+    const snap = await getDoc(doc(db, 'contracts', before))
+    q = query(q, startAfter(snap))
+  }
   const snapshot = await getDocs(q)
   return snapshot.docs.map((doc) => doc.data() as Contract)
 }

web/pages/api/v0/markets.ts

@@ -4,15 +4,46 @@ import { listAllContracts } from 'web/lib/firebase/contracts'
 import { applyCorsHeaders, CORS_UNRESTRICTED } from 'web/lib/api/cors'
 import { toLiteMarket } from './_types'
 
-type Data = any[]
-
 export default async function handler(
   req: NextApiRequest,
-  res: NextApiResponse<Data>
+  res: NextApiResponse
 ) {
   await applyCorsHeaders(req, res, CORS_UNRESTRICTED)
-  const contracts = await listAllContracts()
+  let before: string | undefined
+  let limit: number | undefined
+  if (req.query.before != null) {
+    if (typeof req.query.before !== 'string') {
+      res.status(400).json({ error: 'before must be null or a market ID.' })
+      return
+    }
+    before = req.query.before
+  }
+  if (req.query.limit != null) {
+    if (typeof req.query.limit !== 'string') {
+      res
+        .status(400)
+        .json({ error: 'limit must be null or a number of markets to return.' })
+      return
+    }
+    limit = parseInt(req.query.limit)
+  } else {
+    limit = 1000
+  }
+  if (limit < 1 || limit > 1000) {
+    res.status(400).json({ error: 'limit must be between 1 and 1000.' })
+    return
+  }
+
+  try {
+    const contracts = await listAllContracts(limit, before)
     // Serve from Vercel cache, then update. see https://vercel.com/docs/concepts/functions/edge-caching
     res.setHeader('Cache-Control', 's-maxage=1, stale-while-revalidate')
     res.status(200).json(contracts.map(toLiteMarket))
+  } catch (e) {
+    res.status(400).json({
+      error:
+        'Failed to fetch markets (did you pass an invalid ID as the before parameter?)',
+    })
+    return
+  }
 }