Fix leaderboard out of memory error with batchedWaitAll instead of Promise.all

common/util/promise.ts

@@ -0,0 +1,18 @@
+export const batchedWaitAll = async <T>(
+  createPromises: (() => Promise<T>)[],
+  batchSize = 10
+) => {
+  const numBatches = Math.ceil(createPromises.length / batchSize)
+  const result: T[] = []
+  for (let batchIndex = 0; batchIndex < numBatches; batchIndex++) {
+    const from = batchIndex * batchSize
+    const to = from + batchSize
+
+    const promises = createPromises.slice(from, to).map((f) => f())
+
+    const batch = await Promise.all(promises)
+    result.push(...batch)
+  }
+
+  return result
+}

functions/src/update-contract-metrics.ts

@@ -5,6 +5,7 @@ import * as _ from 'lodash'
 import { getValues } from './utils'
 import { Contract } from '../../common/contract'
 import { Bet } from '../../common/bet'
+import { batchedWaitAll } from '../../common/util/promise'
 
 const firestore = admin.firestore()
 
@@ -17,8 +18,8 @@ export const updateContractMetrics = functions.pubsub
       firestore.collection('contracts')
     )
 
-    await Promise.all(
+    await batchedWaitAll(
-      contracts.map(async (contract) => {
+      contracts.map((contract) => async () => {
         const volume24Hours = await computeVolumeFrom(contract, oneDay)
         const volume7Days = await computeVolumeFrom(contract, oneDay * 7)
 

functions/src/update-user-metrics.ts

@@ -7,6 +7,7 @@ import { Contract } from '../../common/contract'
 import { Bet } from '../../common/bet'
 import { User } from '../../common/user'
 import { calculatePayout } from '../../common/calculate'
+import { batchedWaitAll } from '../../common/util/promise'
 
 const firestore = admin.firestore()
 
@@ -22,8 +23,8 @@ export const updateUserMetrics = functions.pubsub
       contracts.map((contract) => [contract.id, contract])
     )
 
-    await Promise.all(
+    await batchedWaitAll(
-      users.map(async (user) => {
+      users.map((user) => async () => {
         const [investmentValue, creatorVolume] = await Promise.all([
           computeInvestmentValue(user, contractsDict),
           computeTotalPool(user, contractsDict),