diff --git a/functions/src/scripts/update-feed.ts b/functions/src/scripts/update-feed.ts index 3a30f85a..3a83a59a 100644 --- a/functions/src/scripts/update-feed.ts +++ b/functions/src/scripts/update-feed.ts @@ -9,10 +9,7 @@ import { User } from '../../../common/user' import { batchedWaitAll } from '../../../common/util/promise' import { Contract } from '../../../common/contract' import { updateWordScores } from '../update-recommendations' -import { - getFeedContracts, - updateFeed as updateUserFeed, -} from '../update-user-feed' +import { getFeedContracts, doUserFeedUpdate } from '../update-user-feed' const firestore = admin.firestore() @@ -31,7 +28,7 @@ async function updateFeed() { console.log('Updating recs for', user.username) await updateWordScores(user, contracts) console.log('Updating feed for', user.username) - await updateUserFeed(user, feedContracts) + await doUserFeedUpdate(user, feedContracts) }) ) } diff --git a/functions/src/update-recommendations.ts b/functions/src/update-recommendations.ts index 392f6393..e7fd9a2a 100644 --- a/functions/src/update-recommendations.ts +++ b/functions/src/update-recommendations.ts @@ -9,22 +9,40 @@ import { User } from '../../common/user' import { ClickEvent } from '../../common/tracking' import { getWordScores } from '../../common/recommended-contracts' import { batchedWaitAll } from '../../common/util/promise' +import { callCloudFunction } from './call-cloud-function' const firestore = admin.firestore() export const updateRecommendations = functions.pubsub .schedule('every 24 hours') .onRun(async () => { + const users = await getValues(firestore.collection('users')) + + const batchSize = 100 + const userBatches: User[][] = [] + for (let i = 0; i < users.length; i += batchSize) { + userBatches.push(users.slice(i, i + batchSize)) + } + + for (const batch of userBatches) { + await new Promise((resolve) => setTimeout(resolve, 100)) + callCloudFunction('updateRecommendationsBatch', { users: batch }) + } + }) + +export const updateRecommendationsBatch = functions.https.onCall( + async (data: { users: User[] }) => { + const { users } = data + const contracts = await getValues( firestore.collection('contracts') ) - const users = await getValues(firestore.collection('users')) - await batchedWaitAll( users.map((user) => () => updateWordScores(user, contracts)) ) - }) + } +) export const updateWordScores = async (user: User, contracts: Contract[]) => { const [bets, viewCounts, clicks] = await Promise.all([ diff --git a/functions/src/update-user-feed.ts b/functions/src/update-user-feed.ts index 12c82c7b..80dc292d 100644 --- a/functions/src/update-user-feed.ts +++ b/functions/src/update-user-feed.ts @@ -14,22 +14,33 @@ import { import { Bet } from '../../common/bet' import { Comment } from '../../common/comment' import { User } from '../../common/user' -import { batchedWaitAll } from '../../common/util/promise' import { getContractScore } from '../../common/recommended-contracts' +import { callCloudFunction } from './call-cloud-function' const firestore = admin.firestore() const MAX_FEED_CONTRACTS = 75 -export const updateUserFeed = functions.pubsub +export const updateFeed = functions.pubsub .schedule('every 60 minutes') .onRun(async () => { const contracts = await getFeedContracts() const users = await getValues(firestore.collection('users')) - await batchedWaitAll(users.map((user) => () => updateFeed(user, contracts))) + for (const user of users) { + await new Promise((resolve) => setTimeout(resolve, 10)) + callCloudFunction('updateUserFeed', { user, contracts }) + } }) +export const updateUserFeed = functions.https.onCall( + async (data: { contracts: Contract[]; user: User }) => { + const { user, contracts } = data + + await doUserFeedUpdate(user, contracts) + } +) + export async function getFeedContracts() { // Get contracts bet on or created in last week. const contracts = await Promise.all([ @@ -56,7 +67,7 @@ export async function getFeedContracts() { return contracts } -export const updateFeed = async (user: User, contracts: Contract[]) => { +export const doUserFeedUpdate = async (user: User, contracts: Contract[]) => { const userCacheCollection = firestore.collection( `private-users/${user.id}/cache` )