open.source/manifold
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change feed update strategy to update a fixed subset of users every 5 minutes

Browse Source
This commit is contained in:
James Grugett 2022-05-15 15:47:29 -04:00
parent ab8d541f8d
commit 75a1dfd0e4
1 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
functions/src/update-feed.ts
View File

@ -26,18 +26,26 @@ import { CATEGORY_LIST } from '../../common/categories'
const firestore = admin.firestore()
export const updateFeed = functions.pubsub
.schedule('every 60 minutes')
.onRun(async () => {
const users = await getValues<User>(firestore.collection('users'))
const BATCH_SIZE = 30
const MAX_BATCHES = 50
const batchSize = 100
const getUserBatches = async () => {
const users = _.shuffle(await getValues<User>(firestore.collection('users')))
let userBatches: User[][] = []
for (let i = 0; i < users.length; i += batchSize) {
userBatches.push(users.slice(i, i + batchSize))
for (let i = 0; i < users.length; i += BATCH_SIZE) {
userBatches.push(users.slice(i, i + BATCH_SIZE))
}
userBatches = userBatches.slice(0, MAX_BATCHES)
console.log('updating feed batch')
console.log('updating feed batches', userBatches.length, 'of', MAX_BATCHES)
return userBatches
}
export const updateFeed = functions.pubsub
.schedule('every 5 minutes')
.onRun(async () => {
const userBatches = await getUserBatches()
await Promise.all(
userBatches.map((users) =>
@ -72,13 +80,7 @@ export const updateFeedBatch = functions.https.onCall(
export const updateCategoryFeed = functions.https.onCall(
async (data: { category: string }) => {
const { category } = data
const users = await getValues<User>(firestore.collection('users'))
const batchSize = 100
const userBatches: User[][] = []
for (let i = 0; i < users.length; i += batchSize) {
userBatches.push(users.slice(i, i + batchSize))
}
const userBatches = await getUserBatches()
await Promise.all(
userBatches.map(async (users) => {