Make bet denormalization script fast enough to run it on prod
This commit is contained in:
parent
34bf8d771f
commit
298bc04517
|
@ -1,61 +1,35 @@
|
||||||
// Filling in the contract-based fields on comments.
|
// Filling in the user-based fields on bets.
|
||||||
|
|
||||||
import * as admin from 'firebase-admin'
|
import * as admin from 'firebase-admin'
|
||||||
import { initAdmin } from './script-init'
|
import { initAdmin } from './script-init'
|
||||||
import { findDiffs, describeDiff, applyDiff } from './denormalize'
|
import { findDiffs, describeDiff, getDiffUpdate } from './denormalize'
|
||||||
import { DocumentSnapshot, Transaction } from 'firebase-admin/firestore'
|
import { log, writeAsync } from '../utils'
|
||||||
import { log } from '../utils'
|
|
||||||
|
|
||||||
initAdmin()
|
initAdmin()
|
||||||
const firestore = admin.firestore()
|
const firestore = admin.firestore()
|
||||||
|
|
||||||
async function getUsersById(transaction: Transaction) {
|
// not in a transaction for speed -- may need to be run more than once
|
||||||
const users = await transaction.get(firestore.collection('users'))
|
|
||||||
const results = Object.fromEntries(users.docs.map((doc) => [doc.id, doc]))
|
|
||||||
log(`Found ${users.size} users.`)
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getBetsByUserId(transaction: Transaction) {
|
|
||||||
const bets = await transaction.get(firestore.collectionGroup('bets'))
|
|
||||||
const results = new Map<string, DocumentSnapshot[]>()
|
|
||||||
bets.forEach((doc) => {
|
|
||||||
const userId = doc.get('userId')
|
|
||||||
const userBets = results.get(userId) || []
|
|
||||||
userBets.push(doc)
|
|
||||||
results.set(userId, userBets)
|
|
||||||
})
|
|
||||||
log(`Found ${bets.size} bets from ${results.size} users.`)
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
async function denormalize() {
|
async function denormalize() {
|
||||||
let hasMore = true
|
const users = await firestore.collection('users').get()
|
||||||
while (hasMore) {
|
log(`Found ${users.size} users.`)
|
||||||
hasMore = await admin.firestore().runTransaction(async (transaction) => {
|
for (const userDoc of users.docs) {
|
||||||
const [usersById, betsByUserId] = await Promise.all([
|
const userBets = await firestore
|
||||||
getUsersById(transaction),
|
.collectionGroup('bets')
|
||||||
getBetsByUserId(transaction),
|
.where('userId', '==', userDoc.id)
|
||||||
])
|
.get()
|
||||||
const mapping = Object.entries(usersById).map(([id, doc]) => {
|
const mapping = [[userDoc, userBets.docs] as const] as const
|
||||||
return [doc, betsByUserId.get(id) || []] as const
|
const diffs = findDiffs(
|
||||||
})
|
mapping,
|
||||||
const diffs = findDiffs(
|
['avatarUrl', 'userAvatarUrl'],
|
||||||
mapping,
|
['name', 'userName'],
|
||||||
['avatarUrl', 'userAvatarUrl'],
|
['username', 'userUsername']
|
||||||
['name', 'userName'],
|
)
|
||||||
['username', 'userUsername']
|
log(`Found ${diffs.length} bets with mismatched user data.`)
|
||||||
)
|
const updates = diffs.map((d) => {
|
||||||
log(`Found ${diffs.length} bets with mismatched user data.`)
|
log(describeDiff(d))
|
||||||
diffs.slice(0, 500).forEach((d) => {
|
return getDiffUpdate(d)
|
||||||
log(describeDiff(d))
|
|
||||||
applyDiff(transaction, d)
|
|
||||||
})
|
|
||||||
if (diffs.length > 500) {
|
|
||||||
log(`Applying first 500 because of Firestore limit...`)
|
|
||||||
}
|
|
||||||
return diffs.length > 500
|
|
||||||
})
|
})
|
||||||
|
await writeAsync(firestore, updates)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
|
|
||||||
import { DocumentSnapshot, Transaction } from 'firebase-admin/firestore'
|
import { DocumentSnapshot, Transaction } from 'firebase-admin/firestore'
|
||||||
import { isEqual, zip } from 'lodash'
|
import { isEqual, zip } from 'lodash'
|
||||||
|
import { UpdateSpec } from '../utils'
|
||||||
|
|
||||||
export type DocumentValue = {
|
export type DocumentValue = {
|
||||||
doc: DocumentSnapshot
|
doc: DocumentSnapshot
|
||||||
|
@ -20,7 +21,10 @@ export type DocumentDiff = {
|
||||||
|
|
||||||
type PathPair = readonly [string, string]
|
type PathPair = readonly [string, string]
|
||||||
|
|
||||||
export function findDiffs(docs: DocumentMapping[], ...paths: PathPair[]) {
|
export function findDiffs(
|
||||||
|
docs: readonly DocumentMapping[],
|
||||||
|
...paths: PathPair[]
|
||||||
|
) {
|
||||||
const diffs: DocumentDiff[] = []
|
const diffs: DocumentDiff[] = []
|
||||||
const srcPaths = paths.map((p) => p[0])
|
const srcPaths = paths.map((p) => p[0])
|
||||||
const destPaths = paths.map((p) => p[1])
|
const destPaths = paths.map((p) => p[1])
|
||||||
|
@ -46,8 +50,14 @@ export function describeDiff(diff: DocumentDiff) {
|
||||||
return `${describeDocVal(diff.src)} -> ${describeDocVal(diff.dest)}`
|
return `${describeDocVal(diff.src)} -> ${describeDocVal(diff.dest)}`
|
||||||
}
|
}
|
||||||
|
|
||||||
export function applyDiff(transaction: Transaction, diff: DocumentDiff) {
|
export function getDiffUpdate(diff: DocumentDiff) {
|
||||||
const { src, dest } = diff
|
return {
|
||||||
const updateSpec = Object.fromEntries(zip(dest.fields, src.vals))
|
doc: diff.dest.doc.ref,
|
||||||
transaction.update(dest.doc.ref, updateSpec)
|
fields: Object.fromEntries(zip(diff.dest.fields, diff.src.vals)),
|
||||||
|
} as UpdateSpec
|
||||||
|
}
|
||||||
|
|
||||||
|
export function applyDiff(transaction: Transaction, diff: DocumentDiff) {
|
||||||
|
const update = getDiffUpdate(diff)
|
||||||
|
transaction.update(update.doc, update.fields)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user