From 7144e57c93a6254c656d8b5574d685517ec07539 Mon Sep 17 00:00:00 2001 From: Marshall Polaris Date: Wed, 14 Sep 2022 01:33:59 -0700 Subject: [PATCH] Denormalize user display fields onto bets (#853) * Denormalize user display fields onto bets * Make bet denormalization script fast enough to run it on prod * Make `placeBet`/`sellShares` immediately post denormalized info --- common/antes.ts | 16 ++++--- common/bet.ts | 6 +++ common/new-bet.ts | 7 ++- common/sell-bet.ts | 5 +- functions/src/change-user-info.ts | 12 +++++ functions/src/on-create-bet.ts | 6 +++ functions/src/place-bet.ts | 9 +++- .../src/scripts/denormalize-avatar-urls.ts | 48 +++++++------------ .../src/scripts/denormalize-bet-user-data.ts | 38 +++++++++++++++ .../scripts/denormalize-comment-bet-data.ts | 30 ++++++------ .../denormalize-comment-contract-data.ts | 24 ++++------ functions/src/scripts/denormalize.ts | 45 +++++++++++------ functions/src/sell-shares.ts | 3 ++ .../contract/contract-leaderboard.tsx | 5 +- web/components/feed/feed-bets.tsx | 34 +++++-------- web/components/limit-bets.tsx | 8 ++-- 16 files changed, 180 insertions(+), 116 deletions(-) create mode 100644 functions/src/scripts/denormalize-bet-user-data.ts diff --git a/common/antes.ts b/common/antes.ts index ba7c95e8..51aac20f 100644 --- a/common/antes.ts +++ b/common/antes.ts @@ -15,9 +15,13 @@ import { Answer } from './answer' export const HOUSE_LIQUIDITY_PROVIDER_ID = 'IPTOzEqrpkWmEzh6hwvAyY9PqFb2' // @ManifoldMarkets' id export const DEV_HOUSE_LIQUIDITY_PROVIDER_ID = '94YYTk1AFWfbWMpfYcvnnwI1veP2' // @ManifoldMarkets' id - export const UNIQUE_BETTOR_LIQUIDITY_AMOUNT = 20 +type NormalizedBet = Omit< + T, + 'userAvatarUrl' | 'userName' | 'userUsername' +> + export function getCpmmInitialLiquidity( providerId: string, contract: CPMMBinaryContract, @@ -53,7 +57,7 @@ export function getAnteBets( const { createdTime } = contract - const yesBet: Bet = { + const yesBet: NormalizedBet = { id: yesAnteId, userId: creator.id, contractId: contract.id, @@ -67,7 +71,7 @@ export function getAnteBets( fees: noFees, } - const noBet: Bet = { + const noBet: NormalizedBet = { id: noAnteId, userId: creator.id, contractId: contract.id, @@ -95,7 +99,7 @@ export function getFreeAnswerAnte( const { createdTime } = contract - const anteBet: Bet = { + const anteBet: NormalizedBet = { id: anteBetId, userId: anteBettorId, contractId: contract.id, @@ -125,7 +129,7 @@ export function getMultipleChoiceAntes( const { createdTime } = contract - const bets: Bet[] = answers.map((answer, i) => ({ + const bets: NormalizedBet[] = answers.map((answer, i) => ({ id: betDocIds[i], userId: creator.id, contractId: contract.id, @@ -175,7 +179,7 @@ export function getNumericAnte( range(0, bucketCount).map((_, i) => [i, betAnte]) ) - const anteBet: NumericBet = { + const anteBet: NormalizedBet = { id: newBetId, userId: anteBettorId, contractId: contract.id, diff --git a/common/bet.ts b/common/bet.ts index 8afebcd8..ee869bb5 100644 --- a/common/bet.ts +++ b/common/bet.ts @@ -3,6 +3,12 @@ import { Fees } from './fees' export type Bet = { id: string userId: string + + // denormalized for bet lists + userAvatarUrl?: string + userUsername: string + userName: string + contractId: string createdTime: number diff --git a/common/new-bet.ts b/common/new-bet.ts index 7085a4fe..91faf640 100644 --- a/common/new-bet.ts +++ b/common/new-bet.ts @@ -31,7 +31,10 @@ import { floatingLesserEqual, } from './util/math' -export type CandidateBet = Omit +export type CandidateBet = Omit< + T, + 'id' | 'userId' | 'userAvatarUrl' | 'userName' | 'userUsername' +> export type BetInfo = { newBet: CandidateBet newPool?: { [outcome: string]: number } @@ -322,7 +325,7 @@ export const getNewBinaryDpmBetInfo = ( export const getNewMultiBetInfo = ( outcome: string, amount: number, - contract: FreeResponseContract | MultipleChoiceContract, + contract: FreeResponseContract | MultipleChoiceContract ) => { const { pool, totalShares, totalBets } = contract diff --git a/common/sell-bet.ts b/common/sell-bet.ts index bc8fe596..96636ca0 100644 --- a/common/sell-bet.ts +++ b/common/sell-bet.ts @@ -9,7 +9,10 @@ import { CPMMContract, DPMContract } from './contract' import { DPM_CREATOR_FEE, DPM_PLATFORM_FEE, Fees } from './fees' import { sumBy } from 'lodash' -export type CandidateBet = Omit +export type CandidateBet = Omit< + T, + 'id' | 'userId' | 'userAvatarUrl' | 'userName' | 'userUsername' +> export const getSellBetInfo = (bet: Bet, contract: DPMContract) => { const { pool, totalShares, totalBets } = contract diff --git a/functions/src/change-user-info.ts b/functions/src/change-user-info.ts index ca66f1ba..53908741 100644 --- a/functions/src/change-user-info.ts +++ b/functions/src/change-user-info.ts @@ -2,6 +2,7 @@ import * as admin from 'firebase-admin' import { z } from 'zod' import { getUser } from './utils' +import { Bet } from '../../common/bet' import { Contract } from '../../common/contract' import { Comment } from '../../common/comment' import { User } from '../../common/user' @@ -68,10 +69,21 @@ export const changeUser = async ( .get() const answerUpdate: Partial = removeUndefinedProps(update) + const betsSnap = await firestore + .collectionGroup('bets') + .where('userId', '==', user.id) + .get() + const betsUpdate: Partial = removeUndefinedProps({ + userName: update.name, + userUsername: update.username, + userAvatarUrl: update.avatarUrl, + }) + const bulkWriter = firestore.bulkWriter() commentSnap.docs.forEach((d) => bulkWriter.update(d.ref, commentUpdate)) answerSnap.docs.forEach((d) => bulkWriter.update(d.ref, answerUpdate)) contracts.docs.forEach((d) => bulkWriter.update(d.ref, contractUpdate)) + betsSnap.docs.forEach((d) => bulkWriter.update(d.ref, betsUpdate)) await bulkWriter.flush() console.log('Done writing!') diff --git a/functions/src/on-create-bet.ts b/functions/src/on-create-bet.ts index 6b5f7eac..f54d6475 100644 --- a/functions/src/on-create-bet.ts +++ b/functions/src/on-create-bet.ts @@ -61,6 +61,12 @@ export const onCreateBet = functions const bettor = await getUser(bet.userId) if (!bettor) return + await change.ref.update({ + userAvatarUrl: bettor.avatarUrl, + userName: bettor.name, + userUsername: bettor.username, + }) + await updateUniqueBettorsAndGiveCreatorBonus(contract, eventId, bettor) await notifyFills(bet, contract, eventId, bettor) await updateBettingStreak(bettor, bet, contract, eventId) diff --git a/functions/src/place-bet.ts b/functions/src/place-bet.ts index d98430c1..74df7dc3 100644 --- a/functions/src/place-bet.ts +++ b/functions/src/place-bet.ts @@ -139,7 +139,14 @@ export const placebet = newEndpoint({}, async (req, auth) => { } const betDoc = contractDoc.collection('bets').doc() - trans.create(betDoc, { id: betDoc.id, userId: user.id, ...newBet }) + trans.create(betDoc, { + id: betDoc.id, + userId: user.id, + userAvatarUrl: user.avatarUrl, + userUsername: user.username, + userName: user.name, + ...newBet, + }) log('Created new bet document.') if (makers) { diff --git a/functions/src/scripts/denormalize-avatar-urls.ts b/functions/src/scripts/denormalize-avatar-urls.ts index 23b7dfc9..fd95ec8f 100644 --- a/functions/src/scripts/denormalize-avatar-urls.ts +++ b/functions/src/scripts/denormalize-avatar-urls.ts @@ -3,12 +3,7 @@ import * as admin from 'firebase-admin' import { initAdmin } from './script-init' -import { - DocumentCorrespondence, - findDiffs, - describeDiff, - applyDiff, -} from './denormalize' +import { findDiffs, describeDiff, applyDiff } from './denormalize' import { DocumentSnapshot, Transaction } from 'firebase-admin/firestore' initAdmin() @@ -79,43 +74,36 @@ if (require.main === module) { getAnswersByUserId(transaction), ]) - const usersContracts = Array.from( - usersById.entries(), - ([id, doc]): DocumentCorrespondence => { - return [doc, contractsByUserId.get(id) || []] - } - ) - const contractDiffs = findDiffs( - usersContracts, + const usersContracts = Array.from(usersById.entries(), ([id, doc]) => { + return [doc, contractsByUserId.get(id) || []] as const + }) + const contractDiffs = findDiffs(usersContracts, [ 'avatarUrl', - 'creatorAvatarUrl' - ) + 'creatorAvatarUrl', + ]) console.log(`Found ${contractDiffs.length} contracts with mismatches.`) contractDiffs.forEach((d) => { console.log(describeDiff(d)) applyDiff(transaction, d) }) - const usersComments = Array.from( - usersById.entries(), - ([id, doc]): DocumentCorrespondence => { - return [doc, commentsByUserId.get(id) || []] - } - ) - const commentDiffs = findDiffs(usersComments, 'avatarUrl', 'userAvatarUrl') + const usersComments = Array.from(usersById.entries(), ([id, doc]) => { + return [doc, commentsByUserId.get(id) || []] as const + }) + const commentDiffs = findDiffs(usersComments, [ + 'avatarUrl', + 'userAvatarUrl', + ]) console.log(`Found ${commentDiffs.length} comments with mismatches.`) commentDiffs.forEach((d) => { console.log(describeDiff(d)) applyDiff(transaction, d) }) - const usersAnswers = Array.from( - usersById.entries(), - ([id, doc]): DocumentCorrespondence => { - return [doc, answersByUserId.get(id) || []] - } - ) - const answerDiffs = findDiffs(usersAnswers, 'avatarUrl', 'avatarUrl') + const usersAnswers = Array.from(usersById.entries(), ([id, doc]) => { + return [doc, answersByUserId.get(id) || []] as const + }) + const answerDiffs = findDiffs(usersAnswers, ['avatarUrl', 'avatarUrl']) console.log(`Found ${answerDiffs.length} answers with mismatches.`) answerDiffs.forEach((d) => { console.log(describeDiff(d)) diff --git a/functions/src/scripts/denormalize-bet-user-data.ts b/functions/src/scripts/denormalize-bet-user-data.ts new file mode 100644 index 00000000..3c86e140 --- /dev/null +++ b/functions/src/scripts/denormalize-bet-user-data.ts @@ -0,0 +1,38 @@ +// Filling in the user-based fields on bets. + +import * as admin from 'firebase-admin' +import { initAdmin } from './script-init' +import { findDiffs, describeDiff, getDiffUpdate } from './denormalize' +import { log, writeAsync } from '../utils' + +initAdmin() +const firestore = admin.firestore() + +// not in a transaction for speed -- may need to be run more than once +async function denormalize() { + const users = await firestore.collection('users').get() + log(`Found ${users.size} users.`) + for (const userDoc of users.docs) { + const userBets = await firestore + .collectionGroup('bets') + .where('userId', '==', userDoc.id) + .get() + const mapping = [[userDoc, userBets.docs] as const] as const + const diffs = findDiffs( + mapping, + ['avatarUrl', 'userAvatarUrl'], + ['name', 'userName'], + ['username', 'userUsername'] + ) + log(`Found ${diffs.length} bets with mismatched user data.`) + const updates = diffs.map((d) => { + log(describeDiff(d)) + return getDiffUpdate(d) + }) + await writeAsync(firestore, updates) + } +} + +if (require.main === module) { + denormalize().catch((e) => console.error(e)) +} diff --git a/functions/src/scripts/denormalize-comment-bet-data.ts b/functions/src/scripts/denormalize-comment-bet-data.ts index 929626c3..a5fb8759 100644 --- a/functions/src/scripts/denormalize-comment-bet-data.ts +++ b/functions/src/scripts/denormalize-comment-bet-data.ts @@ -3,12 +3,7 @@ import * as admin from 'firebase-admin' import { zip } from 'lodash' import { initAdmin } from './script-init' -import { - DocumentCorrespondence, - findDiffs, - describeDiff, - applyDiff, -} from './denormalize' +import { findDiffs, describeDiff, applyDiff } from './denormalize' import { log } from '../utils' import { Transaction } from 'firebase-admin/firestore' @@ -41,17 +36,20 @@ async function denormalize() { ) ) log(`Found ${bets.length} bets associated with comments.`) - const mapping = zip(bets, betComments) - .map(([bet, comment]): DocumentCorrespondence => { - return [bet!, [comment!]] // eslint-disable-line - }) - .filter(([bet, _]) => bet.exists) // dev DB has some invalid bet IDs - const amountDiffs = findDiffs(mapping, 'amount', 'betAmount') - const outcomeDiffs = findDiffs(mapping, 'outcome', 'betOutcome') - log(`Found ${amountDiffs.length} comments with mismatched amounts.`) - log(`Found ${outcomeDiffs.length} comments with mismatched outcomes.`) - const diffs = amountDiffs.concat(outcomeDiffs) + // dev DB has some invalid bet IDs + const mapping = zip(bets, betComments) + .filter(([bet, _]) => bet!.exists) // eslint-disable-line + .map(([bet, comment]) => { + return [bet!, [comment!]] as const // eslint-disable-line + }) + + const diffs = findDiffs( + mapping, + ['amount', 'betAmount'], + ['outcome', 'betOutcome'] + ) + log(`Found ${diffs.length} comments with mismatched data.`) diffs.slice(0, 500).forEach((d) => { log(describeDiff(d)) applyDiff(trans, d) diff --git a/functions/src/scripts/denormalize-comment-contract-data.ts b/functions/src/scripts/denormalize-comment-contract-data.ts index 0358c5a1..150b833d 100644 --- a/functions/src/scripts/denormalize-comment-contract-data.ts +++ b/functions/src/scripts/denormalize-comment-contract-data.ts @@ -2,12 +2,7 @@ import * as admin from 'firebase-admin' import { initAdmin } from './script-init' -import { - DocumentCorrespondence, - findDiffs, - describeDiff, - applyDiff, -} from './denormalize' +import { findDiffs, describeDiff, applyDiff } from './denormalize' import { DocumentSnapshot, Transaction } from 'firebase-admin/firestore' initAdmin() @@ -43,16 +38,15 @@ async function denormalize() { getContractsById(transaction), getCommentsByContractId(transaction), ]) - const mapping = Object.entries(contractsById).map( - ([id, doc]): DocumentCorrespondence => { - return [doc, commentsByContractId.get(id) || []] - } + const mapping = Object.entries(contractsById).map(([id, doc]) => { + return [doc, commentsByContractId.get(id) || []] as const + }) + const diffs = findDiffs( + mapping, + ['slug', 'contractSlug'], + ['question', 'contractQuestion'] ) - const slugDiffs = findDiffs(mapping, 'slug', 'contractSlug') - const qDiffs = findDiffs(mapping, 'question', 'contractQuestion') - console.log(`Found ${slugDiffs.length} comments with mismatched slugs.`) - console.log(`Found ${qDiffs.length} comments with mismatched questions.`) - const diffs = slugDiffs.concat(qDiffs) + console.log(`Found ${diffs.length} comments with mismatched data.`) diffs.slice(0, 500).forEach((d) => { console.log(describeDiff(d)) applyDiff(transaction, d) diff --git a/functions/src/scripts/denormalize.ts b/functions/src/scripts/denormalize.ts index 20bfc458..d4feb425 100644 --- a/functions/src/scripts/denormalize.ts +++ b/functions/src/scripts/denormalize.ts @@ -2,32 +2,40 @@ // another set of documents. import { DocumentSnapshot, Transaction } from 'firebase-admin/firestore' +import { isEqual, zip } from 'lodash' +import { UpdateSpec } from '../utils' export type DocumentValue = { doc: DocumentSnapshot - field: string - val: unknown + fields: string[] + vals: unknown[] } -export type DocumentCorrespondence = [DocumentSnapshot, DocumentSnapshot[]] +export type DocumentMapping = readonly [ + DocumentSnapshot, + readonly DocumentSnapshot[] +] export type DocumentDiff = { src: DocumentValue dest: DocumentValue } +type PathPair = readonly [string, string] + export function findDiffs( - docs: DocumentCorrespondence[], - srcPath: string, - destPath: string + docs: readonly DocumentMapping[], + ...paths: PathPair[] ) { const diffs: DocumentDiff[] = [] + const srcPaths = paths.map((p) => p[0]) + const destPaths = paths.map((p) => p[1]) for (const [srcDoc, destDocs] of docs) { - const srcVal = srcDoc.get(srcPath) + const srcVals = srcPaths.map((p) => srcDoc.get(p)) for (const destDoc of destDocs) { - const destVal = destDoc.get(destPath) - if (destVal !== srcVal) { + const destVals = destPaths.map((p) => destDoc.get(p)) + if (!isEqual(srcVals, destVals)) { diffs.push({ - src: { doc: srcDoc, field: srcPath, val: srcVal }, - dest: { doc: destDoc, field: destPath, val: destVal }, + src: { doc: srcDoc, fields: srcPaths, vals: srcVals }, + dest: { doc: destDoc, fields: destPaths, vals: destVals }, }) } } @@ -37,12 +45,19 @@ export function findDiffs( export function describeDiff(diff: DocumentDiff) { function describeDocVal(x: DocumentValue): string { - return `${x.doc.ref.path}.${x.field}: ${x.val}` + return `${x.doc.ref.path}.[${x.fields.join('|')}]: [${x.vals.join('|')}]` } return `${describeDocVal(diff.src)} -> ${describeDocVal(diff.dest)}` } -export function applyDiff(transaction: Transaction, diff: DocumentDiff) { - const { src, dest } = diff - transaction.update(dest.doc.ref, dest.field, src.val) +export function getDiffUpdate(diff: DocumentDiff) { + return { + doc: diff.dest.doc.ref, + fields: Object.fromEntries(zip(diff.dest.fields, diff.src.vals)), + } as UpdateSpec +} + +export function applyDiff(transaction: Transaction, diff: DocumentDiff) { + const update = getDiffUpdate(diff) + transaction.update(update.doc, update.fields) } diff --git a/functions/src/sell-shares.ts b/functions/src/sell-shares.ts index 0e88a0b5..f2f475cb 100644 --- a/functions/src/sell-shares.ts +++ b/functions/src/sell-shares.ts @@ -112,6 +112,9 @@ export const sellshares = newEndpoint({}, async (req, auth) => { transaction.create(newBetDoc, { id: newBetDoc.id, userId: user.id, + userAvatarUrl: user.avatarUrl, + userUsername: user.username, + userName: user.name, ...newBet, }) transaction.update( diff --git a/web/components/contract/contract-leaderboard.tsx b/web/components/contract/contract-leaderboard.tsx index 1eaf7043..54b2c79e 100644 --- a/web/components/contract/contract-leaderboard.tsx +++ b/web/components/contract/contract-leaderboard.tsx @@ -6,7 +6,6 @@ import { formatMoney } from 'common/util/format' import { groupBy, mapValues, sumBy, sortBy, keyBy } from 'lodash' import { useState, useMemo, useEffect } from 'react' import { CommentTipMap } from 'web/hooks/use-tip-txns' -import { useUserById } from 'web/hooks/use-user' import { listUsers, User } from 'web/lib/firebase/users' import { FeedBet } from '../feed/feed-bets' import { FeedComment } from '../feed/feed-comments' @@ -88,7 +87,7 @@ export function ContractTopTrades(props: { // Now find the betId with the highest profit const topBetId = sortBy(bets, (b) => -profitById[b.id])[0]?.id - const topBettor = useUserById(betsById[topBetId]?.userId) + const topBettor = betsById[topBetId]?.userName // And also the commentId of the comment with the highest profit const topCommentId = sortBy( @@ -121,7 +120,7 @@ export function ContractTopTrades(props: {
- {topBettor?.name} made {formatMoney(profitById[topBetId] || 0)}! + {topBettor} made {formatMoney(profitById[topBetId] || 0)}!
)} diff --git a/web/components/feed/feed-bets.tsx b/web/components/feed/feed-bets.tsx index cf444061..def97801 100644 --- a/web/components/feed/feed-bets.tsx +++ b/web/components/feed/feed-bets.tsx @@ -1,8 +1,7 @@ import dayjs from 'dayjs' import { Contract } from 'common/contract' import { Bet } from 'common/bet' -import { User } from 'common/user' -import { useUser, useUserById } from 'web/hooks/use-user' +import { useUser } from 'web/hooks/use-user' import { Row } from 'web/components/layout/row' import { Avatar, EmptyAvatar } from 'web/components/avatar' import clsx from 'clsx' @@ -18,29 +17,20 @@ import { UserLink } from 'web/components/user-link' export function FeedBet(props: { contract: Contract; bet: Bet }) { const { contract, bet } = props - const { userId, createdTime } = bet - - const isBeforeJune2022 = dayjs(createdTime).isBefore('2022-06-01') - // eslint-disable-next-line react-hooks/rules-of-hooks - const bettor = isBeforeJune2022 ? undefined : useUserById(userId) - - const user = useUser() - const isSelf = user?.id === userId + const { userAvatarUrl, userUsername, createdTime } = bet + const showUser = dayjs(createdTime).isAfter('2022-06-01') return ( - {isSelf ? ( - - ) : bettor ? ( - + {showUser ? ( + ) : ( )} @@ -50,13 +40,13 @@ export function FeedBet(props: { contract: Contract; bet: Bet }) { export function BetStatusText(props: { contract: Contract bet: Bet - isSelf: boolean - bettor?: User + hideUser?: boolean hideOutcome?: boolean className?: string }) { - const { bet, contract, bettor, isSelf, hideOutcome, className } = props + const { bet, contract, hideUser, hideOutcome, className } = props const { outcomeType } = contract + const self = useUser() const isPseudoNumeric = outcomeType === 'PSEUDO_NUMERIC' const isFreeResponse = outcomeType === 'FREE_RESPONSE' const { amount, outcome, createdTime, challengeSlug } = bet @@ -101,10 +91,10 @@ export function BetStatusText(props: { return (
- {bettor ? ( - + {!hideUser ? ( + ) : ( - {isSelf ? 'You' : 'A trader'} + {self?.id === bet.userId ? 'You' : 'A trader'} )}{' '} {bought} {money} {outOfTotalAmount} diff --git a/web/components/limit-bets.tsx b/web/components/limit-bets.tsx index 466b7a9b..606bc7e0 100644 --- a/web/components/limit-bets.tsx +++ b/web/components/limit-bets.tsx @@ -4,7 +4,7 @@ import { getFormattedMappedValue } from 'common/pseudo-numeric' import { formatMoney, formatPercent } from 'common/util/format' import { sortBy } from 'lodash' import { useState } from 'react' -import { useUser, useUserById } from 'web/hooks/use-user' +import { useUser } from 'web/hooks/use-user' import { cancelBet } from 'web/lib/firebase/api' import { Avatar } from './avatar' import { Button } from './button' @@ -109,16 +109,14 @@ function LimitBet(props: { setIsCancelling(true) } - const user = useUserById(bet.userId) - return ( {!isYou && ( )}