From a6ed8c92282dd26310ee6b7c029365065ec53fdd Mon Sep 17 00:00:00 2001 From: FRC Date: Mon, 12 Sep 2022 16:44:24 +0100 Subject: [PATCH] Fix "500 internal error" in large groups (#856) * Members to memberIds * Moved to update-metrics --- common/group.ts | 15 ++ functions/src/update-metrics.ts | 72 +++++++- web/lib/firebase/groups.ts | 5 +- web/pages/group/[...slugs]/index.tsx | 263 +++++++-------------------- 4 files changed, 145 insertions(+), 210 deletions(-) diff --git a/common/group.ts b/common/group.ts index 19f3b7b8..36654101 100644 --- a/common/group.ts +++ b/common/group.ts @@ -12,7 +12,22 @@ export type Group = { aboutPostId?: string chatDisabled?: boolean mostRecentContractAddedTime?: number + /** @deprecated - members and contracts now stored as subcollections*/ + memberIds?: string[] // Deprecated + /** @deprecated - members and contracts now stored as subcollections*/ + contractIds?: string[] // Deprecated + cachedLeaderboard?: { + topTraders: { + userId: string + score: number + }[] + topCreators: { + userId: string + score: number + }[] + } } + export const MAX_GROUP_NAME_LENGTH = 75 export const MAX_ABOUT_LENGTH = 140 export const MAX_ID_LENGTH = 60 diff --git a/functions/src/update-metrics.ts b/functions/src/update-metrics.ts index 430f3d33..273cd098 100644 --- a/functions/src/update-metrics.ts +++ b/functions/src/update-metrics.ts @@ -4,9 +4,11 @@ import { groupBy, isEmpty, keyBy, last, sortBy } from 'lodash' import { getValues, log, logMemory, writeAsync } from './utils' import { Bet } from '../../common/bet' import { Contract, CPMM } from '../../common/contract' + import { PortfolioMetrics, User } from '../../common/user' import { DAY_MS } from '../../common/util/time' import { getLoanUpdates } from '../../common/loans' +import { scoreTraders, scoreCreators } from '../../common/scoring' import { calculateCreatorVolume, calculateNewPortfolioMetrics, @@ -15,6 +17,7 @@ import { computeVolume, } from '../../common/calculate-metrics' import { getProbability } from '../../common/calculate' +import { Group } from 'common/group' const firestore = admin.firestore() @@ -24,16 +27,29 @@ export const updateMetrics = functions .onRun(updateMetricsCore) export async function updateMetricsCore() { - const [users, contracts, bets, allPortfolioHistories] = await Promise.all([ - getValues(firestore.collection('users')), - getValues(firestore.collection('contracts')), - getValues(firestore.collectionGroup('bets')), - getValues( - firestore - .collectionGroup('portfolioHistory') - .where('timestamp', '>', Date.now() - 31 * DAY_MS) // so it includes just over a month ago - ), - ]) + const [users, contracts, bets, allPortfolioHistories, groups] = + await Promise.all([ + getValues(firestore.collection('users')), + getValues(firestore.collection('contracts')), + getValues(firestore.collectionGroup('bets')), + getValues( + firestore + .collectionGroup('portfolioHistory') + .where('timestamp', '>', Date.now() - 31 * DAY_MS) // so it includes just over a month ago + ), + getValues(firestore.collection('groups')), + ]) + + const contractsByGroup = await Promise.all( + groups.map((group) => { + return getValues( + firestore + .collection('groups') + .doc(group.id) + .collection('groupContracts') + ) + }) + ) log( `Loaded ${users.length} users, ${contracts.length} contracts, and ${bets.length} bets.` ) @@ -162,4 +178,40 @@ export async function updateMetricsCore() { 'set' ) log(`Updated metrics for ${users.length} users.`) + + const groupUpdates = groups.map((group, index) => { + const groupContractIds = contractsByGroup[index] as GroupContractDoc[] + const groupContracts = groupContractIds.map( + (e) => contractsById[e.contractId] + ) + const bets = groupContracts.map((e) => { + return betsByContract[e.id] ?? [] + }) + + const creatorScores = scoreCreators(groupContracts) + const traderScores = scoreTraders(groupContracts, bets) + + const topTraderScores = topUserScores(traderScores) + const topCreatorScores = topUserScores(creatorScores) + + return { + doc: firestore.collection('groups').doc(group.id), + fields: { + cachedLeaderboard: { + topTraders: topTraderScores, + topCreators: topCreatorScores, + }, + }, + } + }) + await writeAsync(firestore, groupUpdates) } + +const topUserScores = (scores: { [userId: string]: number }) => { + const top50 = Object.entries(scores) + .sort(([, scoreA], [, scoreB]) => scoreB - scoreA) + .slice(0, 50) + return top50.map(([userId, score]) => ({ userId, score })) +} + +type GroupContractDoc = { contractId: string; createdTime: number } diff --git a/web/lib/firebase/groups.ts b/web/lib/firebase/groups.ts index 7a372d9a..f27460d9 100644 --- a/web/lib/firebase/groups.ts +++ b/web/lib/firebase/groups.ts @@ -24,7 +24,6 @@ import { Contract } from 'common/contract' import { getContractFromId, updateContract } from 'web/lib/firebase/contracts' import { db } from 'web/lib/firebase/init' import { filterDefined } from 'common/util/array' -import { getUser } from 'web/lib/firebase/users' export const groups = coll('groups') export const groupMembers = (groupId: string) => @@ -253,7 +252,7 @@ export function getGroupLinkToDisplay(contract: Contract) { return groupToDisplay } -export async function listMembers(group: Group) { +export async function listMemberIds(group: Group) { const members = await getValues(groupMembers(group.id)) - return await Promise.all(members.map((m) => m.userId).map(getUser)) + return members.map((m) => m.userId) } diff --git a/web/pages/group/[...slugs]/index.tsx b/web/pages/group/[...slugs]/index.tsx index 768e2f82..f5d68e57 100644 --- a/web/pages/group/[...slugs]/index.tsx +++ b/web/pages/group/[...slugs]/index.tsx @@ -1,28 +1,28 @@ import React, { useState } from 'react' import Link from 'next/link' import { useRouter } from 'next/router' -import { debounce, sortBy, take } from 'lodash' -import { SearchIcon } from '@heroicons/react/outline' import { toast } from 'react-hot-toast' import { Group, GROUP_CHAT_SLUG } from 'common/group' import { Page } from 'web/components/page' -import { listAllBets } from 'web/lib/firebase/bets' import { Contract, listContractsByGroupSlug } from 'web/lib/firebase/contracts' import { addContractToGroup, getGroupBySlug, groupPath, joinGroup, - listMembers, + listMemberIds, updateGroup, } from 'web/lib/firebase/groups' import { Row } from 'web/components/layout/row' import { firebaseLogin, getUser, User } from 'web/lib/firebase/users' import { Col } from 'web/components/layout/col' import { useUser } from 'web/hooks/use-user' -import { useGroup, useGroupContractIds, useMembers } from 'web/hooks/use-group' -import { scoreCreators, scoreTraders } from 'common/scoring' +import { + useGroup, + useGroupContractIds, + useMemberIds, +} from 'web/hooks/use-group' import { Leaderboard } from 'web/components/leaderboard' import { formatMoney } from 'common/util/format' import { EditGroupButton } from 'web/components/groups/edit-group-button' @@ -35,9 +35,7 @@ import { LoadingIndicator } from 'web/components/loading-indicator' import { Modal } from 'web/components/layout/modal' import { ChoicesToggleGroup } from 'web/components/choices-toggle-group' import { ContractSearch } from 'web/components/contract-search' -import { FollowList } from 'web/components/follow-list' import { JoinOrLeaveGroupButton } from 'web/components/groups/groups-button' -import { searchInAny } from 'common/util/parse' import { CopyLinkButton } from 'web/components/copy-link-button' import { ENV_CONFIG } from 'common/envs/constants' import { useSaveReferral } from 'web/hooks/use-save-referral' @@ -59,7 +57,7 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) { const { slugs } = props.params const group = await getGroupBySlug(slugs[0]) - const members = group && (await listMembers(group)) + const memberIds = group && (await listMemberIds(group)) const creatorPromise = group ? getUser(group.creatorId) : null const contracts = @@ -71,19 +69,15 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) { : 'open' const aboutPost = group && group.aboutPostId != null && (await getPost(group.aboutPostId)) - const bets = await Promise.all( - contracts.map((contract: Contract) => listAllBets(contract.id)) - ) const messages = group && (await listAllCommentsOnGroup(group.id)) - const creatorScores = scoreCreators(contracts) - const traderScores = scoreTraders(contracts, bets) - const [topCreators, topTraders] = - (members && [ - toTopUsers(creatorScores, members), - toTopUsers(traderScores, members), - ]) ?? - [] + const cachedTopTraderIds = + (group && group.cachedLeaderboard?.topTraders) ?? [] + const cachedTopCreatorIds = + (group && group.cachedLeaderboard?.topCreators) ?? [] + const topTraders = await toTopUsers(cachedTopTraderIds) + + const topCreators = await toTopUsers(cachedTopCreatorIds) const creator = await creatorPromise // Only count unresolved markets @@ -93,11 +87,9 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) { props: { contractsCount, group, - members, + memberIds, creator, - traderScores, topTraders, - creatorScores, topCreators, messages, aboutPost, @@ -107,19 +99,6 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) { revalidate: 60, // regenerate after a minute } } - -function toTopUsers(userScores: { [userId: string]: number }, users: User[]) { - const topUserPairs = take( - sortBy(Object.entries(userScores), ([_, score]) => -1 * score), - 10 - ).filter(([_, score]) => score >= 0.5) - - const topUsers = topUserPairs.map( - ([userId]) => users.filter((user) => user.id === userId)[0] - ) - return topUsers.filter((user) => user) -} - export async function getStaticPaths() { return { paths: [], fallback: 'blocking' } } @@ -134,12 +113,10 @@ const groupSubpages = [ export default function GroupPage(props: { contractsCount: number group: Group | null - members: User[] + memberIds: string[] creator: User - traderScores: { [userId: string]: number } - topTraders: User[] - creatorScores: { [userId: string]: number } - topCreators: User[] + topTraders: { user: User; score: number }[] + topCreators: { user: User; score: number }[] messages: GroupComment[] aboutPost: Post suggestedFilter: 'open' | 'all' @@ -147,24 +124,15 @@ export default function GroupPage(props: { props = usePropz(props, getStaticPropz) ?? { contractsCount: 0, group: null, - members: [], + memberIds: [], creator: null, - traderScores: {}, topTraders: [], - creatorScores: {}, topCreators: [], messages: [], suggestedFilter: 'open', } - const { - contractsCount, - creator, - traderScores, - topTraders, - creatorScores, - topCreators, - suggestedFilter, - } = props + const { contractsCount, creator, topTraders, topCreators, suggestedFilter } = + props const router = useRouter() const { slugs } = router.query as { slugs: string[] } @@ -175,7 +143,7 @@ export default function GroupPage(props: { const user = useUser() const isAdmin = useAdmin() - const members = useMembers(group?.id) ?? props.members + const memberIds = useMemberIds(group?.id ?? null) ?? props.memberIds useSaveReferral(user, { defaultReferrerUsername: creator.username, @@ -186,18 +154,25 @@ export default function GroupPage(props: { return } const isCreator = user && group && user.id === group.creatorId - const isMember = user && members.map((m) => m.id).includes(user.id) + const isMember = user && memberIds.includes(user.id) + const maxLeaderboardSize = 50 const leaderboard = ( - +
+ + +
) @@ -216,7 +191,7 @@ export default function GroupPage(props: { creator={creator} isCreator={!!isCreator} user={user} - members={members} + memberIds={memberIds} /> ) @@ -312,9 +287,9 @@ function GroupOverview(props: { creator: User user: User | null | undefined isCreator: boolean - members: User[] + memberIds: string[] }) { - const { group, creator, isCreator, user, members } = props + const { group, creator, isCreator, user, memberIds } = props const anyoneCanJoinChoices: { [key: string]: string } = { Closed: 'false', Open: 'true', @@ -333,7 +308,7 @@ function GroupOverview(props: { const shareUrl = `https://${ENV_CONFIG.domain}${groupPath( group.slug )}${postFix}` - const isMember = user ? members.map((m) => m.id).includes(user.id) : false + const isMember = user ? memberIds.includes(user.id) : false return ( <> @@ -399,155 +374,37 @@ function GroupOverview(props: { /> )} - - -
Members
- - ) } -function SearchBar(props: { setQuery: (query: string) => void }) { - const { setQuery } = props - const debouncedQuery = debounce(setQuery, 50) - return ( -
- - debouncedQuery(e.target.value)} - placeholder="Find a member" - className="input input-bordered mb-4 w-full pl-12" - /> -
- ) -} - -function GroupMemberSearch(props: { members: User[]; group: Group }) { - const [query, setQuery] = useState('') - const { group } = props - let { members } = props - - // Use static members on load, but also listen to member changes: - const listenToMembers = useMembers(group.id) - if (listenToMembers) { - members = listenToMembers - } - - // TODO use find-active-contracts to sort by? - const matches = sortBy(members, [(member) => member.name]).filter((m) => - searchInAny(query, m.name, m.username) - ) - const matchLimit = 25 - - return ( -
- - - {matches.length > 0 && ( - m.id)} /> - )} - {matches.length > 25 && ( -
- And {matches.length - matchLimit} more... -
- )} - -
- ) -} - -function SortedLeaderboard(props: { - users: User[] - scoreFunction: (user: User) => number +function GroupLeaderboard(props: { + topUsers: { user: User; score: number }[] title: string + maxToShow: number header: string - maxToShow?: number }) { - const { users, scoreFunction, title, header, maxToShow } = props - const sortedUsers = users.sort((a, b) => scoreFunction(b) - scoreFunction(a)) + const { topUsers, title, maxToShow, header } = props + + const scoresByUser = topUsers.reduce((acc, { user, score }) => { + acc[user.id] = score + return acc + }, {} as { [key: string]: number }) + return ( t.user)} title={title} columns={[ - { header, renderCell: (user) => formatMoney(scoreFunction(user)) }, + { header, renderCell: (user) => formatMoney(scoresByUser[user.id]) }, ]} maxToShow={maxToShow} /> ) } -function GroupLeaderboards(props: { - traderScores: { [userId: string]: number } - creatorScores: { [userId: string]: number } - topTraders: User[] - topCreators: User[] - members: User[] - user: User | null | undefined -}) { - const { traderScores, creatorScores, members, topTraders, topCreators } = - props - const maxToShow = 50 - // Consider hiding M$0 - // If it's just one member (curator), show all bettors, otherwise just show members - return ( - -
- {members.length > 1 ? ( - <> - traderScores[user.id] ?? 0} - title="🏅 Top traders" - header="Profit" - maxToShow={maxToShow} - /> - creatorScores[user.id] ?? 0} - title="🏅 Top creators" - header="Market volume" - maxToShow={maxToShow} - /> - - ) : ( - <> - formatMoney(traderScores[user.id] ?? 0), - }, - ]} - maxToShow={maxToShow} - /> - - formatMoney(creatorScores[user.id] ?? 0), - }, - ]} - maxToShow={maxToShow} - /> - - )} -
- - ) -} - function AddContractButton(props: { group: Group; user: User }) { const { group, user } = props const [open, setOpen] = useState(false) @@ -684,3 +541,15 @@ function JoinGroupButton(props: { ) } + +const toTopUsers = async ( + cachedUserIds: { userId: string; score: number }[] +): Promise<{ user: User; score: number }[]> => + ( + await Promise.all( + cachedUserIds.map(async (e) => { + const user = await getUser(e.userId) + return { user, score: e.score ?? 0 } + }) + ) + ).filter((e) => e.user != null)