Fix "500 internal error" in large groups (#856)

* Members to memberIds

* Moved to update-metrics

common/group.ts

@@ -12,7 +12,22 @@ export type Group = {
   aboutPostId?: string
   chatDisabled?: boolean
   mostRecentContractAddedTime?: number
+  /** @deprecated - members and contracts now stored as subcollections*/
+  memberIds?: string[] // Deprecated
+  /** @deprecated - members and contracts now stored as subcollections*/
+  contractIds?: string[] // Deprecated
+  cachedLeaderboard?: {
+    topTraders: {
+      userId: string
+      score: number
+    }[]
+    topCreators: {
+      userId: string
+      score: number
+    }[]
   }
+}
+
 export const MAX_GROUP_NAME_LENGTH = 75
 export const MAX_ABOUT_LENGTH = 140
 export const MAX_ID_LENGTH = 60

functions/src/update-metrics.ts

@@ -4,9 +4,11 @@ import { groupBy, isEmpty, keyBy, last, sortBy } from 'lodash'
 import { getValues, log, logMemory, writeAsync } from './utils'
 import { Bet } from '../../common/bet'
 import { Contract, CPMM } from '../../common/contract'
+
 import { PortfolioMetrics, User } from '../../common/user'
 import { DAY_MS } from '../../common/util/time'
 import { getLoanUpdates } from '../../common/loans'
+import { scoreTraders, scoreCreators } from '../../common/scoring'
 import {
   calculateCreatorVolume,
   calculateNewPortfolioMetrics,
@@ -15,6 +17,7 @@ import {
   computeVolume,
 } from '../../common/calculate-metrics'
 import { getProbability } from '../../common/calculate'
+import { Group } from 'common/group'
 
 const firestore = admin.firestore()
 
@@ -24,7 +27,8 @@ export const updateMetrics = functions
   .onRun(updateMetricsCore)
 
 export async function updateMetricsCore() {
-  const [users, contracts, bets, allPortfolioHistories] = await Promise.all([
+  const [users, contracts, bets, allPortfolioHistories, groups] =
+    await Promise.all([
       getValues<User>(firestore.collection('users')),
       getValues<Contract>(firestore.collection('contracts')),
       getValues<Bet>(firestore.collectionGroup('bets')),
@@ -33,7 +37,19 @@ export async function updateMetricsCore() {
           .collectionGroup('portfolioHistory')
           .where('timestamp', '>', Date.now() - 31 * DAY_MS) // so it includes just over a month ago
       ),
+      getValues<Group>(firestore.collection('groups')),
     ])
+
+  const contractsByGroup = await Promise.all(
+    groups.map((group) => {
+      return getValues(
+        firestore
+          .collection('groups')
+          .doc(group.id)
+          .collection('groupContracts')
+      )
+    })
+  )
   log(
     `Loaded ${users.length} users, ${contracts.length} contracts, and ${bets.length} bets.`
   )
@@ -162,4 +178,40 @@ export async function updateMetricsCore() {
     'set'
   )
   log(`Updated metrics for ${users.length} users.`)
+
+  const groupUpdates = groups.map((group, index) => {
+    const groupContractIds = contractsByGroup[index] as GroupContractDoc[]
+    const groupContracts = groupContractIds.map(
+      (e) => contractsById[e.contractId]
+    )
+    const bets = groupContracts.map((e) => {
+      return betsByContract[e.id] ?? []
+    })
+
+    const creatorScores = scoreCreators(groupContracts)
+    const traderScores = scoreTraders(groupContracts, bets)
+
+    const topTraderScores = topUserScores(traderScores)
+    const topCreatorScores = topUserScores(creatorScores)
+
+    return {
+      doc: firestore.collection('groups').doc(group.id),
+      fields: {
+        cachedLeaderboard: {
+          topTraders: topTraderScores,
+          topCreators: topCreatorScores,
+        },
+      },
     }
+  })
+  await writeAsync(firestore, groupUpdates)
+}
+
+const topUserScores = (scores: { [userId: string]: number }) => {
+  const top50 = Object.entries(scores)
+    .sort(([, scoreA], [, scoreB]) => scoreB - scoreA)
+    .slice(0, 50)
+  return top50.map(([userId, score]) => ({ userId, score }))
+}
+
+type GroupContractDoc = { contractId: string; createdTime: number }

web/lib/firebase/groups.ts

@@ -24,7 +24,6 @@ import { Contract } from 'common/contract'
 import { getContractFromId, updateContract } from 'web/lib/firebase/contracts'
 import { db } from 'web/lib/firebase/init'
 import { filterDefined } from 'common/util/array'
-import { getUser } from 'web/lib/firebase/users'
 
 export const groups = coll<Group>('groups')
 export const groupMembers = (groupId: string) =>
@@ -253,7 +252,7 @@ export function getGroupLinkToDisplay(contract: Contract) {
   return groupToDisplay
 }
 
-export async function listMembers(group: Group) {
+export async function listMemberIds(group: Group) {
   const members = await getValues<GroupMemberDoc>(groupMembers(group.id))
-  return await Promise.all(members.map((m) => m.userId).map(getUser))
+  return members.map((m) => m.userId)
 }

web/pages/group/[...slugs]/index.tsx

@@ -1,28 +1,28 @@
 import React, { useState } from 'react'
 import Link from 'next/link'
 import { useRouter } from 'next/router'
-import { debounce, sortBy, take } from 'lodash'
-import { SearchIcon } from '@heroicons/react/outline'
 import { toast } from 'react-hot-toast'
 
 import { Group, GROUP_CHAT_SLUG } from 'common/group'
 import { Page } from 'web/components/page'
-import { listAllBets } from 'web/lib/firebase/bets'
 import { Contract, listContractsByGroupSlug } from 'web/lib/firebase/contracts'
 import {
   addContractToGroup,
   getGroupBySlug,
   groupPath,
   joinGroup,
-  listMembers,
+  listMemberIds,
   updateGroup,
 } from 'web/lib/firebase/groups'
 import { Row } from 'web/components/layout/row'
 import { firebaseLogin, getUser, User } from 'web/lib/firebase/users'
 import { Col } from 'web/components/layout/col'
 import { useUser } from 'web/hooks/use-user'
-import { useGroup, useGroupContractIds, useMembers } from 'web/hooks/use-group'
+import {
-import { scoreCreators, scoreTraders } from 'common/scoring'
+  useGroup,
+  useGroupContractIds,
+  useMemberIds,
+} from 'web/hooks/use-group'
 import { Leaderboard } from 'web/components/leaderboard'
 import { formatMoney } from 'common/util/format'
 import { EditGroupButton } from 'web/components/groups/edit-group-button'
@@ -35,9 +35,7 @@ import { LoadingIndicator } from 'web/components/loading-indicator'
 import { Modal } from 'web/components/layout/modal'
 import { ChoicesToggleGroup } from 'web/components/choices-toggle-group'
 import { ContractSearch } from 'web/components/contract-search'
-import { FollowList } from 'web/components/follow-list'
 import { JoinOrLeaveGroupButton } from 'web/components/groups/groups-button'
-import { searchInAny } from 'common/util/parse'
 import { CopyLinkButton } from 'web/components/copy-link-button'
 import { ENV_CONFIG } from 'common/envs/constants'
 import { useSaveReferral } from 'web/hooks/use-save-referral'
@@ -59,7 +57,7 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
   const { slugs } = props.params
 
   const group = await getGroupBySlug(slugs[0])
-  const members = group && (await listMembers(group))
+  const memberIds = group && (await listMemberIds(group))
   const creatorPromise = group ? getUser(group.creatorId) : null
 
   const contracts =
@@ -71,19 +69,15 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
       : 'open'
   const aboutPost =
     group && group.aboutPostId != null && (await getPost(group.aboutPostId))
-  const bets = await Promise.all(
-    contracts.map((contract: Contract) => listAllBets(contract.id))
-  )
   const messages = group && (await listAllCommentsOnGroup(group.id))
 
-  const creatorScores = scoreCreators(contracts)
+  const cachedTopTraderIds =
-  const traderScores = scoreTraders(contracts, bets)
+    (group && group.cachedLeaderboard?.topTraders) ?? []
-  const [topCreators, topTraders] =
+  const cachedTopCreatorIds =
-    (members && [
+    (group && group.cachedLeaderboard?.topCreators) ?? []
-      toTopUsers(creatorScores, members),
+  const topTraders = await toTopUsers(cachedTopTraderIds)
-      toTopUsers(traderScores, members),
+
-    ]) ??
+  const topCreators = await toTopUsers(cachedTopCreatorIds)
-    []
 
   const creator = await creatorPromise
   // Only count unresolved markets
@@ -93,11 +87,9 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
     props: {
       contractsCount,
       group,
-      members,
+      memberIds,
       creator,
-      traderScores,
       topTraders,
-      creatorScores,
       topCreators,
       messages,
       aboutPost,
@@ -107,19 +99,6 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
     revalidate: 60, // regenerate after a minute
   }
 }
-
-function toTopUsers(userScores: { [userId: string]: number }, users: User[]) {
-  const topUserPairs = take(
-    sortBy(Object.entries(userScores), ([_, score]) => -1 * score),
-    10
-  ).filter(([_, score]) => score >= 0.5)
-
-  const topUsers = topUserPairs.map(
-    ([userId]) => users.filter((user) => user.id === userId)[0]
-  )
-  return topUsers.filter((user) => user)
-}
-
 export async function getStaticPaths() {
   return { paths: [], fallback: 'blocking' }
 }
@@ -134,12 +113,10 @@ const groupSubpages = [
 export default function GroupPage(props: {
   contractsCount: number
   group: Group | null
-  members: User[]
+  memberIds: string[]
   creator: User
-  traderScores: { [userId: string]: number }
+  topTraders: { user: User; score: number }[]
-  topTraders: User[]
+  topCreators: { user: User; score: number }[]
-  creatorScores: { [userId: string]: number }
-  topCreators: User[]
   messages: GroupComment[]
   aboutPost: Post
   suggestedFilter: 'open' | 'all'
@@ -147,24 +124,15 @@ export default function GroupPage(props: {
   props = usePropz(props, getStaticPropz) ?? {
     contractsCount: 0,
     group: null,
-    members: [],
+    memberIds: [],
     creator: null,
-    traderScores: {},
     topTraders: [],
-    creatorScores: {},
     topCreators: [],
     messages: [],
     suggestedFilter: 'open',
   }
-  const {
+  const { contractsCount, creator, topTraders, topCreators, suggestedFilter } =
-    contractsCount,
+    props
-    creator,
-    traderScores,
-    topTraders,
-    creatorScores,
-    topCreators,
-    suggestedFilter,
-  } = props
 
   const router = useRouter()
   const { slugs } = router.query as { slugs: string[] }
@@ -175,7 +143,7 @@ export default function GroupPage(props: {
 
   const user = useUser()
   const isAdmin = useAdmin()
-  const members = useMembers(group?.id) ?? props.members
+  const memberIds = useMemberIds(group?.id ?? null) ?? props.memberIds
 
   useSaveReferral(user, {
     defaultReferrerUsername: creator.username,
@@ -186,18 +154,25 @@ export default function GroupPage(props: {
     return <Custom404 />
   }
   const isCreator = user && group && user.id === group.creatorId
-  const isMember = user && members.map((m) => m.id).includes(user.id)
+  const isMember = user && memberIds.includes(user.id)
+  const maxLeaderboardSize = 50
 
   const leaderboard = (
     <Col>
-      <GroupLeaderboards
+      <div className="mt-4 flex flex-col gap-8 px-4 md:flex-row">
-        traderScores={traderScores}
+        <GroupLeaderboard
-        creatorScores={creatorScores}
+          topUsers={topTraders}
-        topTraders={topTraders}
+          title="🏅 Top traders"
-        topCreators={topCreators}
+          header="Profit"
-        members={members}
+          maxToShow={maxLeaderboardSize}
-        user={user}
         />
+        <GroupLeaderboard
+          topUsers={topCreators}
+          title="🏅 Top creators"
+          header="Market volume"
+          maxToShow={maxLeaderboardSize}
+        />
+      </div>
     </Col>
   )
 
@@ -216,7 +191,7 @@ export default function GroupPage(props: {
         creator={creator}
         isCreator={!!isCreator}
         user={user}
-        members={members}
+        memberIds={memberIds}
       />
     </Col>
   )
@@ -312,9 +287,9 @@ function GroupOverview(props: {
   creator: User
   user: User | null | undefined
   isCreator: boolean
-  members: User[]
+  memberIds: string[]
 }) {
-  const { group, creator, isCreator, user, members } = props
+  const { group, creator, isCreator, user, memberIds } = props
   const anyoneCanJoinChoices: { [key: string]: string } = {
     Closed: 'false',
     Open: 'true',
@@ -333,7 +308,7 @@ function GroupOverview(props: {
   const shareUrl = `https://${ENV_CONFIG.domain}${groupPath(
     group.slug
   )}${postFix}`
-  const isMember = user ? members.map((m) => m.id).includes(user.id) : false
+  const isMember = user ? memberIds.includes(user.id) : false
 
   return (
     <>
@@ -399,155 +374,37 @@ function GroupOverview(props: {
             />
           </Col>
         )}
-
-        <Col className={'mt-2'}>
-          <div className="mb-2 text-lg">Members</div>
-          <GroupMemberSearch members={members} group={group} />
-        </Col>
       </Col>
     </>
   )
 }
 
-function SearchBar(props: { setQuery: (query: string) => void }) {
+function GroupLeaderboard(props: {
-  const { setQuery } = props
+  topUsers: { user: User; score: number }[]
-  const debouncedQuery = debounce(setQuery, 50)
-  return (
-    <div className={'relative'}>
-      <SearchIcon className={'absolute left-5 top-3.5 h-5 w-5 text-gray-500'} />
-      <input
-        type="text"
-        onChange={(e) => debouncedQuery(e.target.value)}
-        placeholder="Find a member"
-        className="input input-bordered mb-4 w-full pl-12"
-      />
-    </div>
-  )
-}
-
-function GroupMemberSearch(props: { members: User[]; group: Group }) {
-  const [query, setQuery] = useState('')
-  const { group } = props
-  let { members } = props
-
-  // Use static members on load, but also listen to member changes:
-  const listenToMembers = useMembers(group.id)
-  if (listenToMembers) {
-    members = listenToMembers
-  }
-
-  // TODO use find-active-contracts to sort by?
-  const matches = sortBy(members, [(member) => member.name]).filter((m) =>
-    searchInAny(query, m.name, m.username)
-  )
-  const matchLimit = 25
-
-  return (
-    <div>
-      <SearchBar setQuery={setQuery} />
-      <Col className={'gap-2'}>
-        {matches.length > 0 && (
-          <FollowList userIds={matches.slice(0, matchLimit).map((m) => m.id)} />
-        )}
-        {matches.length > 25 && (
-          <div className={'text-center'}>
-            And {matches.length - matchLimit} more...
-          </div>
-        )}
-      </Col>
-    </div>
-  )
-}
-
-function SortedLeaderboard(props: {
-  users: User[]
-  scoreFunction: (user: User) => number
   title: string
+  maxToShow: number
   header: string
-  maxToShow?: number
 }) {
-  const { users, scoreFunction, title, header, maxToShow } = props
+  const { topUsers, title, maxToShow, header } = props
-  const sortedUsers = users.sort((a, b) => scoreFunction(b) - scoreFunction(a))
+
+  const scoresByUser = topUsers.reduce((acc, { user, score }) => {
+    acc[user.id] = score
+    return acc
+  }, {} as { [key: string]: number })
+
   return (
     <Leaderboard
       className="max-w-xl"
-      users={sortedUsers}
+      users={topUsers.map((t) => t.user)}
       title={title}
       columns={[
-        { header, renderCell: (user) => formatMoney(scoreFunction(user)) },
+        { header, renderCell: (user) => formatMoney(scoresByUser[user.id]) },
       ]}
       maxToShow={maxToShow}
     />
   )
 }
 
-function GroupLeaderboards(props: {
-  traderScores: { [userId: string]: number }
-  creatorScores: { [userId: string]: number }
-  topTraders: User[]
-  topCreators: User[]
-  members: User[]
-  user: User | null | undefined
-}) {
-  const { traderScores, creatorScores, members, topTraders, topCreators } =
-    props
-  const maxToShow = 50
-  // Consider hiding M$0
-  // If it's just one member (curator), show all bettors, otherwise just show members
-  return (
-    <Col>
-      <div className="mt-4 flex flex-col gap-8 px-4 md:flex-row">
-        {members.length > 1 ? (
-          <>
-            <SortedLeaderboard
-              users={members}
-              scoreFunction={(user) => traderScores[user.id] ?? 0}
-              title="🏅 Top traders"
-              header="Profit"
-              maxToShow={maxToShow}
-            />
-            <SortedLeaderboard
-              users={members}
-              scoreFunction={(user) => creatorScores[user.id] ?? 0}
-              title="🏅 Top creators"
-              header="Market volume"
-              maxToShow={maxToShow}
-            />
-          </>
-        ) : (
-          <>
-            <Leaderboard
-              className="max-w-xl"
-              title="🏅 Top traders"
-              users={topTraders}
-              columns={[
-                {
-                  header: 'Profit',
-                  renderCell: (user) => formatMoney(traderScores[user.id] ?? 0),
-                },
-              ]}
-              maxToShow={maxToShow}
-            />
-            <Leaderboard
-              className="max-w-xl"
-              title="🏅 Top creators"
-              users={topCreators}
-              columns={[
-                {
-                  header: 'Market volume',
-                  renderCell: (user) =>
-                    formatMoney(creatorScores[user.id] ?? 0),
-                },
-              ]}
-              maxToShow={maxToShow}
-            />
-          </>
-        )}
-      </div>
-    </Col>
-  )
-}
-
 function AddContractButton(props: { group: Group; user: User }) {
   const { group, user } = props
   const [open, setOpen] = useState(false)
@@ -684,3 +541,15 @@ function JoinGroupButton(props: {
     </div>
   )
 }
+
+const toTopUsers = async (
+  cachedUserIds: { userId: string; score: number }[]
+): Promise<{ user: User; score: number }[]> =>
+  (
+    await Promise.all(
+      cachedUserIds.map(async (e) => {
+        const user = await getUser(e.userId)
+        return { user, score: e.score ?? 0 }
+      })
+    )
+  ).filter((e) => e.user != null)