open.source/manifold
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

"Fix "500 internal error" in large groups (#872)

Browse Source 
* Fix "500 internal error" in large groups (#856)

This reverts commit 28f0c6b1f8.

* Ship without touching prod and with some logs.
This commit is contained in:
FRC 2022-09-12 20:54:11 +01:00 committed by GitHub
parent 3cb36a36ec
commit ff81b859d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 148 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
common/group.ts
View File

@ -12,7 +12,18 @@ export type Group = {
aboutPostId?: string
chatDisabled?: boolean
mostRecentContractAddedTime?: number
cachedLeaderboard?: {
topTraders: {
userId: string
score: number
}[]
topCreators: {
userId: string
score: number
}[]
}
}
export const MAX_GROUP_NAME_LENGTH = 75
export const MAX_ABOUT_LENGTH = 140
export const MAX_ID_LENGTH = 60

61
functions/src/update-metrics.ts
View File

@ -4,9 +4,11 @@ import { groupBy, isEmpty, keyBy, last, sortBy } from 'lodash'
import { getValues, log, logMemory, writeAsync } from './utils'
import { Bet } from '../../common/bet'
import { Contract, CPMM } from '../../common/contract'
import { PortfolioMetrics, User } from '../../common/user'
import { DAY_MS } from '../../common/util/time'
import { getLoanUpdates } from '../../common/loans'
import { scoreTraders, scoreCreators } from '../../common/scoring'
import {
calculateCreatorVolume,
calculateNewPortfolioMetrics,
@ -15,6 +17,7 @@ import {
computeVolume,
} from '../../common/calculate-metrics'
import { getProbability } from '../../common/calculate'
import { Group } from 'common/group'
const firestore = admin.firestore()
@ -24,7 +27,8 @@ export const updateMetrics = functions
.onRun(updateMetricsCore)
export async function updateMetricsCore() {
const [users, contracts, bets, allPortfolioHistories] = await Promise.all([
const [users, contracts, bets, allPortfolioHistories, groups] =
await Promise.all([
getValues<User>(firestore.collection('users')),
getValues<Contract>(firestore.collection('contracts')),
getValues<Bet>(firestore.collectionGroup('bets')),
@ -33,7 +37,19 @@ export async function updateMetricsCore() {
.collectionGroup('portfolioHistory')
.where('timestamp', '>', Date.now() - 31 * DAY_MS) // so it includes just over a month ago
),
getValues<Group>(firestore.collection('groups')),
])
const contractsByGroup = await Promise.all(
groups.map((group) => {
return getValues(
firestore
.collection('groups')
.doc(group.id)
.collection('groupContracts')
)
})
)
log(
`Loaded ${users.length} users, ${contracts.length} contracts, and ${bets.length} bets.`
)
@ -41,6 +57,7 @@ export async function updateMetricsCore() {
const now = Date.now()
const betsByContract = groupBy(bets, (bet) => bet.contractId)
const contractUpdates = contracts
.filter((contract) => contract.id)
.map((contract) => {
@ -162,4 +179,46 @@ export async function updateMetricsCore() {
'set'
)
log(`Updated metrics for ${users.length} users.`)
try {
const groupUpdates = groups.map((group, index) => {
const groupContractIds = contractsByGroup[index] as GroupContractDoc[]
const groupContracts = groupContractIds.map(
(e) => contractsById[e.contractId]
)
const bets = groupContracts.map((e) => {
return betsByContract[e.id] ?? []
})
const creatorScores = scoreCreators(groupContracts)
const traderScores = scoreTraders(groupContracts, bets)
const topTraderScores = topUserScores(traderScores)
const topCreatorScores = topUserScores(creatorScores)
return {
doc: firestore.collection('groups').doc(group.id),
fields: {
cachedLeaderboard: {
topTraders: topTraderScores,
topCreators: topCreatorScores,
},
},
}
})
// Shipping without this for now to check it's working as intended
console.log('Group Leaderboard Updates', groupUpdates)
//await writeAsync(firestore, groupUpdates)
} catch (e) {
console.log('Error While Updating Group Leaderboards', e)
}
}
const topUserScores = (scores: { [userId: string]: number }) => {
const top50 = Object.entries(scores)
.sort(([, scoreA], [, scoreB]) => scoreB - scoreA)
.slice(0, 50)
return top50.map(([userId, score]) => ({ userId, score }))
}
type GroupContractDoc = { contractId: string; createdTime: number }

5
web/lib/firebase/groups.ts
View File

@ -24,7 +24,6 @@ import { Contract } from 'common/contract'
import { getContractFromId, updateContract } from 'web/lib/firebase/contracts'
import { db } from 'web/lib/firebase/init'
import { filterDefined } from 'common/util/array'
import { getUser } from 'web/lib/firebase/users'
export const groups = coll<Group>('groups')
export const groupMembers = (groupId: string) =>
@ -253,7 +252,7 @@ export function getGroupLinkToDisplay(contract: Contract) {
return groupToDisplay
}
export async function listMembers(group: Group) {
export async function listMemberIds(group: Group) {
const members = await getValues<GroupMemberDoc>(groupMembers(group.id))
return await Promise.all(members.map((m) => m.userId).map(getUser))
return members.map((m) => m.userId)
}

261
web/pages/group/[...slugs]/index.tsx
View File

@ -1,28 +1,28 @@
import React, { useState } from 'react'
import Link from 'next/link'
import { useRouter } from 'next/router'
import { debounce, sortBy, take } from 'lodash'
import { SearchIcon } from '@heroicons/react/outline'
import { toast } from 'react-hot-toast'
import { Group, GROUP_CHAT_SLUG } from 'common/group'
import { Page } from 'web/components/page'
import { listAllBets } from 'web/lib/firebase/bets'
import { Contract, listContractsByGroupSlug } from 'web/lib/firebase/contracts'
import {
addContractToGroup,
getGroupBySlug,
groupPath,
joinGroup,
listMembers,
listMemberIds,
updateGroup,
} from 'web/lib/firebase/groups'
import { Row } from 'web/components/layout/row'
import { firebaseLogin, getUser, User } from 'web/lib/firebase/users'
import { Col } from 'web/components/layout/col'
import { useUser } from 'web/hooks/use-user'
import { useGroup, useGroupContractIds, useMembers } from 'web/hooks/use-group'
import { scoreCreators, scoreTraders } from 'common/scoring'
import {
useGroup,
useGroupContractIds,
useMemberIds,
} from 'web/hooks/use-group'
import { Leaderboard } from 'web/components/leaderboard'
import { formatMoney } from 'common/util/format'
import { EditGroupButton } from 'web/components/groups/edit-group-button'
@ -35,9 +35,7 @@ import { LoadingIndicator } from 'web/components/loading-indicator'
import { Modal } from 'web/components/layout/modal'
import { ChoicesToggleGroup } from 'web/components/choices-toggle-group'
import { ContractSearch } from 'web/components/contract-search'
import { FollowList } from 'web/components/follow-list'
import { JoinOrLeaveGroupButton } from 'web/components/groups/groups-button'
import { searchInAny } from 'common/util/parse'
import { CopyLinkButton } from 'web/components/copy-link-button'
import { ENV_CONFIG } from 'common/envs/constants'
import { useSaveReferral } from 'web/hooks/use-save-referral'
@ -59,7 +57,7 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
const { slugs } = props.params
const group = await getGroupBySlug(slugs[0])
const members = group && (await listMembers(group))
const memberIds = group && (await listMemberIds(group))
const creatorPromise = group ? getUser(group.creatorId) : null
const contracts =
@ -71,19 +69,15 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
: 'open'
const aboutPost =
group && group.aboutPostId != null && (await getPost(group.aboutPostId))
const bets = await Promise.all(
contracts.map((contract: Contract) => listAllBets(contract.id))
)
const messages = group && (await listAllCommentsOnGroup(group.id))
const creatorScores = scoreCreators(contracts)
const traderScores = scoreTraders(contracts, bets)
const [topCreators, topTraders] =
(members && [
toTopUsers(creatorScores, members),
toTopUsers(traderScores, members),
]) ??
[]
const cachedTopTraderIds =
(group && group.cachedLeaderboard?.topTraders) ?? []
const cachedTopCreatorIds =
(group && group.cachedLeaderboard?.topCreators) ?? []
const topTraders = await toTopUsers(cachedTopTraderIds)
const topCreators = await toTopUsers(cachedTopCreatorIds)
const creator = await creatorPromise
// Only count unresolved markets
@ -93,11 +87,9 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
props: {
contractsCount,
group,
members,
memberIds,
creator,
traderScores,
topTraders,
creatorScores,
topCreators,
messages,
aboutPost,
@ -107,19 +99,6 @@ export async function getStaticPropz(props: { params: { slugs: string[] } }) {
revalidate: 60, // regenerate after a minute
}
}
function toTopUsers(userScores: { [userId: string]: number }, users: User[]) {
const topUserPairs = take(
sortBy(Object.entries(userScores), ([_, score]) => -1 * score),
10
).filter(([_, score]) => score >= 0.5)
const topUsers = topUserPairs.map(
([userId]) => users.filter((user) => user.id === userId)[0]
)
return topUsers.filter((user) => user)
}
export async function getStaticPaths() {
return { paths: [], fallback: 'blocking' }
}
@ -134,12 +113,10 @@ const groupSubpages = [
export default function GroupPage(props: {
contractsCount: number
group: Group | null
members: User[]
memberIds: string[]
creator: User
traderScores: { [userId: string]: number }
topTraders: User[]
creatorScores: { [userId: string]: number }
topCreators: User[]
topTraders: { user: User; score: number }[]
topCreators: { user: User; score: number }[]
messages: GroupComment[]
aboutPost: Post
suggestedFilter: 'open' | 'all'
@ -147,24 +124,15 @@ export default function GroupPage(props: {
props = usePropz(props, getStaticPropz) ?? {
contractsCount: 0,
group: null,
members: [],
memberIds: [],
creator: null,
traderScores: {},
topTraders: [],
creatorScores: {},
topCreators: [],
messages: [],
suggestedFilter: 'open',
}
const {
contractsCount,
creator,
traderScores,
topTraders,
creatorScores,
topCreators,
suggestedFilter,
} = props
const { contractsCount, creator, topTraders, topCreators, suggestedFilter } =
props
const router = useRouter()
const { slugs } = router.query as { slugs: string[] }
@ -175,7 +143,7 @@ export default function GroupPage(props: {
const user = useUser()
const isAdmin = useAdmin()
const members = useMembers(group?.id) ?? props.members
const memberIds = useMemberIds(group?.id ?? null) ?? props.memberIds
useSaveReferral(user, {
defaultReferrerUsername: creator.username,
@ -186,18 +154,25 @@ export default function GroupPage(props: {
return <Custom404 />
}
const isCreator = user && group && user.id === group.creatorId
const isMember = user && members.map((m) => m.id).includes(user.id)
const isMember = user && memberIds.includes(user.id)
const maxLeaderboardSize = 50
const leaderboard = (
<Col>
<GroupLeaderboards
traderScores={traderScores}
creatorScores={creatorScores}
topTraders={topTraders}
topCreators={topCreators}
members={members}
user={user}
<div className="mt-4 flex flex-col gap-8 px-4 md:flex-row">
<GroupLeaderboard
topUsers={topTraders}
title="🏅 Top traders"
header="Profit"
maxToShow={maxLeaderboardSize}
/>
<GroupLeaderboard
topUsers={topCreators}
title="🏅 Top creators"
header="Market volume"
maxToShow={maxLeaderboardSize}
/>
</div>
</Col>
)
@ -216,7 +191,7 @@ export default function GroupPage(props: {
creator={creator}
isCreator={!!isCreator}
user={user}
members={members}
memberIds={memberIds}
/>
</Col>
)
@ -312,9 +287,9 @@ function GroupOverview(props: {
creator: User
user: User | null | undefined
isCreator: boolean
members: User[]
memberIds: string[]
}) {
const { group, creator, isCreator, user, members } = props
const { group, creator, isCreator, user, memberIds } = props
const anyoneCanJoinChoices: { [key: string]: string } = {
Closed: 'false',
Open: 'true',
@ -333,7 +308,7 @@ function GroupOverview(props: {
const shareUrl = `https://${ENV_CONFIG.domain}${groupPath(
group.slug
)}${postFix}`
const isMember = user ? members.map((m) => m.id).includes(user.id) : false
const isMember = user ? memberIds.includes(user.id) : false
return (
<>
@ -399,155 +374,37 @@ function GroupOverview(props: {
/>
</Col>
)}
<Col className={'mt-2'}>
<div className="mb-2 text-lg">Members</div>
<GroupMemberSearch members={members} group={group} />
</Col>
</Col>
</>
)
}
function SearchBar(props: { setQuery: (query: string) => void }) {
const { setQuery } = props
const debouncedQuery = debounce(setQuery, 50)
return (
<div className={'relative'}>
<SearchIcon className={'absolute left-5 top-3.5 h-5 w-5 text-gray-500'} />
<input
type="text"
onChange={(e) => debouncedQuery(e.target.value)}
placeholder="Find a member"
className="input input-bordered mb-4 w-full pl-12"
/>
</div>
)
}
function GroupMemberSearch(props: { members: User[]; group: Group }) {
const [query, setQuery] = useState('')
const { group } = props
let { members } = props
// Use static members on load, but also listen to member changes:
const listenToMembers = useMembers(group.id)
if (listenToMembers) {
members = listenToMembers
}
// TODO use find-active-contracts to sort by?
const matches = sortBy(members, [(member) => member.name]).filter((m) =>
searchInAny(query, m.name, m.username)
)
const matchLimit = 25
return (
<div>
<SearchBar setQuery={setQuery} />
<Col className={'gap-2'}>
{matches.length > 0 && (
<FollowList userIds={matches.slice(0, matchLimit).map((m) => m.id)} />
)}
{matches.length > 25 && (
<div className={'text-center'}>
And {matches.length - matchLimit} more...
</div>
)}
</Col>
</div>
)
}
function SortedLeaderboard(props: {
users: User[]
scoreFunction: (user: User) => number
function GroupLeaderboard(props: {
topUsers: { user: User; score: number }[]
title: string
maxToShow: number
header: string
maxToShow?: number
}) {
const { users, scoreFunction, title, header, maxToShow } = props
const sortedUsers = users.sort((a, b) => scoreFunction(b) - scoreFunction(a))
const { topUsers, title, maxToShow, header } = props
const scoresByUser = topUsers.reduce((acc, { user, score }) => {
acc[user.id] = score
return acc
}, {} as { [key: string]: number })
return (
<Leaderboard
className="max-w-xl"
users={sortedUsers}
users={topUsers.map((t) => t.user)}
title={title}
columns={[
{ header, renderCell: (user) => formatMoney(scoreFunction(user)) },
{ header, renderCell: (user) => formatMoney(scoresByUser[user.id]) },
]}
maxToShow={maxToShow}
/>
)
}
function GroupLeaderboards(props: {
traderScores: { [userId: string]: number }
creatorScores: { [userId: string]: number }
topTraders: User[]
topCreators: User[]
members: User[]
user: User | null | undefined
}) {
const { traderScores, creatorScores, members, topTraders, topCreators } =
props
const maxToShow = 50
// Consider hiding M$0
// If it's just one member (curator), show all bettors, otherwise just show members
return (
<Col>
<div className="mt-4 flex flex-col gap-8 px-4 md:flex-row">
{members.length > 1 ? (
<>
<SortedLeaderboard
users={members}
scoreFunction={(user) => traderScores[user.id] ?? 0}
title="🏅 Top traders"
header="Profit"
maxToShow={maxToShow}
/>
<SortedLeaderboard
users={members}
scoreFunction={(user) => creatorScores[user.id] ?? 0}
title="🏅 Top creators"
header="Market volume"
maxToShow={maxToShow}
/>
</>
) : (
<>
<Leaderboard
className="max-w-xl"
title="🏅 Top traders"
users={topTraders}
columns={[
{
header: 'Profit',
renderCell: (user) => formatMoney(traderScores[user.id] ?? 0),
},
]}
maxToShow={maxToShow}
/>
<Leaderboard
className="max-w-xl"
title="🏅 Top creators"
users={topCreators}
columns={[
{
header: 'Market volume',
renderCell: (user) =>
formatMoney(creatorScores[user.id] ?? 0),
},
]}
maxToShow={maxToShow}
/>
</>
)}
</div>
</Col>
)
}
function AddContractButton(props: { group: Group; user: User }) {
const { group, user } = props
const [open, setOpen] = useState(false)
@ -684,3 +541,15 @@ function JoinGroupButton(props: {
</div>
)
}
const toTopUsers = async (
cachedUserIds: { userId: string; score: number }[]
): Promise<{ user: User; score: number }[]> =>
(
await Promise.all(
cachedUserIds.map(async (e) => {
const user = await getUser(e.userId)
return { user, score: e.score ?? 0 }
})
)
).filter((e) => e.user != null)