From ec49a73c74b6248ddb8b6ccdb5cb33ff959f0f4b Mon Sep 17 00:00:00 2001 From: James Grugett Date: Sat, 9 Apr 2022 18:10:58 -0500 Subject: [PATCH] Algo feed (#77) * Implement algo feed * Remove 'See more...' from feed items * Fix problem with useUpdatedContracts. * Tweak some params --- common/recommended-contracts.ts | 94 ++++++++++ common/util/math.ts | 6 + web/components/feed/feed-items.tsx | 8 - web/components/feed/find-active-contracts.ts | 5 +- web/hooks/use-algo-feed.ts | 172 +++++++++++++++++++ web/hooks/use-bets.ts | 9 - web/hooks/use-contracts.ts | 13 -- web/hooks/use-find-active-contracts.ts | 150 ---------------- web/hooks/use-user-bets.ts | 14 ++ web/lib/firebase/contracts.ts | 2 +- web/pages/home.tsx | 88 +++------- 11 files changed, 315 insertions(+), 246 deletions(-) create mode 100644 common/recommended-contracts.ts create mode 100644 common/util/math.ts create mode 100644 web/hooks/use-algo-feed.ts delete mode 100644 web/hooks/use-find-active-contracts.ts diff --git a/common/recommended-contracts.ts b/common/recommended-contracts.ts new file mode 100644 index 00000000..fe849d4b --- /dev/null +++ b/common/recommended-contracts.ts @@ -0,0 +1,94 @@ +import _ from 'lodash' +import { Contract } from './contract' +import { filterDefined } from './util/array' +import { addObjects } from './util/object' + +export const getRecommendedContracts = ( + contractsById: { [contractId: string]: Contract }, + yourBetOnContractIds: string[] +) => { + const contracts = Object.values(contractsById) + const yourContracts = filterDefined( + yourBetOnContractIds.map((contractId) => contractsById[contractId]) + ) + + const yourContractIds = new Set(yourContracts.map((c) => c.id)) + const notYourContracts = contracts.filter((c) => !yourContractIds.has(c.id)) + + const yourWordFrequency = contractsToWordFrequency(yourContracts) + const otherWordFrequency = contractsToWordFrequency(notYourContracts) + const words = _.union( + Object.keys(yourWordFrequency), + Object.keys(otherWordFrequency) + ) + + const yourWeightedFrequency = _.fromPairs( + _.map(words, (word) => { + const [yourFreq, otherFreq] = [ + yourWordFrequency[word] ?? 0, + otherWordFrequency[word] ?? 0, + ] + + const score = yourFreq / (yourFreq + otherFreq + 0.0001) + + return [word, score] + }) + ) + + // console.log( + // 'your weighted frequency', + // _.sortBy(_.toPairs(yourWeightedFrequency), ([, freq]) => -freq) + // ) + + const scoredContracts = contracts.map((contract) => { + const wordFrequency = contractToWordFrequency(contract) + + const score = _.sumBy(Object.keys(wordFrequency), (word) => { + const wordFreq = wordFrequency[word] ?? 0 + const weight = yourWeightedFrequency[word] ?? 0 + return wordFreq * weight + }) + + return { + contract, + score, + } + }) + + return _.sortBy(scoredContracts, (scored) => -scored.score).map( + (scored) => scored.contract + ) +} + +const contractToText = (contract: Contract) => { + const { description, question, tags, creatorUsername } = contract + return `${creatorUsername} ${question} ${tags.join(' ')} ${description}` +} + +const getWordsCount = (text: string) => { + const normalizedText = text.replace(/[^a-zA-Z]/g, ' ').toLowerCase() + const words = normalizedText.split(' ').filter((word) => word) + + const counts: { [word: string]: number } = {} + for (const word of words) { + if (counts[word]) counts[word]++ + else counts[word] = 1 + } + return counts +} + +const toFrequency = (counts: { [word: string]: number }) => { + const total = _.sum(Object.values(counts)) + return _.mapValues(counts, (count) => count / total) +} + +const contractToWordFrequency = (contract: Contract) => + toFrequency(getWordsCount(contractToText(contract))) + +const contractsToWordFrequency = (contracts: Contract[]) => { + const frequencySum = contracts + .map(contractToWordFrequency) + .reduce(addObjects, {}) + + return toFrequency(frequencySum) +} diff --git a/common/util/math.ts b/common/util/math.ts new file mode 100644 index 00000000..f89e9d85 --- /dev/null +++ b/common/util/math.ts @@ -0,0 +1,6 @@ +export const logInterpolation = (min: number, max: number, value: number) => { + if (value <= min) return 0 + if (value >= max) return 1 + + return Math.log(value - min + 1) / Math.log(max - min + 1) +} diff --git a/web/components/feed/feed-items.tsx b/web/components/feed/feed-items.tsx index 86c4aaef..fbb9ea11 100644 --- a/web/components/feed/feed-items.tsx +++ b/web/components/feed/feed-items.tsx @@ -340,14 +340,6 @@ export function FeedQuestion(props: { > {question} - {!showDescription && ( - -
See more...
-
- )} {(isBinary || resolution) && ( diff --git a/web/components/feed/find-active-contracts.ts b/web/components/feed/find-active-contracts.ts index 7048f299..42737f47 100644 --- a/web/components/feed/find-active-contracts.ts +++ b/web/components/feed/find-active-contracts.ts @@ -59,7 +59,10 @@ export function findActiveContracts( } let activeContracts = allContracts.filter( - (contract) => contract.visibility === 'public' && !contract.isResolved + (contract) => + contract.visibility === 'public' && + !contract.isResolved && + (contract.closeTime ?? Infinity) > Date.now() ) activeContracts = _.sortBy( activeContracts, diff --git a/web/hooks/use-algo-feed.ts b/web/hooks/use-algo-feed.ts new file mode 100644 index 00000000..68958821 --- /dev/null +++ b/web/hooks/use-algo-feed.ts @@ -0,0 +1,172 @@ +import _ from 'lodash' +import { useState, useEffect, useMemo } from 'react' +import { Bet } from '../../common/bet' +import { Comment } from '../../common/comment' +import { Contract } from '../../common/contract' +import { User } from '../../common/user' +import { logInterpolation } from '../../common/util/math' +import { getRecommendedContracts } from '../../common/recommended-contracts' +import { useSeenContracts } from './use-seen-contracts' +import { useGetUserBetContractIds, useUserBetContracts } from './use-user-bets' + +const MAX_FEED_CONTRACTS = 75 + +export const useAlgoFeed = ( + user: User | null | undefined, + contracts: Contract[] | undefined, + recentBets: Bet[] | undefined, + recentComments: Comment[] | undefined +) => { + const initialContracts = useMemo(() => contracts, [!!contracts]) + const initialBets = useMemo(() => recentBets, [!!recentBets]) + const initialComments = useMemo(() => recentComments, [!!recentComments]) + + const yourBetContractIds = useGetUserBetContractIds(user?.id) + // Update user bet contracts in local storage. + useUserBetContracts(user?.id) + + const seenContracts = useSeenContracts() + + const [algoFeed, setAlgoFeed] = useState([]) + + useEffect(() => { + if (initialContracts && initialBets && initialComments) { + const eligibleContracts = initialContracts.filter( + (c) => !c.isResolved && (c.closeTime ?? Infinity) > Date.now() + ) + const contracts = getAlgoFeed( + eligibleContracts, + initialBets, + initialComments, + yourBetContractIds, + seenContracts + ) + setAlgoFeed(contracts) + } + }, [ + initialBets, + initialComments, + initialContracts, + seenContracts, + yourBetContractIds, + ]) + + return algoFeed +} + +const getAlgoFeed = ( + contracts: Contract[], + recentBets: Bet[], + recentComments: Comment[], + yourBetContractIds: string[], + seenContracts: { [contractId: string]: number } +) => { + const contractsById = _.keyBy(contracts, (c) => c.id) + + const recommended = getRecommendedContracts(contractsById, yourBetContractIds) + const confidence = logInterpolation(0, 100, yourBetContractIds.length) + const recommendedScores = _.fromPairs( + recommended.map((c, index) => { + const score = 1 - index / recommended.length + const withConfidence = score * confidence + (1 - confidence) + return [c.id, withConfidence] as [string, number] + }) + ) + + const seenScores = _.fromPairs( + contracts.map( + (c) => [c.id, getSeenContractsScore(c, seenContracts)] as [string, number] + ) + ) + + const activityScores = getContractsActivityScores( + contracts, + recentComments, + recentBets, + seenContracts + ) + + const combinedScores = contracts.map((contract) => { + const score = + (recommendedScores[contract.id] ?? 0) * + (seenScores[contract.id] ?? 0) * + (activityScores[contract.id] ?? 0) + return { contract, score } + }) + + const sorted = _.sortBy(combinedScores, (c) => -c.score) + return sorted.map((c) => c.contract).slice(0, MAX_FEED_CONTRACTS) +} + +function getContractsActivityScores( + contracts: Contract[], + recentComments: Comment[], + recentBets: Bet[], + seenContracts: { [contractId: string]: number } +) { + const contractBets = _.groupBy(recentBets, (bet) => bet.contractId) + const contractMostRecentBet = _.mapValues( + contractBets, + (bets) => _.maxBy(bets, (bet) => bet.createdTime) as Bet + ) + + const contractComments = _.groupBy( + recentComments, + (comment) => comment.contractId + ) + const contractMostRecentComment = _.mapValues( + contractComments, + (comments) => _.maxBy(comments, (c) => c.createdTime) as Comment + ) + + const scoredContracts = contracts.map((contract) => { + const seenTime = seenContracts[contract.id] + const lastCommentTime = contractMostRecentComment[contract.id]?.createdTime + const hasNewComments = + !seenTime || (lastCommentTime && lastCommentTime > seenTime) + const newCommentScore = hasNewComments ? 1 : 0.75 + + const commentCount = contractComments[contract.id]?.length ?? 0 + const betCount = contractBets[contract.id]?.length ?? 0 + const activtyCount = betCount + commentCount * 5 + const activityCountScore = + 0.5 + 0.5 * logInterpolation(0, 200, activtyCount) + + const lastBetTime = contractMostRecentBet[contract.id]?.createdTime + const timeSinceLastBet = !lastBetTime + ? contract.createdTime + : Date.now() - lastBetTime + const daysAgo = timeSinceLastBet / oneDayMs + const timeAgoScore = 1 - logInterpolation(0, 3, daysAgo) + + const score = newCommentScore * activityCountScore * timeAgoScore + + // Map score to [0.5, 1] since no recent activty is not a deal breaker. + const mappedScore = 0.5 + score / 2 + return [contract.id, mappedScore] as [string, number] + }) + + return _.fromPairs(scoredContracts) +} + +const oneDayMs = 24 * 60 * 60 * 1000 + +function getSeenContractsScore( + contract: Contract, + seenContracts: { [contractId: string]: number } +) { + const lastSeen = seenContracts[contract.id] + if (lastSeen === undefined) { + return 1 + } + + const daysAgo = (Date.now() - lastSeen) / oneDayMs + + if (daysAgo < 0.5) { + const frac = logInterpolation(0, 0.5, daysAgo) + return 0.5 * frac + } + + const frac = logInterpolation(0.5, 14, daysAgo) + return 0.5 + 0.5 * frac +} diff --git a/web/hooks/use-bets.ts b/web/hooks/use-bets.ts index 4c530f5b..5ea66e1c 100644 --- a/web/hooks/use-bets.ts +++ b/web/hooks/use-bets.ts @@ -2,7 +2,6 @@ import { useEffect, useState } from 'react' import { Contract } from '../../common/contract' import { Bet, - getRecentBets, listenForBets, listenForRecentBets, withoutAnteBets, @@ -37,11 +36,3 @@ export const useRecentBets = () => { useEffect(() => listenForRecentBets(setRecentBets), []) return recentBets } - -export const useGetRecentBets = () => { - const [recentBets, setRecentBets] = useState() - useEffect(() => { - getRecentBets().then(setRecentBets) - }, []) - return recentBets -} diff --git a/web/hooks/use-contracts.ts b/web/hooks/use-contracts.ts index af36cd82..26c43ed1 100644 --- a/web/hooks/use-contracts.ts +++ b/web/hooks/use-contracts.ts @@ -39,19 +39,6 @@ export const useInactiveContracts = () => { return contracts } -export const useUpdatedContracts = (initialContracts: Contract[]) => { - const [contracts, setContracts] = useState(initialContracts) - - useEffect(() => { - return listenForContracts((newContracts) => { - const contractMap = _.fromPairs(newContracts.map((c) => [c.id, c])) - setContracts(initialContracts.map((c) => contractMap[c.id])) - }) - }, [initialContracts]) - - return contracts -} - export const useTaggedContracts = (tags: string[] | undefined) => { const [contracts, setContracts] = useState( tags && tags.length === 0 ? [] : undefined diff --git a/web/hooks/use-find-active-contracts.ts b/web/hooks/use-find-active-contracts.ts deleted file mode 100644 index 66573728..00000000 --- a/web/hooks/use-find-active-contracts.ts +++ /dev/null @@ -1,150 +0,0 @@ -import _ from 'lodash' -import { useMemo, useRef } from 'react' - -import { Fold } from '../../common/fold' -import { User } from '../../common/user' -import { filterDefined } from '../../common/util/array' -import { findActiveContracts } from '../components/feed/find-active-contracts' -import { Bet } from '../lib/firebase/bets' -import { Comment, getRecentComments } from '../lib/firebase/comments' -import { Contract, getActiveContracts } from '../lib/firebase/contracts' -import { listAllFolds } from '../lib/firebase/folds' -import { useInactiveContracts } from './use-contracts' -import { useFollowedFoldIds } from './use-fold' -import { useSeenContracts } from './use-seen-contracts' -import { useUserBetContracts } from './use-user-bets' - -// used in static props -export const getAllContractInfo = async () => { - let [contracts, folds] = await Promise.all([ - getActiveContracts().catch((_) => []), - listAllFolds().catch(() => []), - ]) - - const recentComments = await getRecentComments() - - return { contracts, recentComments, folds } -} - -const defaultExcludedTags = [ - 'meta', - 'test', - 'trolling', - 'spam', - 'transaction', - 'personal', -] -const includedWithDefaultFeed = (contract: Contract) => { - const { lowercaseTags } = contract - - if (lowercaseTags.length === 0) return false - if (lowercaseTags.some((tag) => defaultExcludedTags.includes(tag))) - return false - return true -} - -export const useFilterYourContracts = ( - user: User | undefined | null, - folds: Fold[], - contracts: Contract[] -) => { - const followedFoldIds = useFollowedFoldIds(user) - - const followedFolds = filterDefined( - (followedFoldIds ?? []).map((id) => folds.find((fold) => fold.id === id)) - ) - - // Save the initial followed fold slugs. - const followedFoldSlugsRef = useRef() - if (followedFoldIds && !followedFoldSlugsRef.current) - followedFoldSlugsRef.current = followedFolds.map((f) => f.slug) - const initialFollowedFoldSlugs = followedFoldSlugsRef.current - - const tagSet = new Set( - _.flatten(followedFolds.map((fold) => fold.lowercaseTags)) - ) - - const yourBetContractIds = useUserBetContracts(user?.id) - const yourBetContracts = yourBetContractIds - ? new Set(yourBetContractIds) - : undefined - - // Show no contracts before your info is loaded. - let yourContracts: Contract[] = [] - if (yourBetContracts && followedFoldIds) { - // Show default contracts if no folds are followed. - if (followedFoldIds.length === 0) - yourContracts = contracts.filter( - (contract) => - includedWithDefaultFeed(contract) || yourBetContracts.has(contract.id) - ) - else - yourContracts = contracts.filter( - (contract) => - contract.lowercaseTags.some((tag) => tagSet.has(tag)) || - yourBetContracts.has(contract.id) - ) - } - - return { - yourContracts, - initialFollowedFoldSlugs, - } -} - -export const useFindActiveContracts = (props: { - contracts: Contract[] - recentBets: Bet[] - recentComments: Comment[] -}) => { - const { contracts, recentBets, recentComments } = props - - const seenContracts = useSeenContracts() - - const activeContracts = findActiveContracts( - contracts, - recentComments, - recentBets, - seenContracts - ) - - const betsByContract = _.groupBy(recentBets, (bet) => bet.contractId) - - const activeBets = activeContracts.map( - (contract) => betsByContract[contract.id] ?? [] - ) - - const commentsByContract = _.groupBy( - recentComments, - (comment) => comment.contractId - ) - - const activeComments = activeContracts.map( - (contract) => commentsByContract[contract.id] ?? [] - ) - - return { - activeContracts, - activeBets, - activeComments, - } -} - -export const useExploreContracts = (maxContracts = 75) => { - const inactiveContracts = useInactiveContracts() - - const contractsDict = _.fromPairs( - (inactiveContracts ?? []).map((c) => [c.id, c]) - ) - - // Preserve random ordering once inactiveContracts loaded. - const exploreContractIds = useMemo( - () => _.shuffle(Object.keys(contractsDict)), - // eslint-disable-next-line react-hooks/exhaustive-deps - [!!inactiveContracts] - ).slice(0, maxContracts) - - if (!inactiveContracts) return undefined - - return filterDefined(exploreContractIds.map((id) => contractsDict[id])) -} diff --git a/web/hooks/use-user-bets.ts b/web/hooks/use-user-bets.ts index 0f1ceecb..82606933 100644 --- a/web/hooks/use-user-bets.ts +++ b/web/hooks/use-user-bets.ts @@ -52,3 +52,17 @@ export const useUserBetContracts = (userId: string | undefined) => { return contractIds } + +export const useGetUserBetContractIds = (userId: string | undefined) => { + const [contractIds, setContractIds] = useState([]) + + useEffect(() => { + const key = `user-bet-contractIds-${userId}` + const userBetContractJson = localStorage.getItem(key) + if (userBetContractJson) { + setContractIds(JSON.parse(userBetContractJson)) + } + }, [userId]) + + return contractIds +} diff --git a/web/lib/firebase/contracts.ts b/web/lib/firebase/contracts.ts index 46474c3f..9d85991d 100644 --- a/web/lib/firebase/contracts.ts +++ b/web/lib/firebase/contracts.ts @@ -124,7 +124,7 @@ const activeContractsQuery = query( contractCollection, where('isResolved', '==', false), where('visibility', '==', 'public'), - where('volume24Hours', '>', 0) + where('volume7Days', '>', 0) ) export function getActiveContracts() { diff --git a/web/pages/home.tsx b/web/pages/home.tsx index 6d086fca..bc21f9b6 100644 --- a/web/pages/home.tsx +++ b/web/pages/home.tsx @@ -2,96 +2,56 @@ import React from 'react' import Router from 'next/router' import _ from 'lodash' -import { Contract } from '../lib/firebase/contracts' import { Page } from '../components/page' import { ActivityFeed } from '../components/feed/activity-feed' -import { Comment } from '../lib/firebase/comments' import FeedCreate from '../components/feed-create' import { Spacer } from '../components/layout/spacer' import { Col } from '../components/layout/col' import { useUser } from '../hooks/use-user' -import { Fold } from '../../common/fold' import { LoadingIndicator } from '../components/loading-indicator' -import { - getAllContractInfo, - useFilterYourContracts, - useFindActiveContracts, -} from '../hooks/use-find-active-contracts' -import { fromPropz, usePropz } from '../hooks/use-propz' -import { useGetRecentBets, useRecentBets } from '../hooks/use-bets' +import { useRecentBets } from '../hooks/use-bets' import { useActiveContracts } from '../hooks/use-contracts' import { useRecentComments } from '../hooks/use-comments' +import { useAlgoFeed } from '../hooks/use-algo-feed' -export const getStaticProps = fromPropz(getStaticPropz) -export async function getStaticPropz() { - const contractInfo = await getAllContractInfo() - - return { - props: contractInfo, - revalidate: 60, // regenerate after a minute - } -} - -const Home = (props: { - contracts: Contract[] - folds: Fold[] - recentComments: Comment[] -}) => { - props = usePropz(props, getStaticPropz) ?? { - contracts: [], - folds: [], - recentComments: [], - } - const { folds } = props +const Home = () => { const user = useUser() - const contracts = useActiveContracts() ?? props.contracts - const { yourContracts } = useFilterYourContracts(user, folds, contracts) + const contracts = useActiveContracts() + const contractsDict = _.keyBy(contracts, 'id') - const initialRecentBets = useGetRecentBets() - const recentBets = useRecentBets() ?? initialRecentBets - const recentComments = useRecentComments() ?? props.recentComments + const recentBets = useRecentBets() + const recentComments = useRecentComments() - const { activeContracts } = useFindActiveContracts({ - contracts: yourContracts, - recentBets: initialRecentBets ?? [], - recentComments: props.recentComments, - }) + const feedContracts = useAlgoFeed(user, contracts, recentBets, recentComments) + + const updatedContracts = feedContracts.map( + (contract) => contractsDict[contract.id] ?? contract + ) if (user === null) { Router.replace('/') return <> } - const activityContent = recentBets ? ( - - ) : ( - - ) + const activityContent = + contracts && recentBets && recentComments ? ( + + ) : ( + + ) return ( - - - {/* {initialFollowedFoldSlugs !== undefined && - initialFollowedFoldSlugs.length === 0 && - !IS_PRIVATE_MANIFOLD && ( - - )} */} - - - + {activityContent}