Algo feed (#77)

* Implement algo feed

* Remove 'See more...' from feed items

* Fix problem with useUpdatedContracts.

* Tweak some params

common/recommended-contracts.ts

@@ -0,0 +1,94 @@
+import _ from 'lodash'
+import { Contract } from './contract'
+import { filterDefined } from './util/array'
+import { addObjects } from './util/object'
+
+export const getRecommendedContracts = (
+  contractsById: { [contractId: string]: Contract },
+  yourBetOnContractIds: string[]
+) => {
+  const contracts = Object.values(contractsById)
+  const yourContracts = filterDefined(
+    yourBetOnContractIds.map((contractId) => contractsById[contractId])
+  )
+
+  const yourContractIds = new Set(yourContracts.map((c) => c.id))
+  const notYourContracts = contracts.filter((c) => !yourContractIds.has(c.id))
+
+  const yourWordFrequency = contractsToWordFrequency(yourContracts)
+  const otherWordFrequency = contractsToWordFrequency(notYourContracts)
+  const words = _.union(
+    Object.keys(yourWordFrequency),
+    Object.keys(otherWordFrequency)
+  )
+
+  const yourWeightedFrequency = _.fromPairs(
+    _.map(words, (word) => {
+      const [yourFreq, otherFreq] = [
+        yourWordFrequency[word] ?? 0,
+        otherWordFrequency[word] ?? 0,
+      ]
+
+      const score = yourFreq / (yourFreq + otherFreq + 0.0001)
+
+      return [word, score]
+    })
+  )
+
+  // console.log(
+  //   'your weighted frequency',
+  //   _.sortBy(_.toPairs(yourWeightedFrequency), ([, freq]) => -freq)
+  // )
+
+  const scoredContracts = contracts.map((contract) => {
+    const wordFrequency = contractToWordFrequency(contract)
+
+    const score = _.sumBy(Object.keys(wordFrequency), (word) => {
+      const wordFreq = wordFrequency[word] ?? 0
+      const weight = yourWeightedFrequency[word] ?? 0
+      return wordFreq * weight
+    })
+
+    return {
+      contract,
+      score,
+    }
+  })
+
+  return _.sortBy(scoredContracts, (scored) => -scored.score).map(
+    (scored) => scored.contract
+  )
+}
+
+const contractToText = (contract: Contract) => {
+  const { description, question, tags, creatorUsername } = contract
+  return `${creatorUsername} ${question} ${tags.join(' ')} ${description}`
+}
+
+const getWordsCount = (text: string) => {
+  const normalizedText = text.replace(/[^a-zA-Z]/g, ' ').toLowerCase()
+  const words = normalizedText.split(' ').filter((word) => word)
+
+  const counts: { [word: string]: number } = {}
+  for (const word of words) {
+    if (counts[word]) counts[word]++
+    else counts[word] = 1
+  }
+  return counts
+}
+
+const toFrequency = (counts: { [word: string]: number }) => {
+  const total = _.sum(Object.values(counts))
+  return _.mapValues(counts, (count) => count / total)
+}
+
+const contractToWordFrequency = (contract: Contract) =>
+  toFrequency(getWordsCount(contractToText(contract)))
+
+const contractsToWordFrequency = (contracts: Contract[]) => {
+  const frequencySum = contracts
+    .map(contractToWordFrequency)
+    .reduce(addObjects, {})
+
+  return toFrequency(frequencySum)
+}

common/util/math.ts

@@ -0,0 +1,6 @@
+export const logInterpolation = (min: number, max: number, value: number) => {
+  if (value <= min) return 0
+  if (value >= max) return 1
+
+  return Math.log(value - min + 1) / Math.log(max - min + 1)
+}

web/components/feed/feed-items.tsx

@@ -340,14 +340,6 @@ export function FeedQuestion(props: {
             >
               {question}
             </SiteLink>
-            {!showDescription && (
-              <SiteLink
-                href={contractPath(contract)}
-                className="relative top-4 self-end text-sm sm:self-start"
-              >
-                <div className="pb-1.5 text-gray-400">See more...</div>
-              </SiteLink>
-            )}
           </Col>
           {(isBinary || resolution) && (
             <ResolutionOrChance className="items-center" contract={contract} />

web/components/feed/find-active-contracts.ts

@@ -59,7 +59,10 @@ export function findActiveContracts(
   }
 
   let activeContracts = allContracts.filter(
-    (contract) => contract.visibility === 'public' && !contract.isResolved
+    (contract) =>
+      contract.visibility === 'public' &&
+      !contract.isResolved &&
+      (contract.closeTime ?? Infinity) > Date.now()
   )
   activeContracts = _.sortBy(
     activeContracts,

web/hooks/use-algo-feed.ts

@@ -0,0 +1,172 @@
+import _ from 'lodash'
+import { useState, useEffect, useMemo } from 'react'
+import { Bet } from '../../common/bet'
+import { Comment } from '../../common/comment'
+import { Contract } from '../../common/contract'
+import { User } from '../../common/user'
+import { logInterpolation } from '../../common/util/math'
+import { getRecommendedContracts } from '../../common/recommended-contracts'
+import { useSeenContracts } from './use-seen-contracts'
+import { useGetUserBetContractIds, useUserBetContracts } from './use-user-bets'
+
+const MAX_FEED_CONTRACTS = 75
+
+export const useAlgoFeed = (
+  user: User | null | undefined,
+  contracts: Contract[] | undefined,
+  recentBets: Bet[] | undefined,
+  recentComments: Comment[] | undefined
+) => {
+  const initialContracts = useMemo(() => contracts, [!!contracts])
+  const initialBets = useMemo(() => recentBets, [!!recentBets])
+  const initialComments = useMemo(() => recentComments, [!!recentComments])
+
+  const yourBetContractIds = useGetUserBetContractIds(user?.id)
+  // Update user bet contracts in local storage.
+  useUserBetContracts(user?.id)
+
+  const seenContracts = useSeenContracts()
+
+  const [algoFeed, setAlgoFeed] = useState<Contract[]>([])
+
+  useEffect(() => {
+    if (initialContracts && initialBets && initialComments) {
+      const eligibleContracts = initialContracts.filter(
+        (c) => !c.isResolved && (c.closeTime ?? Infinity) > Date.now()
+      )
+      const contracts = getAlgoFeed(
+        eligibleContracts,
+        initialBets,
+        initialComments,
+        yourBetContractIds,
+        seenContracts
+      )
+      setAlgoFeed(contracts)
+    }
+  }, [
+    initialBets,
+    initialComments,
+    initialContracts,
+    seenContracts,
+    yourBetContractIds,
+  ])
+
+  return algoFeed
+}
+
+const getAlgoFeed = (
+  contracts: Contract[],
+  recentBets: Bet[],
+  recentComments: Comment[],
+  yourBetContractIds: string[],
+  seenContracts: { [contractId: string]: number }
+) => {
+  const contractsById = _.keyBy(contracts, (c) => c.id)
+
+  const recommended = getRecommendedContracts(contractsById, yourBetContractIds)
+  const confidence = logInterpolation(0, 100, yourBetContractIds.length)
+  const recommendedScores = _.fromPairs(
+    recommended.map((c, index) => {
+      const score = 1 - index / recommended.length
+      const withConfidence = score * confidence + (1 - confidence)
+      return [c.id, withConfidence] as [string, number]
+    })
+  )
+
+  const seenScores = _.fromPairs(
+    contracts.map(
+      (c) => [c.id, getSeenContractsScore(c, seenContracts)] as [string, number]
+    )
+  )
+
+  const activityScores = getContractsActivityScores(
+    contracts,
+    recentComments,
+    recentBets,
+    seenContracts
+  )
+
+  const combinedScores = contracts.map((contract) => {
+    const score =
+      (recommendedScores[contract.id] ?? 0) *
+      (seenScores[contract.id] ?? 0) *
+      (activityScores[contract.id] ?? 0)
+    return { contract, score }
+  })
+
+  const sorted = _.sortBy(combinedScores, (c) => -c.score)
+  return sorted.map((c) => c.contract).slice(0, MAX_FEED_CONTRACTS)
+}
+
+function getContractsActivityScores(
+  contracts: Contract[],
+  recentComments: Comment[],
+  recentBets: Bet[],
+  seenContracts: { [contractId: string]: number }
+) {
+  const contractBets = _.groupBy(recentBets, (bet) => bet.contractId)
+  const contractMostRecentBet = _.mapValues(
+    contractBets,
+    (bets) => _.maxBy(bets, (bet) => bet.createdTime) as Bet
+  )
+
+  const contractComments = _.groupBy(
+    recentComments,
+    (comment) => comment.contractId
+  )
+  const contractMostRecentComment = _.mapValues(
+    contractComments,
+    (comments) => _.maxBy(comments, (c) => c.createdTime) as Comment
+  )
+
+  const scoredContracts = contracts.map((contract) => {
+    const seenTime = seenContracts[contract.id]
+    const lastCommentTime = contractMostRecentComment[contract.id]?.createdTime
+    const hasNewComments =
+      !seenTime || (lastCommentTime && lastCommentTime > seenTime)
+    const newCommentScore = hasNewComments ? 1 : 0.75
+
+    const commentCount = contractComments[contract.id]?.length ?? 0
+    const betCount = contractBets[contract.id]?.length ?? 0
+    const activtyCount = betCount + commentCount * 5
+    const activityCountScore =
+      0.5 + 0.5 * logInterpolation(0, 200, activtyCount)
+
+    const lastBetTime = contractMostRecentBet[contract.id]?.createdTime
+    const timeSinceLastBet = !lastBetTime
+      ? contract.createdTime
+      : Date.now() - lastBetTime
+    const daysAgo = timeSinceLastBet / oneDayMs
+    const timeAgoScore = 1 - logInterpolation(0, 3, daysAgo)
+
+    const score = newCommentScore * activityCountScore * timeAgoScore
+
+    // Map score to [0.5, 1] since no recent activty is not a deal breaker.
+    const mappedScore = 0.5 + score / 2
+    return [contract.id, mappedScore] as [string, number]
+  })
+
+  return _.fromPairs(scoredContracts)
+}
+
+const oneDayMs = 24 * 60 * 60 * 1000
+
+function getSeenContractsScore(
+  contract: Contract,
+  seenContracts: { [contractId: string]: number }
+) {
+  const lastSeen = seenContracts[contract.id]
+  if (lastSeen === undefined) {
+    return 1
+  }
+
+  const daysAgo = (Date.now() - lastSeen) / oneDayMs
+
+  if (daysAgo < 0.5) {
+    const frac = logInterpolation(0, 0.5, daysAgo)
+    return 0.5 * frac
+  }
+
+  const frac = logInterpolation(0.5, 14, daysAgo)
+  return 0.5 + 0.5 * frac
+}

web/hooks/use-bets.ts

@@ -2,7 +2,6 @@ import { useEffect, useState } from 'react'
 import { Contract } from '../../common/contract'
 import {
   Bet,
-  getRecentBets,
   listenForBets,
   listenForRecentBets,
   withoutAnteBets,
@@ -37,11 +36,3 @@ export const useRecentBets = () => {
   useEffect(() => listenForRecentBets(setRecentBets), [])
   return recentBets
 }
-
-export const useGetRecentBets = () => {
-  const [recentBets, setRecentBets] = useState<Bet[] | undefined>()
-  useEffect(() => {
-    getRecentBets().then(setRecentBets)
-  }, [])
-  return recentBets
-}

web/hooks/use-contracts.ts

@@ -39,19 +39,6 @@ export const useInactiveContracts = () => {
   return contracts
 }
 
-export const useUpdatedContracts = (initialContracts: Contract[]) => {
-  const [contracts, setContracts] = useState(initialContracts)
-
-  useEffect(() => {
-    return listenForContracts((newContracts) => {
-      const contractMap = _.fromPairs(newContracts.map((c) => [c.id, c]))
-      setContracts(initialContracts.map((c) => contractMap[c.id]))
-    })
-  }, [initialContracts])
-
-  return contracts
-}
-
 export const useTaggedContracts = (tags: string[] | undefined) => {
   const [contracts, setContracts] = useState<Contract[] | undefined>(
     tags && tags.length === 0 ? [] : undefined

web/hooks/use-find-active-contracts.ts

@@ -1,150 +0,0 @@
-import _ from 'lodash'
-import { useMemo, useRef } from 'react'
-
-import { Fold } from '../../common/fold'
-import { User } from '../../common/user'
-import { filterDefined } from '../../common/util/array'
-import { findActiveContracts } from '../components/feed/find-active-contracts'
-import { Bet } from '../lib/firebase/bets'
-import { Comment, getRecentComments } from '../lib/firebase/comments'
-import { Contract, getActiveContracts } from '../lib/firebase/contracts'
-import { listAllFolds } from '../lib/firebase/folds'
-import { useInactiveContracts } from './use-contracts'
-import { useFollowedFoldIds } from './use-fold'
-import { useSeenContracts } from './use-seen-contracts'
-import { useUserBetContracts } from './use-user-bets'
-
-// used in static props
-export const getAllContractInfo = async () => {
-  let [contracts, folds] = await Promise.all([
-    getActiveContracts().catch((_) => []),
-    listAllFolds().catch(() => []),
-  ])
-
-  const recentComments = await getRecentComments()
-
-  return { contracts, recentComments, folds }
-}
-
-const defaultExcludedTags = [
-  'meta',
-  'test',
-  'trolling',
-  'spam',
-  'transaction',
-  'personal',
-]
-const includedWithDefaultFeed = (contract: Contract) => {
-  const { lowercaseTags } = contract
-
-  if (lowercaseTags.length === 0) return false
-  if (lowercaseTags.some((tag) => defaultExcludedTags.includes(tag)))
-    return false
-  return true
-}
-
-export const useFilterYourContracts = (
-  user: User | undefined | null,
-  folds: Fold[],
-  contracts: Contract[]
-) => {
-  const followedFoldIds = useFollowedFoldIds(user)
-
-  const followedFolds = filterDefined(
-    (followedFoldIds ?? []).map((id) => folds.find((fold) => fold.id === id))
-  )
-
-  // Save the initial followed fold slugs.
-  const followedFoldSlugsRef = useRef<string[] | undefined>()
-  if (followedFoldIds && !followedFoldSlugsRef.current)
-    followedFoldSlugsRef.current = followedFolds.map((f) => f.slug)
-  const initialFollowedFoldSlugs = followedFoldSlugsRef.current
-
-  const tagSet = new Set(
-    _.flatten(followedFolds.map((fold) => fold.lowercaseTags))
-  )
-
-  const yourBetContractIds = useUserBetContracts(user?.id)
-  const yourBetContracts = yourBetContractIds
-    ? new Set(yourBetContractIds)
-    : undefined
-
-  // Show no contracts before your info is loaded.
-  let yourContracts: Contract[] = []
-  if (yourBetContracts && followedFoldIds) {
-    // Show default contracts if no folds are followed.
-    if (followedFoldIds.length === 0)
-      yourContracts = contracts.filter(
-        (contract) =>
-          includedWithDefaultFeed(contract) || yourBetContracts.has(contract.id)
-      )
-    else
-      yourContracts = contracts.filter(
-        (contract) =>
-          contract.lowercaseTags.some((tag) => tagSet.has(tag)) ||
-          yourBetContracts.has(contract.id)
-      )
-  }
-
-  return {
-    yourContracts,
-    initialFollowedFoldSlugs,
-  }
-}
-
-export const useFindActiveContracts = (props: {
-  contracts: Contract[]
-  recentBets: Bet[]
-  recentComments: Comment[]
-}) => {
-  const { contracts, recentBets, recentComments } = props
-
-  const seenContracts = useSeenContracts()
-
-  const activeContracts = findActiveContracts(
-    contracts,
-    recentComments,
-    recentBets,
-    seenContracts
-  )
-
-  const betsByContract = _.groupBy(recentBets, (bet) => bet.contractId)
-
-  const activeBets = activeContracts.map(
-    (contract) => betsByContract[contract.id] ?? []
-  )
-
-  const commentsByContract = _.groupBy(
-    recentComments,
-    (comment) => comment.contractId
-  )
-
-  const activeComments = activeContracts.map(
-    (contract) => commentsByContract[contract.id] ?? []
-  )
-
-  return {
-    activeContracts,
-    activeBets,
-    activeComments,
-  }
-}
-
-export const useExploreContracts = (maxContracts = 75) => {
-  const inactiveContracts = useInactiveContracts()
-
-  const contractsDict = _.fromPairs(
-    (inactiveContracts ?? []).map((c) => [c.id, c])
-  )
-
-  // Preserve random ordering once inactiveContracts loaded.
-  const exploreContractIds = useMemo(
-    () => _.shuffle(Object.keys(contractsDict)),
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    [!!inactiveContracts]
-  ).slice(0, maxContracts)
-
-  if (!inactiveContracts) return undefined
-
-  return filterDefined(exploreContractIds.map((id) => contractsDict[id]))
-}

web/hooks/use-user-bets.ts

@@ -52,3 +52,17 @@ export const useUserBetContracts = (userId: string | undefined) => {
 
   return contractIds
 }
+
+export const useGetUserBetContractIds = (userId: string | undefined) => {
+  const [contractIds, setContractIds] = useState<string[]>([])
+
+  useEffect(() => {
+    const key = `user-bet-contractIds-${userId}`
+    const userBetContractJson = localStorage.getItem(key)
+    if (userBetContractJson) {
+      setContractIds(JSON.parse(userBetContractJson))
+    }
+  }, [userId])
+
+  return contractIds
+}

web/lib/firebase/contracts.ts

@@ -124,7 +124,7 @@ const activeContractsQuery = query(
   contractCollection,
   where('isResolved', '==', false),
   where('visibility', '==', 'public'),
-  where('volume24Hours', '>', 0)
+  where('volume7Days', '>', 0)
 )
 
 export function getActiveContracts() {

web/pages/home.tsx

@@ -2,96 +2,56 @@ import React from 'react'
 import Router from 'next/router'
 import _ from 'lodash'
 
-import { Contract } from '../lib/firebase/contracts'
 import { Page } from '../components/page'
 import { ActivityFeed } from '../components/feed/activity-feed'
-import { Comment } from '../lib/firebase/comments'
 import FeedCreate from '../components/feed-create'
 import { Spacer } from '../components/layout/spacer'
 import { Col } from '../components/layout/col'
 import { useUser } from '../hooks/use-user'
-import { Fold } from '../../common/fold'
 import { LoadingIndicator } from '../components/loading-indicator'
-import {
-  getAllContractInfo,
-  useFilterYourContracts,
-  useFindActiveContracts,
-} from '../hooks/use-find-active-contracts'
-import { fromPropz, usePropz } from '../hooks/use-propz'
-import { useGetRecentBets, useRecentBets } from '../hooks/use-bets'
+import { useRecentBets } from '../hooks/use-bets'
 import { useActiveContracts } from '../hooks/use-contracts'
 import { useRecentComments } from '../hooks/use-comments'
+import { useAlgoFeed } from '../hooks/use-algo-feed'
 
-export const getStaticProps = fromPropz(getStaticPropz)
-export async function getStaticPropz() {
-  const contractInfo = await getAllContractInfo()
-
-  return {
-    props: contractInfo,
-    revalidate: 60, // regenerate after a minute
-  }
-}
-
-const Home = (props: {
-  contracts: Contract[]
-  folds: Fold[]
-  recentComments: Comment[]
-}) => {
-  props = usePropz(props, getStaticPropz) ?? {
-    contracts: [],
-    folds: [],
-    recentComments: [],
-  }
-  const { folds } = props
+const Home = () => {
   const user = useUser()
 
-  const contracts = useActiveContracts() ?? props.contracts
-  const { yourContracts } = useFilterYourContracts(user, folds, contracts)
+  const contracts = useActiveContracts()
+  const contractsDict = _.keyBy(contracts, 'id')
 
-  const initialRecentBets = useGetRecentBets()
-  const recentBets = useRecentBets() ?? initialRecentBets
-  const recentComments = useRecentComments() ?? props.recentComments
+  const recentBets = useRecentBets()
+  const recentComments = useRecentComments()
 
-  const { activeContracts } = useFindActiveContracts({
-    contracts: yourContracts,
-    recentBets: initialRecentBets ?? [],
-    recentComments: props.recentComments,
-  })
+  const feedContracts = useAlgoFeed(user, contracts, recentBets, recentComments)
+
+  const updatedContracts = feedContracts.map(
+    (contract) => contractsDict[contract.id] ?? contract
+  )
 
   if (user === null) {
     Router.replace('/')
     return <></>
   }
 
-  const activityContent = recentBets ? (
-    <ActivityFeed
-      contracts={activeContracts}
-      recentBets={recentBets}
-      recentComments={recentComments}
-      mode="only-recent"
-    />
-  ) : (
-    <LoadingIndicator className="mt-4" />
-  )
+  const activityContent =
+    contracts && recentBets && recentComments ? (
+      <ActivityFeed
+        contracts={updatedContracts}
+        recentBets={recentBets}
+        recentComments={recentComments}
+        mode="only-recent"
+      />
+    ) : (
+      <LoadingIndicator className="mt-4" />
+    )
 
   return (
     <Page assertUser="signed-in">
       <Col className="items-center">
         <Col className="w-full max-w-[700px]">
           <FeedCreate user={user ?? undefined} />
-          <Spacer h={6} />
-
-          {/* {initialFollowedFoldSlugs !== undefined &&
-            initialFollowedFoldSlugs.length === 0 &&
-            !IS_PRIVATE_MANIFOLD && (
-              <FastFoldFollowing
-                user={user}
-                followedFoldSlugs={initialFollowedFoldSlugs}
-              />
-            )} */}
-
-          <Spacer h={5} />
-
+          <Spacer h={10} />
           {activityContent}
         </Col>
       </Col>