[In progress] Server-side feed computation (#106)

* Store view counts & last viewed time

* Schedule updating user recommendations. Compute using tf-idf.

* Update contract's lastBetTime and lastCommentTime on new bets and comments.

* Remove contract's lastUpdatedTime

* Remove folds activity feed

* Implement getFeed cloud function

* Hook up client to use getFeed

* Script to cache viewCounts and lastViewTime

* Batched wait all userRecommendations

* Cache view script runs on all users

* Update user feed each hour and get feed from cache doc.

* Delete view cache script

* Update feed script

* Tweak feed algorithm

* Compute recommendation scores from updateUserFeed

* Disable lastViewedScore factor

* Update lastCommentTime script

* Comment out console.log

* Fix timeout issue by calling new cloud functions with part of the work.

* Listen for contract updates to feed.

* Handle new user: use default feed of top markets this week

* Track lastUpdatedTime

* Tweak logic of calling cloud functions in batches

* Tweak cloud function batching

common/contract.ts

@@ -20,7 +20,9 @@ export type FullContract<
   visibility: 'public' | 'unlisted'
 
   createdTime: number // Milliseconds since epoch
-  lastUpdatedTime: number // If the question or description was changed
+  lastUpdatedTime?: number // Updated on new bet or comment
+  lastBetTime?: number
+  lastCommentTime?: number
   closeTime?: number // When no more trading is allowed
 
   isResolved: boolean

common/new-contract.ts

@@ -53,7 +53,6 @@ export function getNewContract(
     visibility: 'public',
     isResolved: false,
     createdTime: Date.now(),
-    lastUpdatedTime: Date.now(),
     closeTime,
 
     volume,

common/recommended-contracts.ts

@@ -1,8 +1,12 @@
 import * as _ from 'lodash'
+import { Bet } from './bet'
 import { Contract } from './contract'
+import { ClickEvent } from './tracking'
 import { filterDefined } from './util/array'
 import { addObjects } from './util/object'
 
+export const MAX_FEED_CONTRACTS = 75
+
 export const getRecommendedContracts = (
   contractsById: { [contractId: string]: Contract },
   yourBetOnContractIds: string[]
@@ -92,3 +96,88 @@ const contractsToWordFrequency = (contracts: Contract[]) => {
 
   return toFrequency(frequencySum)
 }
+
+export const getWordScores = (
+  contracts: Contract[],
+  contractViewCounts: { [contractId: string]: number },
+  clicks: ClickEvent[],
+  bets: Bet[]
+) => {
+  const contractClicks = _.groupBy(clicks, (click) => click.contractId)
+  const contractBets = _.groupBy(bets, (bet) => bet.contractId)
+
+  const yourContracts = contracts.filter(
+    (c) =>
+      contractViewCounts[c.id] || contractClicks[c.id] || contractBets[c.id]
+  )
+  const yourTfIdf = calculateContractTfIdf(yourContracts)
+
+  const contractWordScores = _.mapValues(
+    yourTfIdf,
+    (wordsTfIdf, contractId) => {
+      const viewCount = contractViewCounts[contractId] ?? 0
+      const clickCount = contractClicks[contractId]?.length ?? 0
+      const betCount = contractBets[contractId]?.length ?? 0
+
+      const factor =
+        -1 * Math.log(viewCount + 1) +
+        10 * Math.log(betCount + clickCount / 4 + 1)
+
+      return _.mapValues(wordsTfIdf, (tfIdf) => tfIdf * factor)
+    }
+  )
+
+  const wordScores = Object.values(contractWordScores).reduce(addObjects, {})
+  const minScore = Math.min(...Object.values(wordScores))
+  const maxScore = Math.max(...Object.values(wordScores))
+  const normalizedWordScores = _.mapValues(
+    wordScores,
+    (score) => (score - minScore) / (maxScore - minScore)
+  )
+
+  // console.log(
+  //   'your word scores',
+  //   _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(0, 100),
+  //   _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(-100)
+  // )
+
+  return normalizedWordScores
+}
+
+export function getContractScore(
+  contract: Contract,
+  wordScores: { [word: string]: number }
+) {
+  if (Object.keys(wordScores).length === 0) return 1
+
+  const wordFrequency = contractToWordFrequency(contract)
+  const score = _.sumBy(Object.keys(wordFrequency), (word) => {
+    const wordFreq = wordFrequency[word] ?? 0
+    const weight = wordScores[word] ?? 0
+    return wordFreq * weight
+  })
+
+  return score
+}
+
+// Caluculate Term Frequency-Inverse Document Frequency (TF-IDF):
+// https://medium.datadriveninvestor.com/tf-idf-in-natural-language-processing-8db8ef4a7736
+function calculateContractTfIdf(contracts: Contract[]) {
+  const contractFreq = contracts.map((c) => contractToWordFrequency(c))
+  const contractWords = contractFreq.map((freq) => Object.keys(freq))
+
+  const wordsCount: { [word: string]: number } = {}
+  for (const words of contractWords) {
+    for (const word of words) {
+      wordsCount[word] = (wordsCount[word] ?? 0) + 1
+    }
+  }
+
+  const wordIdf = _.mapValues(wordsCount, (count) =>
+    Math.log(contracts.length / count)
+  )
+  const contractWordsTfIdf = _.map(contractFreq, (wordFreq) =>
+    _.mapValues(wordFreq, (freq, word) => freq * wordIdf[word])
+  )
+  return _.fromPairs(contracts.map((c, i) => [c.id, contractWordsTfIdf[i]]))
+}

firestore.rules

@@ -35,6 +35,10 @@ service cloud.firestore {
       allow create: if userId == request.auth.uid;
     }
 
+    match /private-users/{userId}/cache/feed {
+      allow read: if userId == request.auth.uid || isAdmin();
+    }
+
     match /contracts/{contractId} {
       allow read;
       allow update: if request.resource.data.diff(resource.data).affectedKeys()

functions/src/index.ts

@@ -13,12 +13,16 @@ export * from './create-contract'
 export * from './create-user'
 export * from './create-fold'
 export * from './create-answer'
+export * from './on-create-bet'
 export * from './on-create-comment'
 export * from './on-fold-follow'
 export * from './on-fold-delete'
+export * from './on-view'
 export * from './unsubscribe'
 export * from './update-contract-metrics'
 export * from './update-user-metrics'
+export * from './update-recommendations'
+export * from './update-feed'
 export * from './backup-db'
 export * from './change-user-info'
 export * from './market-close-emails'

functions/src/on-create-bet.ts

@@ -0,0 +1,28 @@
+import * as functions from 'firebase-functions'
+import * as admin from 'firebase-admin'
+import * as _ from 'lodash'
+
+import { getContract } from './utils'
+import { Bet } from '../../common/bet'
+
+const firestore = admin.firestore()
+
+export const onCreateBet = functions.firestore
+  .document('contracts/{contractId}/bets/{betId}')
+  .onCreate(async (change, context) => {
+    const { contractId } = context.params as {
+      contractId: string
+    }
+
+    const contract = await getContract(contractId)
+    if (!contract)
+      throw new Error('Could not find contract corresponding with bet')
+
+    const bet = change.data() as Bet
+    const lastBetTime = bet.createdTime
+
+    await firestore
+      .collection('contracts')
+      .doc(contract.id)
+      .update({ lastBetTime, lastUpdatedTime: Date.now() })
+  })

functions/src/on-create-comment.ts

@@ -18,12 +18,19 @@ export const onCreateComment = functions.firestore
     }
 
     const contract = await getContract(contractId)
-    if (!contract) return
+    if (!contract)
+      throw new Error('Could not find contract corresponding with comment')
 
     const comment = change.data() as Comment
+    const lastCommentTime = comment.createdTime
 
     const commentCreator = await getUser(comment.userId)
-    if (!commentCreator) return
+    if (!commentCreator) throw new Error('Could not find contract creator')
+
+    await firestore
+      .collection('contracts')
+      .doc(contract.id)
+      .update({ lastCommentTime, lastUpdatedTime: Date.now() })
 
     let bet: Bet | undefined
     let answer: Answer | undefined

functions/src/on-view.ts

@@ -0,0 +1,24 @@
+import * as functions from 'firebase-functions'
+import * as admin from 'firebase-admin'
+import { View } from '../../common/tracking'
+
+const firestore = admin.firestore()
+
+export const onView = functions.firestore
+  .document('private-users/{userId}/views/{viewId}')
+  .onCreate(async (snapshot, context) => {
+    const { userId } = context.params
+
+    const { contractId, timestamp } = snapshot.data() as View
+
+    await firestore
+      .doc(`private-users/${userId}/cache/viewCounts`)
+      .set(
+        { [contractId]: admin.firestore.FieldValue.increment(1) },
+        { merge: true }
+      )
+
+    await firestore
+      .doc(`private-users/${userId}/cache/lastViewTime`)
+      .set({ [contractId]: timestamp }, { merge: true })
+  })

functions/src/scripts/cache-views.ts

@@ -0,0 +1,78 @@
+import * as admin from 'firebase-admin'
+import * as _ from 'lodash'
+
+import { initAdmin } from './script-init'
+initAdmin()
+
+import { getValues } from '../utils'
+import { View } from '../../../common/tracking'
+import { User } from '../../../common/user'
+import { batchedWaitAll } from '../../../common/util/promise'
+
+const firestore = admin.firestore()
+
+async function cacheViews() {
+  console.log('Caching views')
+
+  const users = await getValues<User>(firestore.collection('users'))
+
+  await batchedWaitAll(
+    users.map((user) => () => {
+      console.log('Caching views for', user.username)
+      return cacheUserViews(user.id)
+    })
+  )
+}
+
+async function cacheUserViews(userId: string) {
+  const views = await getValues<View>(
+    firestore.collection('private-users').doc(userId).collection('views')
+  )
+
+  const viewCounts: { [contractId: string]: number } = {}
+  for (const view of views) {
+    viewCounts[view.contractId] = (viewCounts[view.contractId] ?? 0) + 1
+  }
+
+  const lastViewTime: { [contractId: string]: number } = {}
+  for (const view of views) {
+    lastViewTime[view.contractId] = Math.max(
+      lastViewTime[view.contractId] ?? 0,
+      view.timestamp
+    )
+  }
+
+  await firestore
+    .doc(`private-users/${userId}/cache/viewCounts`)
+    .set(viewCounts, { merge: true })
+
+  await firestore
+    .doc(`private-users/${userId}/cache/lastViewTime`)
+    .set(lastViewTime, { merge: true })
+
+  console.log(viewCounts, lastViewTime)
+}
+
+async function deleteCache() {
+  console.log('Deleting view cache')
+
+  const users = await getValues<User>(firestore.collection('users'))
+
+  await batchedWaitAll(
+    users.map((user) => async () => {
+      console.log('Deleting view cache for', user.username)
+      await firestore.doc(`private-users/${user.id}/cache/viewCounts`).delete()
+      await firestore
+        .doc(`private-users/${user.id}/cache/lastViewTime`)
+        .delete()
+      await firestore
+        .doc(`private-users/${user.id}/cache/contractScores`)
+        .delete()
+      await firestore.doc(`private-users/${user.id}/cache/wordScores`).delete()
+    })
+  )
+}
+
+if (require.main === module) {
+  cacheViews().then(() => process.exit())
+}

functions/src/scripts/update-feed.ts

@@ -0,0 +1,38 @@
+import * as admin from 'firebase-admin'
+import * as _ from 'lodash'
+
+import { initAdmin } from './script-init'
+initAdmin()
+
+import { getValues } from '../utils'
+import { User } from '../../../common/user'
+import { batchedWaitAll } from '../../../common/util/promise'
+import { Contract } from '../../../common/contract'
+import { updateWordScores } from '../update-recommendations'
+import { getFeedContracts, doUserFeedUpdate } from '../update-feed'
+
+const firestore = admin.firestore()
+
+async function updateFeed() {
+  console.log('Updating feed')
+
+  const contracts = await getValues<Contract>(firestore.collection('contracts'))
+  const feedContracts = await getFeedContracts()
+  const users = await getValues<User>(
+    firestore.collection('users')
+    // .where('username', '==', 'JamesGrugett')
+  )
+
+  await batchedWaitAll(
+    users.map((user) => async () => {
+      console.log('Updating recs for', user.username)
+      await updateWordScores(user, contracts)
+      console.log('Updating feed for', user.username)
+      await doUserFeedUpdate(user, feedContracts)
+    })
+  )
+}
+
+if (require.main === module) {
+  updateFeed().then(() => process.exit())
+}

functions/src/scripts/update-last-comment-time.ts

@@ -0,0 +1,43 @@
+import * as admin from 'firebase-admin'
+import * as _ from 'lodash'
+
+import { initAdmin } from './script-init'
+initAdmin()
+
+import { Contract } from '../../../common/contract'
+import { getValues } from '../utils'
+import { Comment } from '../../../common/comment'
+
+async function updateLastCommentTime() {
+  const firestore = admin.firestore()
+  console.log('Updating contracts lastCommentTime')
+
+  const contracts = await getValues<Contract>(firestore.collection('contracts'))
+
+  console.log('Loaded', contracts.length, 'contracts')
+
+  for (const contract of contracts) {
+    const contractRef = firestore.doc(`contracts/${contract.id}`)
+
+    const lastComments = await getValues<Comment>(
+      contractRef.collection('comments').orderBy('createdTime', 'desc').limit(1)
+    )
+
+    if (lastComments.length > 0) {
+      const lastCommentTime = lastComments[0].createdTime
+      console.log(
+        'Updating lastCommentTime',
+        contract.question,
+        lastCommentTime
+      )
+
+      await contractRef.update({
+        lastCommentTime,
+      } as Partial<Contract>)
+    }
+  }
+}
+
+if (require.main === module) {
+  updateLastCommentTime().then(() => process.exit())
+}

functions/src/update-feed.ts

@@ -0,0 +1,210 @@
+import * as _ from 'lodash'
+import * as functions from 'firebase-functions'
+import * as admin from 'firebase-admin'
+
+import { getValue, getValues } from './utils'
+import { Contract } from '../../common/contract'
+import { logInterpolation } from '../../common/util/math'
+import { DAY_MS } from '../../common/util/time'
+import {
+  getProbability,
+  getOutcomeProbability,
+  getTopAnswer,
+} from '../../common/calculate'
+import { Bet } from '../../common/bet'
+import { Comment } from '../../common/comment'
+import { User } from '../../common/user'
+import {
+  getContractScore,
+  MAX_FEED_CONTRACTS,
+} from '../../common/recommended-contracts'
+import { callCloudFunction } from './call-cloud-function'
+
+const firestore = admin.firestore()
+
+export const updateFeed = functions.pubsub
+  .schedule('every 60 minutes')
+  .onRun(async () => {
+    const users = await getValues<User>(firestore.collection('users'))
+
+    const batchSize = 100
+    const userBatches: User[][] = []
+    for (let i = 0; i < users.length; i += batchSize) {
+      userBatches.push(users.slice(i, i + batchSize))
+    }
+
+    await Promise.all(
+      userBatches.map(async (users) =>
+        callCloudFunction('updateFeedBatch', { users })
+      )
+    )
+  })
+
+export const updateFeedBatch = functions.https.onCall(
+  async (data: { users: User[] }) => {
+    const { users } = data
+    const contracts = await getFeedContracts()
+
+    await Promise.all(users.map((user) => doUserFeedUpdate(user, contracts)))
+  }
+)
+
+export async function getFeedContracts() {
+  // Get contracts bet on or created in last week.
+  const contracts = await Promise.all([
+    getValues<Contract>(
+      firestore
+        .collection('contracts')
+        .where('isResolved', '==', false)
+        .where('volume7Days', '>', 0)
+    ),
+
+    getValues<Contract>(
+      firestore
+        .collection('contracts')
+        .where('isResolved', '==', false)
+        .where('createdTime', '>', Date.now() - DAY_MS * 7)
+        .where('volume7Days', '==', 0)
+    ),
+  ]).then(([activeContracts, inactiveContracts]) => {
+    const combined = [...activeContracts, ...inactiveContracts]
+    // Remove closed contracts.
+    return combined.filter((c) => (c.closeTime ?? Infinity) > Date.now())
+  })
+
+  return contracts
+}
+
+export const doUserFeedUpdate = async (user: User, contracts: Contract[]) => {
+  const userCacheCollection = firestore.collection(
+    `private-users/${user.id}/cache`
+  )
+  const [wordScores, lastViewedTime] = await Promise.all([
+    getValue<{ [word: string]: number }>(userCacheCollection.doc('wordScores')),
+    getValue<{ [contractId: string]: number }>(
+      userCacheCollection.doc('lastViewTime')
+    ),
+  ]).then((dicts) => dicts.map((dict) => dict ?? {}))
+
+  const scoredContracts = contracts.map((contract) => {
+    const score = scoreContract(
+      contract,
+      wordScores,
+      lastViewedTime[contract.id]
+    )
+    return [contract, score] as [Contract, number]
+  })
+
+  const sortedContracts = _.sortBy(
+    scoredContracts,
+    ([_, score]) => score
+  ).reverse()
+
+  // console.log(sortedContracts.map(([c, score]) => c.question + ': ' + score))
+
+  const feedContracts = sortedContracts
+    .slice(0, MAX_FEED_CONTRACTS)
+    .map(([c]) => c)
+
+  const feed = await Promise.all(
+    feedContracts.map((contract) => getRecentBetsAndComments(contract))
+  )
+
+  await userCacheCollection.doc('feed').set({ feed })
+}
+
+function scoreContract(
+  contract: Contract,
+  wordScores: { [word: string]: number },
+  viewTime: number | undefined
+) {
+  const recommendationScore = getContractScore(contract, wordScores)
+  const activityScore = getActivityScore(contract, viewTime)
+  // const lastViewedScore = getLastViewedScore(viewTime)
+  return recommendationScore * activityScore
+}
+
+function getActivityScore(contract: Contract, viewTime: number | undefined) {
+  const { createdTime, lastBetTime, lastCommentTime, outcomeType } = contract
+  const hasNewComments =
+    lastCommentTime && (!viewTime || lastCommentTime > viewTime)
+  const newCommentScore = hasNewComments ? 1 : 0.5
+
+  const timeSinceLastComment = Date.now() - (lastCommentTime ?? createdTime)
+  const commentDaysAgo = timeSinceLastComment / DAY_MS
+  const commentTimeScore =
+    0.25 + 0.75 * (1 - logInterpolation(0, 3, commentDaysAgo))
+
+  const timeSinceLastBet = Date.now() - (lastBetTime ?? createdTime)
+  const betDaysAgo = timeSinceLastBet / DAY_MS
+  const betTimeScore = 0.5 + 0.5 * (1 - logInterpolation(0, 3, betDaysAgo))
+
+  let prob = 0.5
+  if (outcomeType === 'BINARY') {
+    prob = getProbability(contract)
+  } else if (outcomeType === 'FREE_RESPONSE') {
+    const topAnswer = getTopAnswer(contract)
+    if (topAnswer)
+      prob = Math.max(0.5, getOutcomeProbability(contract, topAnswer.id))
+  }
+  const frac = 1 - Math.abs(prob - 0.5) ** 2 / 0.25
+  const probScore = 0.5 + frac * 0.5
+
+  const { volume24Hours, volume7Days } = contract
+  const combinedVolume = Math.log(volume24Hours + 1) + Math.log(volume7Days + 1)
+  const volumeScore = 0.5 + 0.5 * logInterpolation(4, 20, combinedVolume)
+
+  const score =
+    newCommentScore * commentTimeScore * betTimeScore * probScore * volumeScore
+
+  // Map score to [0.5, 1] since no recent activty is not a deal breaker.
+  const mappedScore = 0.5 + 0.5 * score
+  const newMappedScore = 0.7 + 0.3 * score
+
+  const isNew = Date.now() < contract.createdTime + DAY_MS
+  return isNew ? newMappedScore : mappedScore
+}
+
+function getLastViewedScore(viewTime: number | undefined) {
+  if (viewTime === undefined) {
+    return 1
+  }
+
+  const daysAgo = (Date.now() - viewTime) / DAY_MS
+
+  if (daysAgo < 0.5) {
+    const frac = logInterpolation(0, 0.5, daysAgo)
+    return 0.5 + 0.25 * frac
+  }
+
+  const frac = logInterpolation(0.5, 14, daysAgo)
+  return 0.75 + 0.25 * frac
+}
+
+async function getRecentBetsAndComments(contract: Contract) {
+  const contractDoc = firestore.collection('contracts').doc(contract.id)
+
+  const [recentBets, recentComments] = await Promise.all([
+    getValues<Bet>(
+      contractDoc
+        .collection('bets')
+        .where('createdTime', '>', Date.now() - DAY_MS)
+        .orderBy('createdTime', 'desc')
+        .limit(1)
+    ),
+
+    getValues<Comment>(
+      contractDoc
+        .collection('comments')
+        .where('createdTime', '>', Date.now() - 3 * DAY_MS)
+        .orderBy('createdTime', 'desc')
+        .limit(3)
+    ),
+  ])
+
+  return {
+    contract,
+    recentBets,
+    recentComments,
+  }
+}

functions/src/update-recommendations.ts

@@ -0,0 +1,71 @@
+import * as functions from 'firebase-functions'
+import * as admin from 'firebase-admin'
+import * as _ from 'lodash'
+
+import { getValue, getValues } from './utils'
+import { Contract } from '../../common/contract'
+import { Bet } from '../../common/bet'
+import { User } from '../../common/user'
+import { ClickEvent } from '../../common/tracking'
+import { getWordScores } from '../../common/recommended-contracts'
+import { batchedWaitAll } from '../../common/util/promise'
+import { callCloudFunction } from './call-cloud-function'
+
+const firestore = admin.firestore()
+
+export const updateRecommendations = functions.pubsub
+  .schedule('every 24 hours')
+  .onRun(async () => {
+    const users = await getValues<User>(firestore.collection('users'))
+
+    const batchSize = 100
+    const userBatches: User[][] = []
+    for (let i = 0; i < users.length; i += batchSize) {
+      userBatches.push(users.slice(i, i + batchSize))
+    }
+
+    await Promise.all(
+      userBatches.map((batch) =>
+        callCloudFunction('updateRecommendationsBatch', { users: batch })
+      )
+    )
+  })
+
+export const updateRecommendationsBatch = functions.https.onCall(
+  async (data: { users: User[] }) => {
+    const { users } = data
+
+    const contracts = await getValues<Contract>(
+      firestore.collection('contracts')
+    )
+
+    await batchedWaitAll(
+      users.map((user) => () => updateWordScores(user, contracts))
+    )
+  }
+)
+
+export const updateWordScores = async (user: User, contracts: Contract[]) => {
+  const [bets, viewCounts, clicks] = await Promise.all([
+    getValues<Bet>(
+      firestore.collectionGroup('bets').where('userId', '==', user.id)
+    ),
+
+    getValue<{ [contractId: string]: number }>(
+      firestore.doc(`private-users/${user.id}/cache/viewCounts`)
+    ),
+
+    getValues<ClickEvent>(
+      firestore
+        .collection(`private-users/${user.id}/events`)
+        .where('type', '==', 'click')
+    ),
+  ])
+
+  const wordScores = getWordScores(contracts, viewCounts ?? {}, clicks, bets)
+
+  const cachedCollection = firestore.collection(
+    `private-users/${user.id}/cache`
+  )
+  await cachedCollection.doc('wordScores').set(wordScores)
+}

functions/src/utils.ts

@@ -6,27 +6,33 @@ import { PrivateUser, User } from '../../common/user'
 export const isProd =
   admin.instanceId().app.options.projectId === 'mantic-markets'
 
-export const getValue = async <T>(collection: string, doc: string) => {
+export const getDoc = async <T>(collection: string, doc: string) => {
   const snap = await admin.firestore().collection(collection).doc(doc).get()
 
   return snap.exists ? (snap.data() as T) : undefined
 }
 
+export const getValue = async <T>(ref: admin.firestore.DocumentReference) => {
+  const snap = await ref.get()
+
+  return snap.exists ? (snap.data() as T) : undefined
+}
+
 export const getValues = async <T>(query: admin.firestore.Query) => {
   const snap = await query.get()
   return snap.docs.map((doc) => doc.data() as T)
 }
 
 export const getContract = (contractId: string) => {
-  return getValue<Contract>('contracts', contractId)
+  return getDoc<Contract>('contracts', contractId)
 }
 
 export const getUser = (userId: string) => {
-  return getValue<User>('users', userId)
+  return getDoc<User>('users', userId)
 }
 
 export const getPrivateUser = (userId: string) => {
-  return getValue<PrivateUser>('private-users', userId)
+  return getDoc<PrivateUser>('private-users', userId)
 }
 
 export const getUserByUsername = async (username: string) => {

web/components/feed/activity-feed.tsx

@@ -8,31 +8,27 @@ import { useUser } from '../../hooks/use-user'
 import { ContractActivity } from './contract-activity'
 
 export function ActivityFeed(props: {
-  contracts: Contract[]
+  feed: {
-  recentBets: Bet[]
+    contract: Contract
-  recentComments: Comment[]
+    recentBets: Bet[]
+    recentComments: Comment[]
+  }[]
   mode: 'only-recent' | 'abbreviated' | 'all'
   getContractPath?: (contract: Contract) => string
 }) {
-  const { contracts, recentBets, recentComments, mode, getContractPath } = props
+  const { feed, mode, getContractPath } = props
 
   const user = useUser()
 
-  const groupedBets = _.groupBy(recentBets, (bet) => bet.contractId)
-  const groupedComments = _.groupBy(
-    recentComments,
-    (comment) => comment.contractId
-  )
-
   return (
     <FeedContainer
-      contracts={contracts}
+      feed={feed}
-      renderContract={(contract) => (
+      renderItem={({ contract, recentBets, recentComments }) => (
         <ContractActivity
           user={user}
           contract={contract}
-          bets={groupedBets[contract.id] ?? []}
+          bets={recentBets}
-          comments={groupedComments[contract.id] ?? []}
+          comments={recentComments}
           mode={mode}
           contractPath={getContractPath ? getContractPath(contract) : undefined}
         />
@@ -42,18 +38,26 @@ export function ActivityFeed(props: {
 }
 
 function FeedContainer(props: {
-  contracts: Contract[]
+  feed: {
-  renderContract: (contract: Contract) => any
+    contract: Contract
+    recentBets: Bet[]
+    recentComments: Comment[]
+  }[]
+  renderItem: (item: {
+    contract: Contract
+    recentBets: Bet[]
+    recentComments: Comment[]
+  }) => any
 }) {
-  const { contracts, renderContract } = props
+  const { feed, renderItem } = props
 
   return (
     <Col className="items-center">
       <Col className="w-full max-w-3xl">
         <Col className="w-full divide-y divide-gray-300 self-center bg-white">
-          {contracts.map((contract) => (
+          {feed.map((item) => (
-            <div key={contract.id} className="py-6 px-2 sm:px-4">
+            <div key={item.contract.id} className="py-6 px-2 sm:px-4">
-              {renderContract(contract)}
+              {renderItem(item)}
             </div>
           ))}
         </Col>

web/components/feed/find-active-contracts.ts

@@ -9,11 +9,7 @@ const MAX_ACTIVE_CONTRACTS = 75
 // TODO: Maybe store last activity time directly in the contract?
 // Pros: simplifies this code; cons: harder to tweak "activity" definition later
 function lastActivityTime(contract: Contract) {
-  return Math.max(
+  return Math.max(contract.resolutionTime || 0, contract.createdTime)
-    contract.resolutionTime || 0,
-    contract.lastUpdatedTime,
-    contract.createdTime
-  )
 }
 
 // Types of activity to surface:

web/hooks/use-algo-feed.ts

@@ -1,213 +1,60 @@
 import _ from 'lodash'
-import { useState, useEffect, useMemo } from 'react'
+import { useState, useEffect } from 'react'
 import { Bet } from '../../common/bet'
 import { Comment } from '../../common/comment'
 import { Contract } from '../../common/contract'
-import { User } from '../../common/user'
-import { logInterpolation } from '../../common/util/math'
-import { getRecommendedContracts } from '../../common/recommended-contracts'
-import { useSeenContracts } from './use-seen-contracts'
-import { useGetUserBetContractIds, useUserBetContracts } from './use-user-bets'
-import { DAY_MS } from '../../common/util/time'
-import {
-  getProbability,
-  getOutcomeProbability,
-  getTopAnswer,
-} from '../../common/calculate'
 import { useTimeSinceFirstRender } from './use-time-since-first-render'
 import { trackLatency } from '../lib/firebase/tracking'
+import { User } from '../../common/user'
+import { getUserFeed } from '../lib/firebase/users'
+import { useUpdatedContracts } from './use-contracts'
+import {
+  getRecentBetsAndComments,
+  getTopWeeklyContracts,
+} from '../lib/firebase/contracts'
 
-const MAX_FEED_CONTRACTS = 75
+type feed = {
+  contract: Contract
+  recentBets: Bet[]
+  recentComments: Comment[]
+}[]
 
-export const useAlgoFeed = (
+export const useAlgoFeed = (user: User | null | undefined) => {
-  user: User | null | undefined,
+  const [feed, setFeed] = useState<feed>()
-  contracts: Contract[] | undefined,
-  recentBets: Bet[] | undefined,
-  recentComments: Comment[] | undefined
-) => {
-  const initialContracts = useMemo(() => contracts, [!!contracts])
-  const initialBets = useMemo(() => recentBets, [!!recentBets])
-  const initialComments = useMemo(() => recentComments, [!!recentComments])
-
-  const yourBetContractIds = useGetUserBetContractIds(user?.id)
-  // Update user bet contracts in local storage.
-  useUserBetContracts(user?.id)
-
-  const seenContracts = useSeenContracts()
-
-  const [algoFeed, setAlgoFeed] = useState<Contract[]>([])
 
   const getTime = useTimeSinceFirstRender()
 
   useEffect(() => {
-    if (
+    if (user) {
-      initialContracts &&
+      getUserFeed(user.id).then((feed) => {
-      initialBets &&
+        if (feed.length === 0) {
-      initialComments &&
+          getDefaultFeed().then((feed) => setFeed(feed))
-      yourBetContractIds
+        } else setFeed(feed)
-    ) {
+
-      const eligibleContracts = initialContracts.filter(
+        trackLatency('feed', getTime())
-        (c) => !c.isResolved && (c.closeTime ?? Infinity) > Date.now()
+        console.log('feed load time', getTime())
-      )
+      })
-      const contracts = getAlgoFeed(
-        eligibleContracts,
-        initialBets,
-        initialComments,
-        yourBetContractIds,
-        seenContracts
-      )
-      setAlgoFeed(contracts)
-      trackLatency('feed', getTime())
     }
-  }, [
+  }, [user?.id])
-    initialBets,
-    initialComments,
-    initialContracts,
-    seenContracts,
-    yourBetContractIds,
-    getTime,
-  ])
 
-  return algoFeed
+  return useUpdateFeed(feed)
 }
 
-const getAlgoFeed = (
+const useUpdateFeed = (feed: feed | undefined) => {
-  contracts: Contract[],
+  const contracts = useUpdatedContracts(feed?.map((item) => item.contract))
-  recentBets: Bet[],
-  recentComments: Comment[],
-  yourBetContractIds: string[],
-  seenContracts: { [contractId: string]: number }
-) => {
-  const contractsById = _.keyBy(contracts, (c) => c.id)
 
-  const recommended = getRecommendedContracts(contractsById, yourBetContractIds)
+  return feed && contracts
-  const confidence = logInterpolation(0, 100, yourBetContractIds.length)
+    ? feed.map(({ contract, ...other }, i) => ({
-  const recommendedScores = _.fromPairs(
+        ...other,
-    recommended.map((c, index) => {
+        contract: contracts[i],
-      const score = 1 - index / recommended.length
+      }))
-      const withConfidence = score * confidence + (1 - confidence)
+    : undefined
-      return [c.id, withConfidence] as [string, number]
-    })
-  )
-
-  const seenScores = _.fromPairs(
-    contracts.map(
-      (c) => [c.id, getSeenContractsScore(c, seenContracts)] as [string, number]
-    )
-  )
-
-  const activityScores = getContractsActivityScores(
-    contracts,
-    recentComments,
-    recentBets,
-    seenContracts
-  )
-
-  const combinedScores = contracts.map((contract) => {
-    const score =
-      (recommendedScores[contract.id] ?? 0) *
-      (seenScores[contract.id] ?? 0) *
-      (activityScores[contract.id] ?? 0)
-    return { contract, score }
-  })
-
-  const sorted = _.sortBy(combinedScores, (c) => -c.score)
-  return sorted.map((c) => c.contract).slice(0, MAX_FEED_CONTRACTS)
 }
 
-function getContractsActivityScores(
+const getDefaultFeed = async () => {
-  contracts: Contract[],
+  const contracts = await getTopWeeklyContracts()
-  recentComments: Comment[],
+  const feed = await Promise.all(
-  recentBets: Bet[],
+    contracts.map((c) => getRecentBetsAndComments(c))
-  seenContracts: { [contractId: string]: number }
-) {
-  const contractBets = _.groupBy(recentBets, (bet) => bet.contractId)
-  const contractMostRecentBet = _.mapValues(
-    contractBets,
-    (bets) => _.maxBy(bets, (bet) => bet.createdTime) as Bet
   )
-
+  return feed
-  const contractComments = _.groupBy(
-    recentComments,
-    (comment) => comment.contractId
-  )
-  const contractMostRecentComment = _.mapValues(
-    contractComments,
-    (comments) => _.maxBy(comments, (c) => c.createdTime) as Comment
-  )
-
-  const scoredContracts = contracts.map((contract) => {
-    const { outcomeType } = contract
-
-    const seenTime = seenContracts[contract.id]
-    const lastCommentTime = contractMostRecentComment[contract.id]?.createdTime
-    const hasNewComments =
-      !seenTime || (lastCommentTime && lastCommentTime > seenTime)
-    const newCommentScore = hasNewComments ? 1 : 0.5
-
-    const commentCount = contractComments[contract.id]?.length ?? 0
-    const betCount = contractBets[contract.id]?.length ?? 0
-    const activtyCount = betCount + commentCount * 5
-    const activityCountScore =
-      0.5 + 0.5 * logInterpolation(0, 200, activtyCount)
-
-    const { volume7Days, volume } = contract
-    const combinedVolume = Math.log(volume7Days + 1) + Math.log(volume + 1)
-    const volumeScore = 0.5 + 0.5 * logInterpolation(4, 25, combinedVolume)
-
-    const lastBetTime =
-      contractMostRecentBet[contract.id]?.createdTime ?? contract.createdTime
-    const timeSinceLastBet = Date.now() - lastBetTime
-    const daysAgo = timeSinceLastBet / DAY_MS
-    const timeAgoScore = 1 - logInterpolation(0, 3, daysAgo)
-
-    let prob = 0.5
-    if (outcomeType === 'BINARY') {
-      prob = getProbability(contract)
-    } else if (outcomeType === 'FREE_RESPONSE') {
-      const topAnswer = getTopAnswer(contract)
-      if (topAnswer)
-        prob = Math.max(0.5, getOutcomeProbability(contract, topAnswer.id))
-    }
-    const frac = 1 - Math.abs(prob - 0.5) ** 2 / 0.25
-    const probScore = 0.5 + frac * 0.5
-
-    const score =
-      newCommentScore *
-      activityCountScore *
-      volumeScore *
-      timeAgoScore *
-      probScore
-
-    // Map score to [0.5, 1] since no recent activty is not a deal breaker.
-    const mappedScore = 0.5 + score / 2
-    const newMappedScore = 0.75 + score / 4
-
-    const isNew = Date.now() < contract.createdTime + DAY_MS
-    const activityScore = isNew ? newMappedScore : mappedScore
-
-    return [contract.id, activityScore] as [string, number]
-  })
-
-  return _.fromPairs(scoredContracts)
-}
-
-function getSeenContractsScore(
-  contract: Contract,
-  seenContracts: { [contractId: string]: number }
-) {
-  const lastSeen = seenContracts[contract.id]
-  if (lastSeen === undefined) {
-    return 1
-  }
-
-  const daysAgo = (Date.now() - lastSeen) / DAY_MS
-
-  if (daysAgo < 0.5) {
-    const frac = logInterpolation(0, 0.5, daysAgo)
-    return 0.5 * frac
-  }
-
-  const frac = logInterpolation(0.5, 14, daysAgo)
-  return 0.5 + 0.5 * frac
 }

web/hooks/use-contracts.ts

@@ -1,8 +1,9 @@
 import _ from 'lodash'
-import { useEffect, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import {
   Contract,
   listenForActiveContracts,
+  listenForContract,
   listenForContracts,
   listenForHotContracts,
   listenForInactiveContracts,
@@ -71,3 +72,36 @@ export const useHotContracts = () => {
 
   return hotContracts
 }
+
+export const useUpdatedContracts = (contracts: Contract[] | undefined) => {
+  const [__, triggerUpdate] = useState(0)
+  const contractDict = useRef<{ [id: string]: Contract }>({})
+
+  useEffect(() => {
+    if (contracts === undefined) return
+
+    contractDict.current = _.fromPairs(contracts.map((c) => [c.id, c]))
+
+    const disposes = contracts.map((contract) => {
+      const { id } = contract
+
+      return listenForContract(id, (contract) => {
+        const curr = contractDict.current[id]
+        if (!_.isEqual(curr, contract)) {
+          contractDict.current[id] = contract as Contract
+          triggerUpdate((n) => n + 1)
+        }
+      })
+    })
+
+    triggerUpdate((n) => n + 1)
+
+    return () => {
+      disposes.forEach((dispose) => dispose())
+    }
+  }, [!!contracts])
+
+  return contracts && Object.keys(contractDict.current).length > 0
+    ? contracts.map((c) => contractDict.current[c.id])
+    : undefined
+}

web/lib/firebase/contracts.ts

@@ -23,6 +23,9 @@ import { createRNG, shuffle } from '../../../common/util/random'
 import { getCpmmProbability } from '../../../common/calculate-cpmm'
 import { formatMoney, formatPercent } from '../../../common/util/format'
 import { DAY_MS } from '../../../common/util/time'
+import { MAX_FEED_CONTRACTS } from '../../../common/recommended-contracts'
+import { Bet } from '../../../common/bet'
+import { Comment } from '../../../common/comment'
 export type { Contract }
 
 export function contractPath(contract: Contract) {
@@ -231,6 +234,16 @@ export async function getHotContracts() {
   )
 }
 
+const topWeeklyQuery = query(
+  contractCollection,
+  where('isResolved', '==', false),
+  orderBy('volume7Days', 'desc'),
+  limit(MAX_FEED_CONTRACTS)
+)
+export async function getTopWeeklyContracts() {
+  return await getValues<Contract>(topWeeklyQuery)
+}
+
 const closingSoonQuery = query(
   contractCollection,
   where('isResolved', '==', false),
@@ -276,3 +289,33 @@ export async function getDailyContracts(
 
   return contractsByDay
 }
+
+export async function getRecentBetsAndComments(contract: Contract) {
+  const contractDoc = doc(db, 'contracts', contract.id)
+
+  const [recentBets, recentComments] = await Promise.all([
+    getValues<Bet>(
+      query(
+        collection(contractDoc, 'bets'),
+        where('createdTime', '>', Date.now() - DAY_MS),
+        orderBy('createdTime', 'desc'),
+        limit(1)
+      )
+    ),
+
+    getValues<Comment>(
+      query(
+        collection(contractDoc, 'comments'),
+        where('createdTime', '>', Date.now() - 3 * DAY_MS),
+        orderBy('createdTime', 'desc'),
+        limit(3)
+      )
+    ),
+  ])
+
+  return {
+    contract,
+    recentBets,
+    recentComments,
+  }
+}

web/lib/firebase/users.ts

@@ -23,8 +23,11 @@ import _ from 'lodash'
 import { app } from './init'
 import { PrivateUser, User } from '../../../common/user'
 import { createUser } from './api-call'
-import { getValues, listenForValue, listenForValues } from './utils'
+import { getValue, getValues, listenForValue, listenForValues } from './utils'
 import { DAY_MS } from '../../../common/util/time'
+import { Contract } from './contracts'
+import { Bet } from './bets'
+import { Comment } from './comments'
 
 export type { User }
 
@@ -207,3 +210,15 @@ export async function getDailyNewUsers(
 
   return usersByDay
 }
+
+export async function getUserFeed(userId: string) {
+  const feedDoc = doc(db, 'private-users', userId, 'cache', 'feed')
+  const userFeed = await getValue<{
+    feed: {
+      contract: Contract
+      recentBets: Bet[]
+      recentComments: Comment[]
+    }[]
+  }>(feedDoc)
+  return userFeed?.feed ?? []
+}

web/pages/fold/[...slugs]/index.tsx

@@ -1,5 +1,4 @@
 import _ from 'lodash'
-import Link from 'next/link'
 
 import { Fold } from '../../../../common/fold'
 import { Comment } from '../../../../common/comment'
@@ -23,22 +22,17 @@ import { useUser } from '../../../hooks/use-user'
 import { useFold } from '../../../hooks/use-fold'
 import { SearchableGrid } from '../../../components/contract/contracts-list'
 import { useRouter } from 'next/router'
-import clsx from 'clsx'
 import { scoreCreators, scoreTraders } from '../../../../common/scoring'
 import { Leaderboard } from '../../../components/leaderboard'
-import { formatMoney, toCamelCase } from '../../../../common/util/format'
+import { formatMoney } from '../../../../common/util/format'
 import { EditFoldButton } from '../../../components/folds/edit-fold-button'
 import Custom404 from '../../404'
 import { FollowFoldButton } from '../../../components/folds/follow-fold-button'
-import FeedCreate from '../../../components/feed-create'
 import { SEO } from '../../../components/SEO'
 import { useTaggedContracts } from '../../../hooks/use-contracts'
 import { Linkify } from '../../../components/linkify'
 import { fromPropz, usePropz } from '../../../hooks/use-propz'
 import { filterDefined } from '../../../../common/util/array'
-import { useRecentBets } from '../../../hooks/use-bets'
-import { useRecentComments } from '../../../hooks/use-comments'
-import { LoadingIndicator } from '../../../components/loading-indicator'
 import { findActiveContracts } from '../../../components/feed/find-active-contracts'
 import { Tabs } from '../../../components/layout/tabs'
 
@@ -149,12 +143,6 @@ export default function FoldPage(props: {
   const contracts = filterDefined(
     props.contracts.map((contract) => contractsMap[contract.id])
   )
-  const activeContracts = filterDefined(
-    props.activeContracts.map((contract) => contractsMap[contract.id])
-  )
-
-  const recentBets = useRecentBets()
-  const recentComments = useRecentComments()
 
   if (fold === null || !foldSubpages.includes(page) || slugs[2]) {
     return <Custom404 />
@@ -178,37 +166,6 @@ export default function FoldPage(props: {
     </Col>
   )
 
-  const activityTab = (
-    <Col className="flex-1">
-      {user !== null && !fold.disallowMarketCreation && (
-        <FeedCreate
-          className={clsx('border-b-2')}
-          user={user}
-          tag={toCamelCase(fold.name)}
-          placeholder={`Type your question about ${fold.name}`}
-        />
-      )}
-      {recentBets && recentComments ? (
-        <>
-          <ActivityFeed
-            contracts={activeContracts}
-            recentBets={recentBets ?? []}
-            recentComments={recentComments ?? []}
-            mode="abbreviated"
-          />
-          {activeContracts.length === 0 && (
-            <div className="mx-2 mt-4 text-gray-500 lg:mx-0">
-              No activity from matching markets.{' '}
-              {isCurator && 'Try editing to add more tags!'}
-            </div>
-          )}
-        </>
-      ) : (
-        <LoadingIndicator className="mt-4" />
-      )}
-    </Col>
-  )
-
   const leaderboardsTab = (
     <Col className="gap-8 px-4 lg:flex-row">
       <FoldLeaderboards
@@ -248,13 +205,8 @@ export default function FoldPage(props: {
       </div>
 
       <Tabs
-        defaultIndex={page === 'leaderboards' ? 2 : page === 'markets' ? 1 : 0}
+        defaultIndex={page === 'leaderboards' ? 1 : 0}
         tabs={[
-          {
-            title: 'Activity',
-            content: activityTab,
-            href: foldPath(fold),
-          },
           {
             title: 'Markets',
             content: <SearchableGrid contracts={contracts} />,

web/pages/home.tsx

@@ -9,30 +9,19 @@ import { Spacer } from '../components/layout/spacer'
 import { Col } from '../components/layout/col'
 import { useUser } from '../hooks/use-user'
 import { LoadingIndicator } from '../components/loading-indicator'
-import { useRecentBets } from '../hooks/use-bets'
-import { useActiveContracts } from '../hooks/use-contracts'
-import { useRecentComments } from '../hooks/use-comments'
 import { useAlgoFeed } from '../hooks/use-algo-feed'
 import { ContractPageContent } from './[username]/[contractSlug]'
 
 const Home = () => {
   const user = useUser()
 
-  const contracts = useActiveContracts()
+  const feed = useAlgoFeed(user)
-  const contractsDict = _.keyBy(contracts, 'id')
-
-  const recentBets = useRecentBets()
-  const recentComments = useRecentComments()
-
-  const feedContracts = useAlgoFeed(user, contracts, recentBets, recentComments)
-
-  const updatedContracts = feedContracts.map(
-    (contract) => contractsDict[contract.id] ?? contract
-  )
 
   const router = useRouter()
   const { u: username, s: slug } = router.query
-  const contract = feedContracts.find((c) => c.slug === slug)
+  const contract = feed?.find(
+    ({ contract }) => contract.slug === slug
+  )?.contract
 
   useEffect(() => {
     // If the page initially loads with query params, redirect to the contract page.
@@ -54,11 +43,9 @@ const Home = () => {
           <Col className="w-full max-w-[700px]">
             <FeedCreate user={user ?? undefined} />
             <Spacer h={10} />
-            {contracts && recentBets && recentComments ? (
+            {feed ? (
               <ActivityFeed
-                contracts={updatedContracts}
+                feed={feed}
-                recentBets={recentBets}
-                recentComments={recentComments}
                 mode="only-recent"
                 getContractPath={(c) =>
                   `home?u=${c.creatorUsername}&s=${c.slug}`