From 06b7e49e98184b78ad2cc65dc4fd7571988d2e35 Mon Sep 17 00:00:00 2001 From: James Grugett Date: Sun, 1 May 2022 11:36:54 -0500 Subject: [PATCH] [In progress] Server-side feed computation (#106) * Store view counts & last viewed time * Schedule updating user recommendations. Compute using tf-idf. * Update contract's lastBetTime and lastCommentTime on new bets and comments. * Remove contract's lastUpdatedTime * Remove folds activity feed * Implement getFeed cloud function * Hook up client to use getFeed * Script to cache viewCounts and lastViewTime * Batched wait all userRecommendations * Cache view script runs on all users * Update user feed each hour and get feed from cache doc. * Delete view cache script * Update feed script * Tweak feed algorithm * Compute recommendation scores from updateUserFeed * Disable lastViewedScore factor * Update lastCommentTime script * Comment out console.log * Fix timeout issue by calling new cloud functions with part of the work. * Listen for contract updates to feed. * Handle new user: use default feed of top markets this week * Track lastUpdatedTime * Tweak logic of calling cloud functions in batches * Tweak cloud function batching --- common/contract.ts | 4 +- common/new-contract.ts | 1 - common/recommended-contracts.ts | 89 +++++++ firestore.rules | 4 + functions/src/index.ts | 4 + functions/src/on-create-bet.ts | 28 +++ functions/src/on-create-comment.ts | 11 +- functions/src/on-view.ts | 24 ++ functions/src/scripts/cache-views.ts | 78 ++++++ functions/src/scripts/update-feed.ts | 38 +++ .../src/scripts/update-last-comment-time.ts | 43 ++++ functions/src/update-feed.ts | 210 ++++++++++++++++ functions/src/update-recommendations.ts | 71 ++++++ functions/src/utils.ts | 14 +- web/components/feed/activity-feed.tsx | 44 ++-- web/components/feed/find-active-contracts.ts | 6 +- web/hooks/use-algo-feed.ts | 231 +++--------------- web/hooks/use-contracts.ts | 36 ++- web/lib/firebase/contracts.ts | 43 ++++ web/lib/firebase/users.ts | 17 +- web/pages/fold/[...slugs]/index.tsx | 52 +--- web/pages/home.tsx | 25 +- 22 files changed, 777 insertions(+), 296 deletions(-) create mode 100644 functions/src/on-create-bet.ts create mode 100644 functions/src/on-view.ts create mode 100644 functions/src/scripts/cache-views.ts create mode 100644 functions/src/scripts/update-feed.ts create mode 100644 functions/src/scripts/update-last-comment-time.ts create mode 100644 functions/src/update-feed.ts create mode 100644 functions/src/update-recommendations.ts diff --git a/common/contract.ts b/common/contract.ts index 6e362de0..82a330b5 100644 --- a/common/contract.ts +++ b/common/contract.ts @@ -20,7 +20,9 @@ export type FullContract< visibility: 'public' | 'unlisted' createdTime: number // Milliseconds since epoch - lastUpdatedTime: number // If the question or description was changed + lastUpdatedTime?: number // Updated on new bet or comment + lastBetTime?: number + lastCommentTime?: number closeTime?: number // When no more trading is allowed isResolved: boolean diff --git a/common/new-contract.ts b/common/new-contract.ts index ffd27e3f..b86ebb71 100644 --- a/common/new-contract.ts +++ b/common/new-contract.ts @@ -53,7 +53,6 @@ export function getNewContract( visibility: 'public', isResolved: false, createdTime: Date.now(), - lastUpdatedTime: Date.now(), closeTime, volume, diff --git a/common/recommended-contracts.ts b/common/recommended-contracts.ts index 7de2e501..be50c5cd 100644 --- a/common/recommended-contracts.ts +++ b/common/recommended-contracts.ts @@ -1,8 +1,12 @@ import * as _ from 'lodash' +import { Bet } from './bet' import { Contract } from './contract' +import { ClickEvent } from './tracking' import { filterDefined } from './util/array' import { addObjects } from './util/object' +export const MAX_FEED_CONTRACTS = 75 + export const getRecommendedContracts = ( contractsById: { [contractId: string]: Contract }, yourBetOnContractIds: string[] @@ -92,3 +96,88 @@ const contractsToWordFrequency = (contracts: Contract[]) => { return toFrequency(frequencySum) } + +export const getWordScores = ( + contracts: Contract[], + contractViewCounts: { [contractId: string]: number }, + clicks: ClickEvent[], + bets: Bet[] +) => { + const contractClicks = _.groupBy(clicks, (click) => click.contractId) + const contractBets = _.groupBy(bets, (bet) => bet.contractId) + + const yourContracts = contracts.filter( + (c) => + contractViewCounts[c.id] || contractClicks[c.id] || contractBets[c.id] + ) + const yourTfIdf = calculateContractTfIdf(yourContracts) + + const contractWordScores = _.mapValues( + yourTfIdf, + (wordsTfIdf, contractId) => { + const viewCount = contractViewCounts[contractId] ?? 0 + const clickCount = contractClicks[contractId]?.length ?? 0 + const betCount = contractBets[contractId]?.length ?? 0 + + const factor = + -1 * Math.log(viewCount + 1) + + 10 * Math.log(betCount + clickCount / 4 + 1) + + return _.mapValues(wordsTfIdf, (tfIdf) => tfIdf * factor) + } + ) + + const wordScores = Object.values(contractWordScores).reduce(addObjects, {}) + const minScore = Math.min(...Object.values(wordScores)) + const maxScore = Math.max(...Object.values(wordScores)) + const normalizedWordScores = _.mapValues( + wordScores, + (score) => (score - minScore) / (maxScore - minScore) + ) + + // console.log( + // 'your word scores', + // _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(0, 100), + // _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(-100) + // ) + + return normalizedWordScores +} + +export function getContractScore( + contract: Contract, + wordScores: { [word: string]: number } +) { + if (Object.keys(wordScores).length === 0) return 1 + + const wordFrequency = contractToWordFrequency(contract) + const score = _.sumBy(Object.keys(wordFrequency), (word) => { + const wordFreq = wordFrequency[word] ?? 0 + const weight = wordScores[word] ?? 0 + return wordFreq * weight + }) + + return score +} + +// Caluculate Term Frequency-Inverse Document Frequency (TF-IDF): +// https://medium.datadriveninvestor.com/tf-idf-in-natural-language-processing-8db8ef4a7736 +function calculateContractTfIdf(contracts: Contract[]) { + const contractFreq = contracts.map((c) => contractToWordFrequency(c)) + const contractWords = contractFreq.map((freq) => Object.keys(freq)) + + const wordsCount: { [word: string]: number } = {} + for (const words of contractWords) { + for (const word of words) { + wordsCount[word] = (wordsCount[word] ?? 0) + 1 + } + } + + const wordIdf = _.mapValues(wordsCount, (count) => + Math.log(contracts.length / count) + ) + const contractWordsTfIdf = _.map(contractFreq, (wordFreq) => + _.mapValues(wordFreq, (freq, word) => freq * wordIdf[word]) + ) + return _.fromPairs(contracts.map((c, i) => [c.id, contractWordsTfIdf[i]])) +} diff --git a/firestore.rules b/firestore.rules index 65783ba1..28e03e64 100644 --- a/firestore.rules +++ b/firestore.rules @@ -35,6 +35,10 @@ service cloud.firestore { allow create: if userId == request.auth.uid; } + match /private-users/{userId}/cache/feed { + allow read: if userId == request.auth.uid || isAdmin(); + } + match /contracts/{contractId} { allow read; allow update: if request.resource.data.diff(resource.data).affectedKeys() diff --git a/functions/src/index.ts b/functions/src/index.ts index f8aa50e3..3c0dc8f8 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -13,12 +13,16 @@ export * from './create-contract' export * from './create-user' export * from './create-fold' export * from './create-answer' +export * from './on-create-bet' export * from './on-create-comment' export * from './on-fold-follow' export * from './on-fold-delete' +export * from './on-view' export * from './unsubscribe' export * from './update-contract-metrics' export * from './update-user-metrics' +export * from './update-recommendations' +export * from './update-feed' export * from './backup-db' export * from './change-user-info' export * from './market-close-emails' diff --git a/functions/src/on-create-bet.ts b/functions/src/on-create-bet.ts new file mode 100644 index 00000000..deaa4c4a --- /dev/null +++ b/functions/src/on-create-bet.ts @@ -0,0 +1,28 @@ +import * as functions from 'firebase-functions' +import * as admin from 'firebase-admin' +import * as _ from 'lodash' + +import { getContract } from './utils' +import { Bet } from '../../common/bet' + +const firestore = admin.firestore() + +export const onCreateBet = functions.firestore + .document('contracts/{contractId}/bets/{betId}') + .onCreate(async (change, context) => { + const { contractId } = context.params as { + contractId: string + } + + const contract = await getContract(contractId) + if (!contract) + throw new Error('Could not find contract corresponding with bet') + + const bet = change.data() as Bet + const lastBetTime = bet.createdTime + + await firestore + .collection('contracts') + .doc(contract.id) + .update({ lastBetTime, lastUpdatedTime: Date.now() }) + }) diff --git a/functions/src/on-create-comment.ts b/functions/src/on-create-comment.ts index 02ade1fe..18fc6757 100644 --- a/functions/src/on-create-comment.ts +++ b/functions/src/on-create-comment.ts @@ -18,12 +18,19 @@ export const onCreateComment = functions.firestore } const contract = await getContract(contractId) - if (!contract) return + if (!contract) + throw new Error('Could not find contract corresponding with comment') const comment = change.data() as Comment + const lastCommentTime = comment.createdTime const commentCreator = await getUser(comment.userId) - if (!commentCreator) return + if (!commentCreator) throw new Error('Could not find contract creator') + + await firestore + .collection('contracts') + .doc(contract.id) + .update({ lastCommentTime, lastUpdatedTime: Date.now() }) let bet: Bet | undefined let answer: Answer | undefined diff --git a/functions/src/on-view.ts b/functions/src/on-view.ts new file mode 100644 index 00000000..d2f746d5 --- /dev/null +++ b/functions/src/on-view.ts @@ -0,0 +1,24 @@ +import * as functions from 'firebase-functions' +import * as admin from 'firebase-admin' +import { View } from '../../common/tracking' + +const firestore = admin.firestore() + +export const onView = functions.firestore + .document('private-users/{userId}/views/{viewId}') + .onCreate(async (snapshot, context) => { + const { userId } = context.params + + const { contractId, timestamp } = snapshot.data() as View + + await firestore + .doc(`private-users/${userId}/cache/viewCounts`) + .set( + { [contractId]: admin.firestore.FieldValue.increment(1) }, + { merge: true } + ) + + await firestore + .doc(`private-users/${userId}/cache/lastViewTime`) + .set({ [contractId]: timestamp }, { merge: true }) + }) diff --git a/functions/src/scripts/cache-views.ts b/functions/src/scripts/cache-views.ts new file mode 100644 index 00000000..c7145a1e --- /dev/null +++ b/functions/src/scripts/cache-views.ts @@ -0,0 +1,78 @@ +import * as admin from 'firebase-admin' +import * as _ from 'lodash' + +import { initAdmin } from './script-init' +initAdmin() + +import { getValues } from '../utils' +import { View } from '../../../common/tracking' +import { User } from '../../../common/user' +import { batchedWaitAll } from '../../../common/util/promise' + +const firestore = admin.firestore() + +async function cacheViews() { + console.log('Caching views') + + const users = await getValues(firestore.collection('users')) + + await batchedWaitAll( + users.map((user) => () => { + console.log('Caching views for', user.username) + return cacheUserViews(user.id) + }) + ) +} + +async function cacheUserViews(userId: string) { + const views = await getValues( + firestore.collection('private-users').doc(userId).collection('views') + ) + + const viewCounts: { [contractId: string]: number } = {} + for (const view of views) { + viewCounts[view.contractId] = (viewCounts[view.contractId] ?? 0) + 1 + } + + const lastViewTime: { [contractId: string]: number } = {} + for (const view of views) { + lastViewTime[view.contractId] = Math.max( + lastViewTime[view.contractId] ?? 0, + view.timestamp + ) + } + + await firestore + .doc(`private-users/${userId}/cache/viewCounts`) + .set(viewCounts, { merge: true }) + + await firestore + .doc(`private-users/${userId}/cache/lastViewTime`) + .set(lastViewTime, { merge: true }) + + console.log(viewCounts, lastViewTime) +} + +async function deleteCache() { + console.log('Deleting view cache') + + const users = await getValues(firestore.collection('users')) + + await batchedWaitAll( + users.map((user) => async () => { + console.log('Deleting view cache for', user.username) + await firestore.doc(`private-users/${user.id}/cache/viewCounts`).delete() + await firestore + .doc(`private-users/${user.id}/cache/lastViewTime`) + .delete() + await firestore + .doc(`private-users/${user.id}/cache/contractScores`) + .delete() + await firestore.doc(`private-users/${user.id}/cache/wordScores`).delete() + }) + ) +} + +if (require.main === module) { + cacheViews().then(() => process.exit()) +} diff --git a/functions/src/scripts/update-feed.ts b/functions/src/scripts/update-feed.ts new file mode 100644 index 00000000..25a0b14f --- /dev/null +++ b/functions/src/scripts/update-feed.ts @@ -0,0 +1,38 @@ +import * as admin from 'firebase-admin' +import * as _ from 'lodash' + +import { initAdmin } from './script-init' +initAdmin() + +import { getValues } from '../utils' +import { User } from '../../../common/user' +import { batchedWaitAll } from '../../../common/util/promise' +import { Contract } from '../../../common/contract' +import { updateWordScores } from '../update-recommendations' +import { getFeedContracts, doUserFeedUpdate } from '../update-feed' + +const firestore = admin.firestore() + +async function updateFeed() { + console.log('Updating feed') + + const contracts = await getValues(firestore.collection('contracts')) + const feedContracts = await getFeedContracts() + const users = await getValues( + firestore.collection('users') + // .where('username', '==', 'JamesGrugett') + ) + + await batchedWaitAll( + users.map((user) => async () => { + console.log('Updating recs for', user.username) + await updateWordScores(user, contracts) + console.log('Updating feed for', user.username) + await doUserFeedUpdate(user, feedContracts) + }) + ) +} + +if (require.main === module) { + updateFeed().then(() => process.exit()) +} diff --git a/functions/src/scripts/update-last-comment-time.ts b/functions/src/scripts/update-last-comment-time.ts new file mode 100644 index 00000000..ae950fbe --- /dev/null +++ b/functions/src/scripts/update-last-comment-time.ts @@ -0,0 +1,43 @@ +import * as admin from 'firebase-admin' +import * as _ from 'lodash' + +import { initAdmin } from './script-init' +initAdmin() + +import { Contract } from '../../../common/contract' +import { getValues } from '../utils' +import { Comment } from '../../../common/comment' + +async function updateLastCommentTime() { + const firestore = admin.firestore() + console.log('Updating contracts lastCommentTime') + + const contracts = await getValues(firestore.collection('contracts')) + + console.log('Loaded', contracts.length, 'contracts') + + for (const contract of contracts) { + const contractRef = firestore.doc(`contracts/${contract.id}`) + + const lastComments = await getValues( + contractRef.collection('comments').orderBy('createdTime', 'desc').limit(1) + ) + + if (lastComments.length > 0) { + const lastCommentTime = lastComments[0].createdTime + console.log( + 'Updating lastCommentTime', + contract.question, + lastCommentTime + ) + + await contractRef.update({ + lastCommentTime, + } as Partial) + } + } +} + +if (require.main === module) { + updateLastCommentTime().then(() => process.exit()) +} diff --git a/functions/src/update-feed.ts b/functions/src/update-feed.ts new file mode 100644 index 00000000..accd48e8 --- /dev/null +++ b/functions/src/update-feed.ts @@ -0,0 +1,210 @@ +import * as _ from 'lodash' +import * as functions from 'firebase-functions' +import * as admin from 'firebase-admin' + +import { getValue, getValues } from './utils' +import { Contract } from '../../common/contract' +import { logInterpolation } from '../../common/util/math' +import { DAY_MS } from '../../common/util/time' +import { + getProbability, + getOutcomeProbability, + getTopAnswer, +} from '../../common/calculate' +import { Bet } from '../../common/bet' +import { Comment } from '../../common/comment' +import { User } from '../../common/user' +import { + getContractScore, + MAX_FEED_CONTRACTS, +} from '../../common/recommended-contracts' +import { callCloudFunction } from './call-cloud-function' + +const firestore = admin.firestore() + +export const updateFeed = functions.pubsub + .schedule('every 60 minutes') + .onRun(async () => { + const users = await getValues(firestore.collection('users')) + + const batchSize = 100 + const userBatches: User[][] = [] + for (let i = 0; i < users.length; i += batchSize) { + userBatches.push(users.slice(i, i + batchSize)) + } + + await Promise.all( + userBatches.map(async (users) => + callCloudFunction('updateFeedBatch', { users }) + ) + ) + }) + +export const updateFeedBatch = functions.https.onCall( + async (data: { users: User[] }) => { + const { users } = data + const contracts = await getFeedContracts() + + await Promise.all(users.map((user) => doUserFeedUpdate(user, contracts))) + } +) + +export async function getFeedContracts() { + // Get contracts bet on or created in last week. + const contracts = await Promise.all([ + getValues( + firestore + .collection('contracts') + .where('isResolved', '==', false) + .where('volume7Days', '>', 0) + ), + + getValues( + firestore + .collection('contracts') + .where('isResolved', '==', false) + .where('createdTime', '>', Date.now() - DAY_MS * 7) + .where('volume7Days', '==', 0) + ), + ]).then(([activeContracts, inactiveContracts]) => { + const combined = [...activeContracts, ...inactiveContracts] + // Remove closed contracts. + return combined.filter((c) => (c.closeTime ?? Infinity) > Date.now()) + }) + + return contracts +} + +export const doUserFeedUpdate = async (user: User, contracts: Contract[]) => { + const userCacheCollection = firestore.collection( + `private-users/${user.id}/cache` + ) + const [wordScores, lastViewedTime] = await Promise.all([ + getValue<{ [word: string]: number }>(userCacheCollection.doc('wordScores')), + getValue<{ [contractId: string]: number }>( + userCacheCollection.doc('lastViewTime') + ), + ]).then((dicts) => dicts.map((dict) => dict ?? {})) + + const scoredContracts = contracts.map((contract) => { + const score = scoreContract( + contract, + wordScores, + lastViewedTime[contract.id] + ) + return [contract, score] as [Contract, number] + }) + + const sortedContracts = _.sortBy( + scoredContracts, + ([_, score]) => score + ).reverse() + + // console.log(sortedContracts.map(([c, score]) => c.question + ': ' + score)) + + const feedContracts = sortedContracts + .slice(0, MAX_FEED_CONTRACTS) + .map(([c]) => c) + + const feed = await Promise.all( + feedContracts.map((contract) => getRecentBetsAndComments(contract)) + ) + + await userCacheCollection.doc('feed').set({ feed }) +} + +function scoreContract( + contract: Contract, + wordScores: { [word: string]: number }, + viewTime: number | undefined +) { + const recommendationScore = getContractScore(contract, wordScores) + const activityScore = getActivityScore(contract, viewTime) + // const lastViewedScore = getLastViewedScore(viewTime) + return recommendationScore * activityScore +} + +function getActivityScore(contract: Contract, viewTime: number | undefined) { + const { createdTime, lastBetTime, lastCommentTime, outcomeType } = contract + const hasNewComments = + lastCommentTime && (!viewTime || lastCommentTime > viewTime) + const newCommentScore = hasNewComments ? 1 : 0.5 + + const timeSinceLastComment = Date.now() - (lastCommentTime ?? createdTime) + const commentDaysAgo = timeSinceLastComment / DAY_MS + const commentTimeScore = + 0.25 + 0.75 * (1 - logInterpolation(0, 3, commentDaysAgo)) + + const timeSinceLastBet = Date.now() - (lastBetTime ?? createdTime) + const betDaysAgo = timeSinceLastBet / DAY_MS + const betTimeScore = 0.5 + 0.5 * (1 - logInterpolation(0, 3, betDaysAgo)) + + let prob = 0.5 + if (outcomeType === 'BINARY') { + prob = getProbability(contract) + } else if (outcomeType === 'FREE_RESPONSE') { + const topAnswer = getTopAnswer(contract) + if (topAnswer) + prob = Math.max(0.5, getOutcomeProbability(contract, topAnswer.id)) + } + const frac = 1 - Math.abs(prob - 0.5) ** 2 / 0.25 + const probScore = 0.5 + frac * 0.5 + + const { volume24Hours, volume7Days } = contract + const combinedVolume = Math.log(volume24Hours + 1) + Math.log(volume7Days + 1) + const volumeScore = 0.5 + 0.5 * logInterpolation(4, 20, combinedVolume) + + const score = + newCommentScore * commentTimeScore * betTimeScore * probScore * volumeScore + + // Map score to [0.5, 1] since no recent activty is not a deal breaker. + const mappedScore = 0.5 + 0.5 * score + const newMappedScore = 0.7 + 0.3 * score + + const isNew = Date.now() < contract.createdTime + DAY_MS + return isNew ? newMappedScore : mappedScore +} + +function getLastViewedScore(viewTime: number | undefined) { + if (viewTime === undefined) { + return 1 + } + + const daysAgo = (Date.now() - viewTime) / DAY_MS + + if (daysAgo < 0.5) { + const frac = logInterpolation(0, 0.5, daysAgo) + return 0.5 + 0.25 * frac + } + + const frac = logInterpolation(0.5, 14, daysAgo) + return 0.75 + 0.25 * frac +} + +async function getRecentBetsAndComments(contract: Contract) { + const contractDoc = firestore.collection('contracts').doc(contract.id) + + const [recentBets, recentComments] = await Promise.all([ + getValues( + contractDoc + .collection('bets') + .where('createdTime', '>', Date.now() - DAY_MS) + .orderBy('createdTime', 'desc') + .limit(1) + ), + + getValues( + contractDoc + .collection('comments') + .where('createdTime', '>', Date.now() - 3 * DAY_MS) + .orderBy('createdTime', 'desc') + .limit(3) + ), + ]) + + return { + contract, + recentBets, + recentComments, + } +} diff --git a/functions/src/update-recommendations.ts b/functions/src/update-recommendations.ts new file mode 100644 index 00000000..4e656dda --- /dev/null +++ b/functions/src/update-recommendations.ts @@ -0,0 +1,71 @@ +import * as functions from 'firebase-functions' +import * as admin from 'firebase-admin' +import * as _ from 'lodash' + +import { getValue, getValues } from './utils' +import { Contract } from '../../common/contract' +import { Bet } from '../../common/bet' +import { User } from '../../common/user' +import { ClickEvent } from '../../common/tracking' +import { getWordScores } from '../../common/recommended-contracts' +import { batchedWaitAll } from '../../common/util/promise' +import { callCloudFunction } from './call-cloud-function' + +const firestore = admin.firestore() + +export const updateRecommendations = functions.pubsub + .schedule('every 24 hours') + .onRun(async () => { + const users = await getValues(firestore.collection('users')) + + const batchSize = 100 + const userBatches: User[][] = [] + for (let i = 0; i < users.length; i += batchSize) { + userBatches.push(users.slice(i, i + batchSize)) + } + + await Promise.all( + userBatches.map((batch) => + callCloudFunction('updateRecommendationsBatch', { users: batch }) + ) + ) + }) + +export const updateRecommendationsBatch = functions.https.onCall( + async (data: { users: User[] }) => { + const { users } = data + + const contracts = await getValues( + firestore.collection('contracts') + ) + + await batchedWaitAll( + users.map((user) => () => updateWordScores(user, contracts)) + ) + } +) + +export const updateWordScores = async (user: User, contracts: Contract[]) => { + const [bets, viewCounts, clicks] = await Promise.all([ + getValues( + firestore.collectionGroup('bets').where('userId', '==', user.id) + ), + + getValue<{ [contractId: string]: number }>( + firestore.doc(`private-users/${user.id}/cache/viewCounts`) + ), + + getValues( + firestore + .collection(`private-users/${user.id}/events`) + .where('type', '==', 'click') + ), + ]) + + const wordScores = getWordScores(contracts, viewCounts ?? {}, clicks, bets) + + const cachedCollection = firestore.collection( + `private-users/${user.id}/cache` + ) + await cachedCollection.doc('wordScores').set(wordScores) +} diff --git a/functions/src/utils.ts b/functions/src/utils.ts index 88c25570..28ef5445 100644 --- a/functions/src/utils.ts +++ b/functions/src/utils.ts @@ -6,27 +6,33 @@ import { PrivateUser, User } from '../../common/user' export const isProd = admin.instanceId().app.options.projectId === 'mantic-markets' -export const getValue = async (collection: string, doc: string) => { +export const getDoc = async (collection: string, doc: string) => { const snap = await admin.firestore().collection(collection).doc(doc).get() return snap.exists ? (snap.data() as T) : undefined } +export const getValue = async (ref: admin.firestore.DocumentReference) => { + const snap = await ref.get() + + return snap.exists ? (snap.data() as T) : undefined +} + export const getValues = async (query: admin.firestore.Query) => { const snap = await query.get() return snap.docs.map((doc) => doc.data() as T) } export const getContract = (contractId: string) => { - return getValue('contracts', contractId) + return getDoc('contracts', contractId) } export const getUser = (userId: string) => { - return getValue('users', userId) + return getDoc('users', userId) } export const getPrivateUser = (userId: string) => { - return getValue('private-users', userId) + return getDoc('private-users', userId) } export const getUserByUsername = async (username: string) => { diff --git a/web/components/feed/activity-feed.tsx b/web/components/feed/activity-feed.tsx index d7e3ab99..19ec1299 100644 --- a/web/components/feed/activity-feed.tsx +++ b/web/components/feed/activity-feed.tsx @@ -8,31 +8,27 @@ import { useUser } from '../../hooks/use-user' import { ContractActivity } from './contract-activity' export function ActivityFeed(props: { - contracts: Contract[] - recentBets: Bet[] - recentComments: Comment[] + feed: { + contract: Contract + recentBets: Bet[] + recentComments: Comment[] + }[] mode: 'only-recent' | 'abbreviated' | 'all' getContractPath?: (contract: Contract) => string }) { - const { contracts, recentBets, recentComments, mode, getContractPath } = props + const { feed, mode, getContractPath } = props const user = useUser() - const groupedBets = _.groupBy(recentBets, (bet) => bet.contractId) - const groupedComments = _.groupBy( - recentComments, - (comment) => comment.contractId - ) - return ( ( + feed={feed} + renderItem={({ contract, recentBets, recentComments }) => ( @@ -42,18 +38,26 @@ export function ActivityFeed(props: { } function FeedContainer(props: { - contracts: Contract[] - renderContract: (contract: Contract) => any + feed: { + contract: Contract + recentBets: Bet[] + recentComments: Comment[] + }[] + renderItem: (item: { + contract: Contract + recentBets: Bet[] + recentComments: Comment[] + }) => any }) { - const { contracts, renderContract } = props + const { feed, renderItem } = props return ( - {contracts.map((contract) => ( -
- {renderContract(contract)} + {feed.map((item) => ( +
+ {renderItem(item)}
))} diff --git a/web/components/feed/find-active-contracts.ts b/web/components/feed/find-active-contracts.ts index 42737f47..6f40806f 100644 --- a/web/components/feed/find-active-contracts.ts +++ b/web/components/feed/find-active-contracts.ts @@ -9,11 +9,7 @@ const MAX_ACTIVE_CONTRACTS = 75 // TODO: Maybe store last activity time directly in the contract? // Pros: simplifies this code; cons: harder to tweak "activity" definition later function lastActivityTime(contract: Contract) { - return Math.max( - contract.resolutionTime || 0, - contract.lastUpdatedTime, - contract.createdTime - ) + return Math.max(contract.resolutionTime || 0, contract.createdTime) } // Types of activity to surface: diff --git a/web/hooks/use-algo-feed.ts b/web/hooks/use-algo-feed.ts index e8d6396b..b8cfb7a2 100644 --- a/web/hooks/use-algo-feed.ts +++ b/web/hooks/use-algo-feed.ts @@ -1,213 +1,60 @@ import _ from 'lodash' -import { useState, useEffect, useMemo } from 'react' +import { useState, useEffect } from 'react' import { Bet } from '../../common/bet' import { Comment } from '../../common/comment' import { Contract } from '../../common/contract' -import { User } from '../../common/user' -import { logInterpolation } from '../../common/util/math' -import { getRecommendedContracts } from '../../common/recommended-contracts' -import { useSeenContracts } from './use-seen-contracts' -import { useGetUserBetContractIds, useUserBetContracts } from './use-user-bets' -import { DAY_MS } from '../../common/util/time' -import { - getProbability, - getOutcomeProbability, - getTopAnswer, -} from '../../common/calculate' import { useTimeSinceFirstRender } from './use-time-since-first-render' import { trackLatency } from '../lib/firebase/tracking' +import { User } from '../../common/user' +import { getUserFeed } from '../lib/firebase/users' +import { useUpdatedContracts } from './use-contracts' +import { + getRecentBetsAndComments, + getTopWeeklyContracts, +} from '../lib/firebase/contracts' -const MAX_FEED_CONTRACTS = 75 +type feed = { + contract: Contract + recentBets: Bet[] + recentComments: Comment[] +}[] -export const useAlgoFeed = ( - user: User | null | undefined, - contracts: Contract[] | undefined, - recentBets: Bet[] | undefined, - recentComments: Comment[] | undefined -) => { - const initialContracts = useMemo(() => contracts, [!!contracts]) - const initialBets = useMemo(() => recentBets, [!!recentBets]) - const initialComments = useMemo(() => recentComments, [!!recentComments]) - - const yourBetContractIds = useGetUserBetContractIds(user?.id) - // Update user bet contracts in local storage. - useUserBetContracts(user?.id) - - const seenContracts = useSeenContracts() - - const [algoFeed, setAlgoFeed] = useState([]) +export const useAlgoFeed = (user: User | null | undefined) => { + const [feed, setFeed] = useState() const getTime = useTimeSinceFirstRender() useEffect(() => { - if ( - initialContracts && - initialBets && - initialComments && - yourBetContractIds - ) { - const eligibleContracts = initialContracts.filter( - (c) => !c.isResolved && (c.closeTime ?? Infinity) > Date.now() - ) - const contracts = getAlgoFeed( - eligibleContracts, - initialBets, - initialComments, - yourBetContractIds, - seenContracts - ) - setAlgoFeed(contracts) - trackLatency('feed', getTime()) + if (user) { + getUserFeed(user.id).then((feed) => { + if (feed.length === 0) { + getDefaultFeed().then((feed) => setFeed(feed)) + } else setFeed(feed) + + trackLatency('feed', getTime()) + console.log('feed load time', getTime()) + }) } - }, [ - initialBets, - initialComments, - initialContracts, - seenContracts, - yourBetContractIds, - getTime, - ]) + }, [user?.id]) - return algoFeed + return useUpdateFeed(feed) } -const getAlgoFeed = ( - contracts: Contract[], - recentBets: Bet[], - recentComments: Comment[], - yourBetContractIds: string[], - seenContracts: { [contractId: string]: number } -) => { - const contractsById = _.keyBy(contracts, (c) => c.id) +const useUpdateFeed = (feed: feed | undefined) => { + const contracts = useUpdatedContracts(feed?.map((item) => item.contract)) - const recommended = getRecommendedContracts(contractsById, yourBetContractIds) - const confidence = logInterpolation(0, 100, yourBetContractIds.length) - const recommendedScores = _.fromPairs( - recommended.map((c, index) => { - const score = 1 - index / recommended.length - const withConfidence = score * confidence + (1 - confidence) - return [c.id, withConfidence] as [string, number] - }) - ) - - const seenScores = _.fromPairs( - contracts.map( - (c) => [c.id, getSeenContractsScore(c, seenContracts)] as [string, number] - ) - ) - - const activityScores = getContractsActivityScores( - contracts, - recentComments, - recentBets, - seenContracts - ) - - const combinedScores = contracts.map((contract) => { - const score = - (recommendedScores[contract.id] ?? 0) * - (seenScores[contract.id] ?? 0) * - (activityScores[contract.id] ?? 0) - return { contract, score } - }) - - const sorted = _.sortBy(combinedScores, (c) => -c.score) - return sorted.map((c) => c.contract).slice(0, MAX_FEED_CONTRACTS) + return feed && contracts + ? feed.map(({ contract, ...other }, i) => ({ + ...other, + contract: contracts[i], + })) + : undefined } -function getContractsActivityScores( - contracts: Contract[], - recentComments: Comment[], - recentBets: Bet[], - seenContracts: { [contractId: string]: number } -) { - const contractBets = _.groupBy(recentBets, (bet) => bet.contractId) - const contractMostRecentBet = _.mapValues( - contractBets, - (bets) => _.maxBy(bets, (bet) => bet.createdTime) as Bet +const getDefaultFeed = async () => { + const contracts = await getTopWeeklyContracts() + const feed = await Promise.all( + contracts.map((c) => getRecentBetsAndComments(c)) ) - - const contractComments = _.groupBy( - recentComments, - (comment) => comment.contractId - ) - const contractMostRecentComment = _.mapValues( - contractComments, - (comments) => _.maxBy(comments, (c) => c.createdTime) as Comment - ) - - const scoredContracts = contracts.map((contract) => { - const { outcomeType } = contract - - const seenTime = seenContracts[contract.id] - const lastCommentTime = contractMostRecentComment[contract.id]?.createdTime - const hasNewComments = - !seenTime || (lastCommentTime && lastCommentTime > seenTime) - const newCommentScore = hasNewComments ? 1 : 0.5 - - const commentCount = contractComments[contract.id]?.length ?? 0 - const betCount = contractBets[contract.id]?.length ?? 0 - const activtyCount = betCount + commentCount * 5 - const activityCountScore = - 0.5 + 0.5 * logInterpolation(0, 200, activtyCount) - - const { volume7Days, volume } = contract - const combinedVolume = Math.log(volume7Days + 1) + Math.log(volume + 1) - const volumeScore = 0.5 + 0.5 * logInterpolation(4, 25, combinedVolume) - - const lastBetTime = - contractMostRecentBet[contract.id]?.createdTime ?? contract.createdTime - const timeSinceLastBet = Date.now() - lastBetTime - const daysAgo = timeSinceLastBet / DAY_MS - const timeAgoScore = 1 - logInterpolation(0, 3, daysAgo) - - let prob = 0.5 - if (outcomeType === 'BINARY') { - prob = getProbability(contract) - } else if (outcomeType === 'FREE_RESPONSE') { - const topAnswer = getTopAnswer(contract) - if (topAnswer) - prob = Math.max(0.5, getOutcomeProbability(contract, topAnswer.id)) - } - const frac = 1 - Math.abs(prob - 0.5) ** 2 / 0.25 - const probScore = 0.5 + frac * 0.5 - - const score = - newCommentScore * - activityCountScore * - volumeScore * - timeAgoScore * - probScore - - // Map score to [0.5, 1] since no recent activty is not a deal breaker. - const mappedScore = 0.5 + score / 2 - const newMappedScore = 0.75 + score / 4 - - const isNew = Date.now() < contract.createdTime + DAY_MS - const activityScore = isNew ? newMappedScore : mappedScore - - return [contract.id, activityScore] as [string, number] - }) - - return _.fromPairs(scoredContracts) -} - -function getSeenContractsScore( - contract: Contract, - seenContracts: { [contractId: string]: number } -) { - const lastSeen = seenContracts[contract.id] - if (lastSeen === undefined) { - return 1 - } - - const daysAgo = (Date.now() - lastSeen) / DAY_MS - - if (daysAgo < 0.5) { - const frac = logInterpolation(0, 0.5, daysAgo) - return 0.5 * frac - } - - const frac = logInterpolation(0.5, 14, daysAgo) - return 0.5 + 0.5 * frac + return feed } diff --git a/web/hooks/use-contracts.ts b/web/hooks/use-contracts.ts index c6d2be0e..0402613f 100644 --- a/web/hooks/use-contracts.ts +++ b/web/hooks/use-contracts.ts @@ -1,8 +1,9 @@ import _ from 'lodash' -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { Contract, listenForActiveContracts, + listenForContract, listenForContracts, listenForHotContracts, listenForInactiveContracts, @@ -71,3 +72,36 @@ export const useHotContracts = () => { return hotContracts } + +export const useUpdatedContracts = (contracts: Contract[] | undefined) => { + const [__, triggerUpdate] = useState(0) + const contractDict = useRef<{ [id: string]: Contract }>({}) + + useEffect(() => { + if (contracts === undefined) return + + contractDict.current = _.fromPairs(contracts.map((c) => [c.id, c])) + + const disposes = contracts.map((contract) => { + const { id } = contract + + return listenForContract(id, (contract) => { + const curr = contractDict.current[id] + if (!_.isEqual(curr, contract)) { + contractDict.current[id] = contract as Contract + triggerUpdate((n) => n + 1) + } + }) + }) + + triggerUpdate((n) => n + 1) + + return () => { + disposes.forEach((dispose) => dispose()) + } + }, [!!contracts]) + + return contracts && Object.keys(contractDict.current).length > 0 + ? contracts.map((c) => contractDict.current[c.id]) + : undefined +} diff --git a/web/lib/firebase/contracts.ts b/web/lib/firebase/contracts.ts index f41d6902..f1ab7bba 100644 --- a/web/lib/firebase/contracts.ts +++ b/web/lib/firebase/contracts.ts @@ -23,6 +23,9 @@ import { createRNG, shuffle } from '../../../common/util/random' import { getCpmmProbability } from '../../../common/calculate-cpmm' import { formatMoney, formatPercent } from '../../../common/util/format' import { DAY_MS } from '../../../common/util/time' +import { MAX_FEED_CONTRACTS } from '../../../common/recommended-contracts' +import { Bet } from '../../../common/bet' +import { Comment } from '../../../common/comment' export type { Contract } export function contractPath(contract: Contract) { @@ -231,6 +234,16 @@ export async function getHotContracts() { ) } +const topWeeklyQuery = query( + contractCollection, + where('isResolved', '==', false), + orderBy('volume7Days', 'desc'), + limit(MAX_FEED_CONTRACTS) +) +export async function getTopWeeklyContracts() { + return await getValues(topWeeklyQuery) +} + const closingSoonQuery = query( contractCollection, where('isResolved', '==', false), @@ -276,3 +289,33 @@ export async function getDailyContracts( return contractsByDay } + +export async function getRecentBetsAndComments(contract: Contract) { + const contractDoc = doc(db, 'contracts', contract.id) + + const [recentBets, recentComments] = await Promise.all([ + getValues( + query( + collection(contractDoc, 'bets'), + where('createdTime', '>', Date.now() - DAY_MS), + orderBy('createdTime', 'desc'), + limit(1) + ) + ), + + getValues( + query( + collection(contractDoc, 'comments'), + where('createdTime', '>', Date.now() - 3 * DAY_MS), + orderBy('createdTime', 'desc'), + limit(3) + ) + ), + ]) + + return { + contract, + recentBets, + recentComments, + } +} diff --git a/web/lib/firebase/users.ts b/web/lib/firebase/users.ts index e7805626..e07d138c 100644 --- a/web/lib/firebase/users.ts +++ b/web/lib/firebase/users.ts @@ -23,8 +23,11 @@ import _ from 'lodash' import { app } from './init' import { PrivateUser, User } from '../../../common/user' import { createUser } from './api-call' -import { getValues, listenForValue, listenForValues } from './utils' +import { getValue, getValues, listenForValue, listenForValues } from './utils' import { DAY_MS } from '../../../common/util/time' +import { Contract } from './contracts' +import { Bet } from './bets' +import { Comment } from './comments' export type { User } @@ -207,3 +210,15 @@ export async function getDailyNewUsers( return usersByDay } + +export async function getUserFeed(userId: string) { + const feedDoc = doc(db, 'private-users', userId, 'cache', 'feed') + const userFeed = await getValue<{ + feed: { + contract: Contract + recentBets: Bet[] + recentComments: Comment[] + }[] + }>(feedDoc) + return userFeed?.feed ?? [] +} diff --git a/web/pages/fold/[...slugs]/index.tsx b/web/pages/fold/[...slugs]/index.tsx index 79cd6340..b0d52793 100644 --- a/web/pages/fold/[...slugs]/index.tsx +++ b/web/pages/fold/[...slugs]/index.tsx @@ -1,5 +1,4 @@ import _ from 'lodash' -import Link from 'next/link' import { Fold } from '../../../../common/fold' import { Comment } from '../../../../common/comment' @@ -23,22 +22,17 @@ import { useUser } from '../../../hooks/use-user' import { useFold } from '../../../hooks/use-fold' import { SearchableGrid } from '../../../components/contract/contracts-list' import { useRouter } from 'next/router' -import clsx from 'clsx' import { scoreCreators, scoreTraders } from '../../../../common/scoring' import { Leaderboard } from '../../../components/leaderboard' -import { formatMoney, toCamelCase } from '../../../../common/util/format' +import { formatMoney } from '../../../../common/util/format' import { EditFoldButton } from '../../../components/folds/edit-fold-button' import Custom404 from '../../404' import { FollowFoldButton } from '../../../components/folds/follow-fold-button' -import FeedCreate from '../../../components/feed-create' import { SEO } from '../../../components/SEO' import { useTaggedContracts } from '../../../hooks/use-contracts' import { Linkify } from '../../../components/linkify' import { fromPropz, usePropz } from '../../../hooks/use-propz' import { filterDefined } from '../../../../common/util/array' -import { useRecentBets } from '../../../hooks/use-bets' -import { useRecentComments } from '../../../hooks/use-comments' -import { LoadingIndicator } from '../../../components/loading-indicator' import { findActiveContracts } from '../../../components/feed/find-active-contracts' import { Tabs } from '../../../components/layout/tabs' @@ -149,12 +143,6 @@ export default function FoldPage(props: { const contracts = filterDefined( props.contracts.map((contract) => contractsMap[contract.id]) ) - const activeContracts = filterDefined( - props.activeContracts.map((contract) => contractsMap[contract.id]) - ) - - const recentBets = useRecentBets() - const recentComments = useRecentComments() if (fold === null || !foldSubpages.includes(page) || slugs[2]) { return @@ -178,37 +166,6 @@ export default function FoldPage(props: { ) - const activityTab = ( - - {user !== null && !fold.disallowMarketCreation && ( - - )} - {recentBets && recentComments ? ( - <> - - {activeContracts.length === 0 && ( -
- No activity from matching markets.{' '} - {isCurator && 'Try editing to add more tags!'} -
- )} - - ) : ( - - )} - - ) - const leaderboardsTab = ( , diff --git a/web/pages/home.tsx b/web/pages/home.tsx index d593935c..6f1ec93c 100644 --- a/web/pages/home.tsx +++ b/web/pages/home.tsx @@ -9,30 +9,19 @@ import { Spacer } from '../components/layout/spacer' import { Col } from '../components/layout/col' import { useUser } from '../hooks/use-user' import { LoadingIndicator } from '../components/loading-indicator' -import { useRecentBets } from '../hooks/use-bets' -import { useActiveContracts } from '../hooks/use-contracts' -import { useRecentComments } from '../hooks/use-comments' import { useAlgoFeed } from '../hooks/use-algo-feed' import { ContractPageContent } from './[username]/[contractSlug]' const Home = () => { const user = useUser() - const contracts = useActiveContracts() - const contractsDict = _.keyBy(contracts, 'id') - - const recentBets = useRecentBets() - const recentComments = useRecentComments() - - const feedContracts = useAlgoFeed(user, contracts, recentBets, recentComments) - - const updatedContracts = feedContracts.map( - (contract) => contractsDict[contract.id] ?? contract - ) + const feed = useAlgoFeed(user) const router = useRouter() const { u: username, s: slug } = router.query - const contract = feedContracts.find((c) => c.slug === slug) + const contract = feed?.find( + ({ contract }) => contract.slug === slug + )?.contract useEffect(() => { // If the page initially loads with query params, redirect to the contract page. @@ -54,11 +43,9 @@ const Home = () => { - {contracts && recentBets && recentComments ? ( + {feed ? ( `home?u=${c.creatorUsername}&s=${c.slug}`