From 6b2ba35046851256017dd51ae807b2ae0418e316 Mon Sep 17 00:00:00 2001 From: Marshall Polaris Date: Wed, 17 Aug 2022 14:26:07 -0700 Subject: [PATCH] Kill some old algo feed kind of code --- common/recommended-contracts.ts | 187 -------------------------------- web/hooks/use-algo-feed.ts | 51 --------- web/lib/firebase/contracts.ts | 11 -- web/lib/firebase/users.ts | 30 +---- 4 files changed, 1 insertion(+), 278 deletions(-) delete mode 100644 common/recommended-contracts.ts delete mode 100644 web/hooks/use-algo-feed.ts diff --git a/common/recommended-contracts.ts b/common/recommended-contracts.ts deleted file mode 100644 index 3a6eca38..00000000 --- a/common/recommended-contracts.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { union, sum, sumBy, sortBy, groupBy, mapValues } from 'lodash' -import { Bet } from './bet' -import { Contract } from './contract' -import { ClickEvent } from './tracking' -import { filterDefined } from './util/array' -import { addObjects } from './util/object' - -export const MAX_FEED_CONTRACTS = 75 - -export const getRecommendedContracts = ( - contractsById: { [contractId: string]: Contract }, - yourBetOnContractIds: string[] -) => { - const contracts = Object.values(contractsById) - const yourContracts = filterDefined( - yourBetOnContractIds.map((contractId) => contractsById[contractId]) - ) - - const yourContractIds = new Set(yourContracts.map((c) => c.id)) - const notYourContracts = contracts.filter((c) => !yourContractIds.has(c.id)) - - const yourWordFrequency = contractsToWordFrequency(yourContracts) - const otherWordFrequency = contractsToWordFrequency(notYourContracts) - const words = union( - Object.keys(yourWordFrequency), - Object.keys(otherWordFrequency) - ) - - const yourWeightedFrequency = Object.fromEntries( - words.map((word) => { - const [yourFreq, otherFreq] = [ - yourWordFrequency[word] ?? 0, - otherWordFrequency[word] ?? 0, - ] - - const score = yourFreq / (yourFreq + otherFreq + 0.0001) - - return [word, score] - }) - ) - - // console.log( - // 'your weighted frequency', - // _.sortBy(_.toPairs(yourWeightedFrequency), ([, freq]) => -freq) - // ) - - const scoredContracts = contracts.map((contract) => { - const wordFrequency = contractToWordFrequency(contract) - - const score = sumBy(Object.keys(wordFrequency), (word) => { - const wordFreq = wordFrequency[word] ?? 0 - const weight = yourWeightedFrequency[word] ?? 0 - return wordFreq * weight - }) - - return { - contract, - score, - } - }) - - return sortBy(scoredContracts, (scored) => -scored.score).map( - (scored) => scored.contract - ) -} - -const contractToText = (contract: Contract) => { - const { description, question, tags, creatorUsername } = contract - return `${creatorUsername} ${question} ${tags.join(' ')} ${description}` -} - -const MAX_CHARS_IN_WORD = 100 - -const getWordsCount = (text: string) => { - const normalizedText = text.replace(/[^a-zA-Z]/g, ' ').toLowerCase() - const words = normalizedText - .split(' ') - .filter((word) => word) - .filter((word) => word.length <= MAX_CHARS_IN_WORD) - - const counts: { [word: string]: number } = {} - for (const word of words) { - if (counts[word]) counts[word]++ - else counts[word] = 1 - } - return counts -} - -const toFrequency = (counts: { [word: string]: number }) => { - const total = sum(Object.values(counts)) - return mapValues(counts, (count) => count / total) -} - -const contractToWordFrequency = (contract: Contract) => - toFrequency(getWordsCount(contractToText(contract))) - -const contractsToWordFrequency = (contracts: Contract[]) => { - const frequencySum = contracts - .map(contractToWordFrequency) - .reduce(addObjects, {}) - - return toFrequency(frequencySum) -} - -export const getWordScores = ( - contracts: Contract[], - contractViewCounts: { [contractId: string]: number }, - clicks: ClickEvent[], - bets: Bet[] -) => { - const contractClicks = groupBy(clicks, (click) => click.contractId) - const contractBets = groupBy(bets, (bet) => bet.contractId) - - const yourContracts = contracts.filter( - (c) => - contractViewCounts[c.id] || contractClicks[c.id] || contractBets[c.id] - ) - const yourTfIdf = calculateContractTfIdf(yourContracts) - - const contractWordScores = mapValues(yourTfIdf, (wordsTfIdf, contractId) => { - const viewCount = contractViewCounts[contractId] ?? 0 - const clickCount = contractClicks[contractId]?.length ?? 0 - const betCount = contractBets[contractId]?.length ?? 0 - - const factor = - -1 * Math.log(viewCount + 1) + - 10 * Math.log(betCount + clickCount / 4 + 1) - - return mapValues(wordsTfIdf, (tfIdf) => tfIdf * factor) - }) - - const wordScores = Object.values(contractWordScores).reduce(addObjects, {}) - const minScore = Math.min(...Object.values(wordScores)) - const maxScore = Math.max(...Object.values(wordScores)) - const normalizedWordScores = mapValues( - wordScores, - (score) => (score - minScore) / (maxScore - minScore) - ) - - // console.log( - // 'your word scores', - // _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(0, 100), - // _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(-100) - // ) - - return normalizedWordScores -} - -export function getContractScore( - contract: Contract, - wordScores: { [word: string]: number } -) { - if (Object.keys(wordScores).length === 0) return 1 - - const wordFrequency = contractToWordFrequency(contract) - const score = sumBy(Object.keys(wordFrequency), (word) => { - const wordFreq = wordFrequency[word] ?? 0 - const weight = wordScores[word] ?? 0 - return wordFreq * weight - }) - - return score -} - -// Caluculate Term Frequency-Inverse Document Frequency (TF-IDF): -// https://medium.datadriveninvestor.com/tf-idf-in-natural-language-processing-8db8ef4a7736 -function calculateContractTfIdf(contracts: Contract[]) { - const contractFreq = contracts.map((c) => contractToWordFrequency(c)) - const contractWords = contractFreq.map((freq) => Object.keys(freq)) - - const wordsCount: { [word: string]: number } = {} - for (const words of contractWords) { - for (const word of words) { - wordsCount[word] = (wordsCount[word] ?? 0) + 1 - } - } - - const wordIdf = mapValues(wordsCount, (count) => - Math.log(contracts.length / count) - ) - const contractWordsTfIdf = contractFreq.map((wordFreq) => - mapValues(wordFreq, (freq, word) => freq * wordIdf[word]) - ) - return Object.fromEntries( - contracts.map((c, i) => [c.id, contractWordsTfIdf[i]]) - ) -} diff --git a/web/hooks/use-algo-feed.ts b/web/hooks/use-algo-feed.ts deleted file mode 100644 index e195936f..00000000 --- a/web/hooks/use-algo-feed.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { useState, useEffect } from 'react' -import type { feed } from 'common/feed' -import { useTimeSinceFirstRender } from './use-time-since-first-render' -import { trackLatency } from 'web/lib/firebase/tracking' -import { User } from 'common/user' -import { getCategoryFeeds, getUserFeed } from 'web/lib/firebase/users' -import { - getRecentBetsAndComments, - getTopWeeklyContracts, -} from 'web/lib/firebase/contracts' - -export const useAlgoFeed = ( - user: User | null | undefined, - category: string -) => { - const [allFeed, setAllFeed] = useState() - const [categoryFeeds, setCategoryFeeds] = useState<{ [x: string]: feed }>() - - const getTime = useTimeSinceFirstRender() - - useEffect(() => { - if (user) { - getUserFeed(user.id).then((feed) => { - if (feed.length === 0) { - getDefaultFeed().then((feed) => setAllFeed(feed)) - } else setAllFeed(feed) - - trackLatency(user.id, 'feed', getTime()) - console.log('"all" feed load time', getTime()) - }) - - getCategoryFeeds(user.id).then((feeds) => { - setCategoryFeeds(feeds) - console.log('category feeds load time', getTime()) - }) - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [user?.id]) - - const feed = category === 'all' ? allFeed : categoryFeeds?.[category] - - return feed -} - -const getDefaultFeed = async () => { - const contracts = await getTopWeeklyContracts() - const feed = await Promise.all( - contracts.map((c) => getRecentBetsAndComments(c)) - ) - return feed -} diff --git a/web/lib/firebase/contracts.ts b/web/lib/firebase/contracts.ts index b31b8d04..243a453a 100644 --- a/web/lib/firebase/contracts.ts +++ b/web/lib/firebase/contracts.ts @@ -22,7 +22,6 @@ import { createRNG, shuffle } from 'common/util/random' import { getCpmmProbability } from 'common/calculate-cpmm' import { formatMoney, formatPercent } from 'common/util/format' import { DAY_MS } from 'common/util/time' -import { MAX_FEED_CONTRACTS } from 'common/recommended-contracts' import { Bet } from 'common/bet' import { Comment } from 'common/comment' import { ENV_CONFIG } from 'common/envs/constants' @@ -285,16 +284,6 @@ export async function getContractsBySlugs(slugs: string[]) { return sortBy(data, (contract) => -1 * contract.volume24Hours) } -const topWeeklyQuery = query( - contracts, - where('isResolved', '==', false), - orderBy('volume7Days', 'desc'), - limit(MAX_FEED_CONTRACTS) -) -export async function getTopWeeklyContracts() { - return await getValues(topWeeklyQuery) -} - const closingSoonQuery = query( contracts, where('isResolved', '==', false), diff --git a/web/lib/firebase/users.ts b/web/lib/firebase/users.ts index 7db087b9..6cfee163 100644 --- a/web/lib/firebase/users.ts +++ b/web/lib/firebase/users.ts @@ -15,18 +15,9 @@ import { } from 'firebase/firestore' import { getAuth } from 'firebase/auth' import { GoogleAuthProvider, signInWithPopup } from 'firebase/auth' -import { zip } from 'lodash' import { app, db } from './init' import { PortfolioMetrics, PrivateUser, User } from 'common/user' -import { - coll, - getValue, - getValues, - listenForValue, - listenForValues, -} from './utils' -import { feed } from 'common/feed' -import { CATEGORY_LIST } from 'common/categories' +import { coll, getValues, listenForValue, listenForValues } from './utils' import { safeLocalStorage } from '../util/local' import { filterDefined } from 'common/util/array' import { addUserToGroupViaId } from 'web/lib/firebase/groups' @@ -248,25 +239,6 @@ export function getUsers() { return getValues(users) } -export async function getUserFeed(userId: string) { - const feedDoc = doc(privateUsers, userId, 'cache', 'feed') - const userFeed = await getValue<{ - feed: feed - }>(feedDoc) - return userFeed?.feed ?? [] -} - -export async function getCategoryFeeds(userId: string) { - const cacheCollection = collection(privateUsers, userId, 'cache') - const feedData = await Promise.all( - CATEGORY_LIST.map((category) => - getValue<{ feed: feed }>(doc(cacheCollection, `feed-${category}`)) - ) - ) - const feeds = feedData.map((data) => data?.feed ?? []) - return Object.fromEntries(zip(CATEGORY_LIST, feeds) as [string, feed][]) -} - export async function follow(userId: string, followedUserId: string) { const followDoc = doc(collection(users, userId, 'follows'), followedUserId) await setDoc(followDoc, {