open.source/manifold
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Kill some old algo feed kind of code

Browse Source
This commit is contained in:
Marshall Polaris 2022-08-17 14:26:07 -07:00
parent 75e08a3c4c
commit 6b2ba35046
4 changed files with 1 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
common/recommended-contracts.ts
View File

@ -1,187 +0,0 @@
import { union, sum, sumBy, sortBy, groupBy, mapValues } from 'lodash'
import { Bet } from './bet'
import { Contract } from './contract'
import { ClickEvent } from './tracking'
import { filterDefined } from './util/array'
import { addObjects } from './util/object'
export const MAX_FEED_CONTRACTS = 75
export const getRecommendedContracts = (
contractsById: { [contractId: string]: Contract },
yourBetOnContractIds: string[]
) => {
const contracts = Object.values(contractsById)
const yourContracts = filterDefined(
yourBetOnContractIds.map((contractId) => contractsById[contractId])
)
const yourContractIds = new Set(yourContracts.map((c) => c.id))
const notYourContracts = contracts.filter((c) => !yourContractIds.has(c.id))
const yourWordFrequency = contractsToWordFrequency(yourContracts)
const otherWordFrequency = contractsToWordFrequency(notYourContracts)
const words = union(
Object.keys(yourWordFrequency),
Object.keys(otherWordFrequency)
)
const yourWeightedFrequency = Object.fromEntries(
words.map((word) => {
const [yourFreq, otherFreq] = [
yourWordFrequency[word] ?? 0,
otherWordFrequency[word] ?? 0,
]
const score = yourFreq / (yourFreq + otherFreq + 0.0001)
return [word, score]
})
)
// console.log(
// 'your weighted frequency',
// _.sortBy(_.toPairs(yourWeightedFrequency), ([, freq]) => -freq)
// )
const scoredContracts = contracts.map((contract) => {
const wordFrequency = contractToWordFrequency(contract)
const score = sumBy(Object.keys(wordFrequency), (word) => {
const wordFreq = wordFrequency[word] ?? 0
const weight = yourWeightedFrequency[word] ?? 0
return wordFreq * weight
})
return {
contract,
score,
}
})
return sortBy(scoredContracts, (scored) => -scored.score).map(
(scored) => scored.contract
)
}
const contractToText = (contract: Contract) => {
const { description, question, tags, creatorUsername } = contract
return `${creatorUsername} ${question} ${tags.join(' ')} ${description}`
}
const MAX_CHARS_IN_WORD = 100
const getWordsCount = (text: string) => {
const normalizedText = text.replace(/[^a-zA-Z]/g, ' ').toLowerCase()
const words = normalizedText
.split(' ')
.filter((word) => word)
.filter((word) => word.length <= MAX_CHARS_IN_WORD)
const counts: { [word: string]: number } = {}
for (const word of words) {
if (counts[word]) counts[word]++
else counts[word] = 1
}
return counts
}
const toFrequency = (counts: { [word: string]: number }) => {
const total = sum(Object.values(counts))
return mapValues(counts, (count) => count / total)
}
const contractToWordFrequency = (contract: Contract) =>
toFrequency(getWordsCount(contractToText(contract)))
const contractsToWordFrequency = (contracts: Contract[]) => {
const frequencySum = contracts
.map(contractToWordFrequency)
.reduce(addObjects, {})
return toFrequency(frequencySum)
}
export const getWordScores = (
contracts: Contract[],
contractViewCounts: { [contractId: string]: number },
clicks: ClickEvent[],
bets: Bet[]
) => {
const contractClicks = groupBy(clicks, (click) => click.contractId)
const contractBets = groupBy(bets, (bet) => bet.contractId)
const yourContracts = contracts.filter(
(c) =>
contractViewCounts[c.id] || contractClicks[c.id] || contractBets[c.id]
)
const yourTfIdf = calculateContractTfIdf(yourContracts)
const contractWordScores = mapValues(yourTfIdf, (wordsTfIdf, contractId) => {
const viewCount = contractViewCounts[contractId] ?? 0
const clickCount = contractClicks[contractId]?.length ?? 0
const betCount = contractBets[contractId]?.length ?? 0
const factor =
-1 * Math.log(viewCount + 1) +
10 * Math.log(betCount + clickCount / 4 + 1)
return mapValues(wordsTfIdf, (tfIdf) => tfIdf * factor)
})
const wordScores = Object.values(contractWordScores).reduce(addObjects, {})
const minScore = Math.min(...Object.values(wordScores))
const maxScore = Math.max(...Object.values(wordScores))
const normalizedWordScores = mapValues(
wordScores,
(score) => (score - minScore) / (maxScore - minScore)
)
// console.log(
// 'your word scores',
// _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(0, 100),
// _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(-100)
// )
return normalizedWordScores
}
export function getContractScore(
contract: Contract,
wordScores: { [word: string]: number }
) {
if (Object.keys(wordScores).length === 0) return 1
const wordFrequency = contractToWordFrequency(contract)
const score = sumBy(Object.keys(wordFrequency), (word) => {
const wordFreq = wordFrequency[word] ?? 0
const weight = wordScores[word] ?? 0
return wordFreq * weight
})
return score
}
// Caluculate Term Frequency-Inverse Document Frequency (TF-IDF):
// https://medium.datadriveninvestor.com/tf-idf-in-natural-language-processing-8db8ef4a7736
function calculateContractTfIdf(contracts: Contract[]) {
const contractFreq = contracts.map((c) => contractToWordFrequency(c))
const contractWords = contractFreq.map((freq) => Object.keys(freq))
const wordsCount: { [word: string]: number } = {}
for (const words of contractWords) {
for (const word of words) {
wordsCount[word] = (wordsCount[word] ?? 0) + 1
}
}
const wordIdf = mapValues(wordsCount, (count) =>
Math.log(contracts.length / count)
)
const contractWordsTfIdf = contractFreq.map((wordFreq) =>
mapValues(wordFreq, (freq, word) => freq * wordIdf[word])
)
return Object.fromEntries(
contracts.map((c, i) => [c.id, contractWordsTfIdf[i]])
)
}

51
web/hooks/use-algo-feed.ts
View File

@ -1,51 +0,0 @@
import { useState, useEffect } from 'react'
import type { feed } from 'common/feed'
import { useTimeSinceFirstRender } from './use-time-since-first-render'
import { trackLatency } from 'web/lib/firebase/tracking'
import { User } from 'common/user'
import { getCategoryFeeds, getUserFeed } from 'web/lib/firebase/users'
import {
getRecentBetsAndComments,
getTopWeeklyContracts,
} from 'web/lib/firebase/contracts'
export const useAlgoFeed = (
user: User | null | undefined,
category: string
) => {
const [allFeed, setAllFeed] = useState<feed>()
const [categoryFeeds, setCategoryFeeds] = useState<{ [x: string]: feed }>()
const getTime = useTimeSinceFirstRender()
useEffect(() => {
if (user) {
getUserFeed(user.id).then((feed) => {
if (feed.length === 0) {
getDefaultFeed().then((feed) => setAllFeed(feed))
} else setAllFeed(feed)
trackLatency(user.id, 'feed', getTime())
console.log('"all" feed load time', getTime())
})
getCategoryFeeds(user.id).then((feeds) => {
setCategoryFeeds(feeds)
console.log('category feeds load time', getTime())
})
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [user?.id])
const feed = category === 'all' ? allFeed : categoryFeeds?.[category]
return feed
}
const getDefaultFeed = async () => {
const contracts = await getTopWeeklyContracts()
const feed = await Promise.all(
contracts.map((c) => getRecentBetsAndComments(c))
)
return feed
}

11
web/lib/firebase/contracts.ts
View File

@ -22,7 +22,6 @@ import { createRNG, shuffle } from 'common/util/random'
import { getCpmmProbability } from 'common/calculate-cpmm' import { getCpmmProbability } from 'common/calculate-cpmm'
import { formatMoney, formatPercent } from 'common/util/format' import { formatMoney, formatPercent } from 'common/util/format'
import { DAY_MS } from 'common/util/time' import { DAY_MS } from 'common/util/time'
import { MAX_FEED_CONTRACTS } from 'common/recommended-contracts'
import { Bet } from 'common/bet' import { Bet } from 'common/bet'
import { Comment } from 'common/comment' import { Comment } from 'common/comment'
import { ENV_CONFIG } from 'common/envs/constants' import { ENV_CONFIG } from 'common/envs/constants'
@ -285,16 +284,6 @@ export async function getContractsBySlugs(slugs: string[]) {
return sortBy(data, (contract) => -1 * contract.volume24Hours) return sortBy(data, (contract) => -1 * contract.volume24Hours)
} }
const topWeeklyQuery = query(
contracts,
where('isResolved', '==', false),
orderBy('volume7Days', 'desc'),
limit(MAX_FEED_CONTRACTS)
)
export async function getTopWeeklyContracts() {
return await getValues<Contract>(topWeeklyQuery)
}
const closingSoonQuery = query( const closingSoonQuery = query(
contracts, contracts,
where('isResolved', '==', false), where('isResolved', '==', false),

30
web/lib/firebase/users.ts
View File

@ -15,18 +15,9 @@ import {
} from 'firebase/firestore' } from 'firebase/firestore'
import { getAuth } from 'firebase/auth' import { getAuth } from 'firebase/auth'
import { GoogleAuthProvider, signInWithPopup } from 'firebase/auth' import { GoogleAuthProvider, signInWithPopup } from 'firebase/auth'
import { zip } from 'lodash'
import { app, db } from './init' import { app, db } from './init'
import { PortfolioMetrics, PrivateUser, User } from 'common/user' import { PortfolioMetrics, PrivateUser, User } from 'common/user'
import { import { coll, getValues, listenForValue, listenForValues } from './utils'
coll,
getValue,
getValues,
listenForValue,
listenForValues,
} from './utils'
import { feed } from 'common/feed'
import { CATEGORY_LIST } from 'common/categories'
import { safeLocalStorage } from '../util/local' import { safeLocalStorage } from '../util/local'
import { filterDefined } from 'common/util/array' import { filterDefined } from 'common/util/array'
import { addUserToGroupViaId } from 'web/lib/firebase/groups' import { addUserToGroupViaId } from 'web/lib/firebase/groups'
@ -248,25 +239,6 @@ export function getUsers() {
return getValues<User>(users) return getValues<User>(users)
} }
export async function getUserFeed(userId: string) {
const feedDoc = doc(privateUsers, userId, 'cache', 'feed')
const userFeed = await getValue<{
feed: feed
}>(feedDoc)
return userFeed?.feed ?? []
}
export async function getCategoryFeeds(userId: string) {
const cacheCollection = collection(privateUsers, userId, 'cache')
const feedData = await Promise.all(
CATEGORY_LIST.map((category) =>
getValue<{ feed: feed }>(doc(cacheCollection, `feed-${category}`))
)
)
const feeds = feedData.map((data) => data?.feed ?? [])
return Object.fromEntries(zip(CATEGORY_LIST, feeds) as [string, feed][])
}
export async function follow(userId: string, followedUserId: string) { export async function follow(userId: string, followedUserId: string) {
const followDoc = doc(collection(users, userId, 'follows'), followedUserId) const followDoc = doc(collection(users, userId, 'follows'), followedUserId)
await setDoc(followDoc, { await setDoc(followDoc, {