Remove some dead code related to tags, categories, and old feed stuff (#765)

* Remove dead image storage code

* Kill tag page

* Kill tag and categories related component UI

* Kill some old algo feed kind of code
This commit is contained in:
Marshall Polaris 2022-08-18 15:46:11 -07:00 committed by GitHub
parent 4f6d478211
commit 0cf9a90cfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1 additions and 625 deletions

View File

@ -1,187 +0,0 @@
import { union, sum, sumBy, sortBy, groupBy, mapValues } from 'lodash'
import { Bet } from './bet'
import { Contract } from './contract'
import { ClickEvent } from './tracking'
import { filterDefined } from './util/array'
import { addObjects } from './util/object'
export const MAX_FEED_CONTRACTS = 75
export const getRecommendedContracts = (
contractsById: { [contractId: string]: Contract },
yourBetOnContractIds: string[]
) => {
const contracts = Object.values(contractsById)
const yourContracts = filterDefined(
yourBetOnContractIds.map((contractId) => contractsById[contractId])
)
const yourContractIds = new Set(yourContracts.map((c) => c.id))
const notYourContracts = contracts.filter((c) => !yourContractIds.has(c.id))
const yourWordFrequency = contractsToWordFrequency(yourContracts)
const otherWordFrequency = contractsToWordFrequency(notYourContracts)
const words = union(
Object.keys(yourWordFrequency),
Object.keys(otherWordFrequency)
)
const yourWeightedFrequency = Object.fromEntries(
words.map((word) => {
const [yourFreq, otherFreq] = [
yourWordFrequency[word] ?? 0,
otherWordFrequency[word] ?? 0,
]
const score = yourFreq / (yourFreq + otherFreq + 0.0001)
return [word, score]
})
)
// console.log(
// 'your weighted frequency',
// _.sortBy(_.toPairs(yourWeightedFrequency), ([, freq]) => -freq)
// )
const scoredContracts = contracts.map((contract) => {
const wordFrequency = contractToWordFrequency(contract)
const score = sumBy(Object.keys(wordFrequency), (word) => {
const wordFreq = wordFrequency[word] ?? 0
const weight = yourWeightedFrequency[word] ?? 0
return wordFreq * weight
})
return {
contract,
score,
}
})
return sortBy(scoredContracts, (scored) => -scored.score).map(
(scored) => scored.contract
)
}
const contractToText = (contract: Contract) => {
const { description, question, tags, creatorUsername } = contract
return `${creatorUsername} ${question} ${tags.join(' ')} ${description}`
}
const MAX_CHARS_IN_WORD = 100
const getWordsCount = (text: string) => {
const normalizedText = text.replace(/[^a-zA-Z]/g, ' ').toLowerCase()
const words = normalizedText
.split(' ')
.filter((word) => word)
.filter((word) => word.length <= MAX_CHARS_IN_WORD)
const counts: { [word: string]: number } = {}
for (const word of words) {
if (counts[word]) counts[word]++
else counts[word] = 1
}
return counts
}
const toFrequency = (counts: { [word: string]: number }) => {
const total = sum(Object.values(counts))
return mapValues(counts, (count) => count / total)
}
const contractToWordFrequency = (contract: Contract) =>
toFrequency(getWordsCount(contractToText(contract)))
const contractsToWordFrequency = (contracts: Contract[]) => {
const frequencySum = contracts
.map(contractToWordFrequency)
.reduce(addObjects, {})
return toFrequency(frequencySum)
}
export const getWordScores = (
contracts: Contract[],
contractViewCounts: { [contractId: string]: number },
clicks: ClickEvent[],
bets: Bet[]
) => {
const contractClicks = groupBy(clicks, (click) => click.contractId)
const contractBets = groupBy(bets, (bet) => bet.contractId)
const yourContracts = contracts.filter(
(c) =>
contractViewCounts[c.id] || contractClicks[c.id] || contractBets[c.id]
)
const yourTfIdf = calculateContractTfIdf(yourContracts)
const contractWordScores = mapValues(yourTfIdf, (wordsTfIdf, contractId) => {
const viewCount = contractViewCounts[contractId] ?? 0
const clickCount = contractClicks[contractId]?.length ?? 0
const betCount = contractBets[contractId]?.length ?? 0
const factor =
-1 * Math.log(viewCount + 1) +
10 * Math.log(betCount + clickCount / 4 + 1)
return mapValues(wordsTfIdf, (tfIdf) => tfIdf * factor)
})
const wordScores = Object.values(contractWordScores).reduce(addObjects, {})
const minScore = Math.min(...Object.values(wordScores))
const maxScore = Math.max(...Object.values(wordScores))
const normalizedWordScores = mapValues(
wordScores,
(score) => (score - minScore) / (maxScore - minScore)
)
// console.log(
// 'your word scores',
// _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(0, 100),
// _.sortBy(_.toPairs(normalizedWordScores), ([, score]) => -score).slice(-100)
// )
return normalizedWordScores
}
export function getContractScore(
contract: Contract,
wordScores: { [word: string]: number }
) {
if (Object.keys(wordScores).length === 0) return 1
const wordFrequency = contractToWordFrequency(contract)
const score = sumBy(Object.keys(wordFrequency), (word) => {
const wordFreq = wordFrequency[word] ?? 0
const weight = wordScores[word] ?? 0
return wordFreq * weight
})
return score
}
// Caluculate Term Frequency-Inverse Document Frequency (TF-IDF):
// https://medium.datadriveninvestor.com/tf-idf-in-natural-language-processing-8db8ef4a7736
function calculateContractTfIdf(contracts: Contract[]) {
const contractFreq = contracts.map((c) => contractToWordFrequency(c))
const contractWords = contractFreq.map((freq) => Object.keys(freq))
const wordsCount: { [word: string]: number } = {}
for (const words of contractWords) {
for (const word of words) {
wordsCount[word] = (wordsCount[word] ?? 0) + 1
}
}
const wordIdf = mapValues(wordsCount, (count) =>
Math.log(contracts.length / count)
)
const contractWordsTfIdf = contractFreq.map((wordFreq) =>
mapValues(wordFreq, (freq, word) => freq * wordIdf[word])
)
return Object.fromEntries(
contracts.map((c, i) => [c.id, contractWordsTfIdf[i]])
)
}

View File

@ -1,167 +0,0 @@
import clsx from 'clsx'
import { PencilIcon } from '@heroicons/react/outline'
import { union, difference } from 'lodash'
import { Row } from '../layout/row'
import { CATEGORIES, category, CATEGORY_LIST } from '../../../common/categories'
import { Modal } from '../layout/modal'
import { Col } from '../layout/col'
import { useState } from 'react'
import { updateUser, User } from 'web/lib/firebase/users'
import { Checkbox } from '../checkbox'
import { track } from 'web/lib/service/analytics'
export function CategorySelector(props: {
category: string
setCategory: (category: string) => void
className?: string
}) {
const { className, category, setCategory } = props
return (
<Row
className={clsx(
'carousel mr-2 items-center space-x-2 space-y-2 overflow-x-scroll pb-4 sm:flex-wrap',
className
)}
>
<div />
<CategoryButton
key="all"
category="All"
isFollowed={category === 'all'}
toggle={() => {
setCategory('all')
}}
/>
<CategoryButton
key="following"
category="Following"
isFollowed={category === 'following'}
toggle={() => {
setCategory('following')
}}
/>
{CATEGORY_LIST.map((cat) => (
<CategoryButton
key={cat}
category={CATEGORIES[cat as category].split(' ')[0]}
isFollowed={cat === category}
toggle={() => {
setCategory(cat)
}}
/>
))}
</Row>
)
}
function CategoryButton(props: {
category: string
isFollowed: boolean
toggle: () => void
className?: string
}) {
const { toggle, category, isFollowed, className } = props
return (
<div
className={clsx(
className,
'rounded-full border-2 px-4 py-1 shadow-md hover:bg-gray-200',
'cursor-pointer select-none',
isFollowed ? 'border-gray-300 bg-gray-300' : 'bg-white'
)}
onClick={toggle}
>
<span className="text-sm text-gray-500">{category}</span>
</div>
)
}
export function EditCategoriesButton(props: {
user: User
className?: string
}) {
const { user, className } = props
const [isOpen, setIsOpen] = useState(false)
return (
<div
className={clsx(
className,
'btn btn-sm btn-ghost cursor-pointer gap-2 whitespace-nowrap text-sm normal-case text-gray-700'
)}
onClick={() => {
setIsOpen(true)
track('edit categories button')
}}
>
<PencilIcon className="inline h-4 w-4" />
Categories
<CategorySelectorModal
user={user}
isOpen={isOpen}
setIsOpen={setIsOpen}
/>
</div>
)
}
function CategorySelectorModal(props: {
user: User
isOpen: boolean
setIsOpen: (isOpen: boolean) => void
}) {
const { user, isOpen, setIsOpen } = props
const followedCategories =
user?.followedCategories === undefined
? CATEGORY_LIST
: user.followedCategories
const selectAll =
user.followedCategories === undefined ||
followedCategories.length < CATEGORY_LIST.length
return (
<Modal open={isOpen} setOpen={setIsOpen}>
<Col className="rounded bg-white p-6">
<button
className="btn btn-sm btn-outline mb-4 self-start normal-case"
onClick={() => {
if (selectAll) {
updateUser(user.id, {
followedCategories: CATEGORY_LIST,
})
} else {
updateUser(user.id, {
followedCategories: [],
})
}
}}
>
Select {selectAll ? 'all' : 'none'}
</button>
<Col className="grid w-full grid-cols-2 gap-4">
{CATEGORY_LIST.map((cat) => (
<Checkbox
className="col-span-1"
key={cat}
label={CATEGORIES[cat as category].split(' ')[0]}
checked={followedCategories.includes(cat)}
toggle={(checked) => {
updateUser(user.id, {
followedCategories: checked
? difference(followedCategories, [cat])
: union([cat], followedCategories),
})
}}
/>
))}
</Col>
</Col>
</Modal>
)
}

View File

@ -1,55 +0,0 @@
import clsx from 'clsx'
import { useState } from 'react'
import { parseWordsAsTags } from 'common/util/parse'
import { Contract, updateContract } from 'web/lib/firebase/contracts'
import { Col } from './layout/col'
import { Row } from './layout/row'
import { TagsList } from './tags-list'
import { MAX_TAG_LENGTH } from 'common/contract'
export function TagsInput(props: { contract: Contract; className?: string }) {
const { contract, className } = props
const { tags } = contract
const [tagText, setTagText] = useState('')
const newTags = parseWordsAsTags(`${tags.join(' ')} ${tagText}`)
const [isSubmitting, setIsSubmitting] = useState(false)
const updateTags = async () => {
setIsSubmitting(true)
await updateContract(contract.id, {
tags: newTags,
lowercaseTags: newTags.map((tag) => tag.toLowerCase()),
})
setIsSubmitting(false)
setTagText('')
}
return (
<Col className={clsx('gap-4', className)}>
<TagsList tags={newTags} noLabel />
<Row className="items-center gap-4">
<input
style={{ maxWidth: 150 }}
placeholder="Type a tag..."
className="input input-sm input-bordered resize-none"
disabled={isSubmitting}
value={tagText}
maxLength={MAX_TAG_LENGTH}
onChange={(e) => setTagText(e.target.value || '')}
onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault()
updateTags()
}
}}
/>
<button className="btn btn-xs btn-outline" onClick={updateTags}>
Save tags
</button>
</Row>
</Col>
)
}

View File

@ -1,86 +0,0 @@
import clsx from 'clsx'
import { CATEGORIES, category } from '../../common/categories'
import { Col } from './layout/col'
import { Row } from './layout/row'
import { SiteLink } from './site-link'
function Hashtag(props: { tag: string; noLink?: boolean }) {
const { tag, noLink } = props
const category = CATEGORIES[tag.replace('#', '').toLowerCase() as category]
const body = (
<div className={clsx('', !noLink && 'cursor-pointer')}>
<span className="text-sm">{category ? '#' + category : tag} </span>
</div>
)
if (noLink) return body
return (
<SiteLink href={`/tag/${tag.substring(1)}`} className="flex items-center">
{body}
</SiteLink>
)
}
export function TagsList(props: {
tags: string[]
className?: string
noLink?: boolean
noLabel?: boolean
label?: string
}) {
const { tags, className, noLink, noLabel, label } = props
return (
<Row className={clsx('flex-wrap items-center gap-2', className)}>
{!noLabel && <div className="mr-1">{label || 'Tags'}</div>}
{tags.map((tag) => (
<Hashtag
key={tag}
tag={tag.startsWith('#') ? tag : `#${tag}`}
noLink={noLink}
/>
))}
</Row>
)
}
export function FoldTag(props: { fold: { slug: string; name: string } }) {
const { fold } = props
const { slug, name } = fold
return (
<SiteLink href={`/fold/${slug}`} className="flex items-center">
<div
className={clsx(
'rounded-full border-2 bg-white px-4 py-1 shadow-md',
'cursor-pointer'
)}
>
<span className="text-sm text-gray-500">{name}</span>
</div>
</SiteLink>
)
}
export function FoldTagList(props: {
folds: { slug: string; name: string }[]
noLabel?: boolean
className?: string
}) {
const { folds, noLabel, className } = props
return (
<Col className="gap-2">
{!noLabel && <div className="mr-1 text-gray-500">Communities</div>}
<Row className={clsx('flex-wrap items-center gap-2', className)}>
{folds.length > 0 && (
<>
{folds.map((fold) => (
<FoldTag key={fold.slug} fold={fold} />
))}
</>
)}
</Row>
</Col>
)
}

View File

@ -1,51 +0,0 @@
import { useState, useEffect } from 'react'
import type { feed } from 'common/feed'
import { useTimeSinceFirstRender } from './use-time-since-first-render'
import { trackLatency } from 'web/lib/firebase/tracking'
import { User } from 'common/user'
import { getCategoryFeeds, getUserFeed } from 'web/lib/firebase/users'
import {
getRecentBetsAndComments,
getTopWeeklyContracts,
} from 'web/lib/firebase/contracts'
export const useAlgoFeed = (
user: User | null | undefined,
category: string
) => {
const [allFeed, setAllFeed] = useState<feed>()
const [categoryFeeds, setCategoryFeeds] = useState<{ [x: string]: feed }>()
const getTime = useTimeSinceFirstRender()
useEffect(() => {
if (user) {
getUserFeed(user.id).then((feed) => {
if (feed.length === 0) {
getDefaultFeed().then((feed) => setAllFeed(feed))
} else setAllFeed(feed)
trackLatency(user.id, 'feed', getTime())
console.log('"all" feed load time', getTime())
})
getCategoryFeeds(user.id).then((feeds) => {
setCategoryFeeds(feeds)
console.log('category feeds load time', getTime())
})
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [user?.id])
const feed = category === 'all' ? allFeed : categoryFeeds?.[category]
return feed
}
const getDefaultFeed = async () => {
const contracts = await getTopWeeklyContracts()
const feed = await Promise.all(
contracts.map((c) => getRecentBetsAndComments(c))
)
return feed
}

View File

@ -22,7 +22,6 @@ import { createRNG, shuffle } from 'common/util/random'
import { getCpmmProbability } from 'common/calculate-cpmm'
import { formatMoney, formatPercent } from 'common/util/format'
import { DAY_MS } from 'common/util/time'
import { MAX_FEED_CONTRACTS } from 'common/recommended-contracts'
import { Bet } from 'common/bet'
import { Comment } from 'common/comment'
import { ENV_CONFIG } from 'common/envs/constants'
@ -285,16 +284,6 @@ export async function getContractsBySlugs(slugs: string[]) {
return sortBy(data, (contract) => -1 * contract.volume24Hours)
}
const topWeeklyQuery = query(
contracts,
where('isResolved', '==', false),
orderBy('volume7Days', 'desc'),
limit(MAX_FEED_CONTRACTS)
)
export async function getTopWeeklyContracts() {
return await getValues<Contract>(topWeeklyQuery)
}
const closingSoonQuery = query(
contracts,
where('isResolved', '==', false),

View File

@ -14,20 +14,10 @@ import {
onSnapshot,
} from 'firebase/firestore'
import { getAuth } from 'firebase/auth'
import { ref, getStorage, uploadBytes, getDownloadURL } from 'firebase/storage'
import { GoogleAuthProvider, signInWithPopup } from 'firebase/auth'
import { zip } from 'lodash'
import { app, db } from './init'
import { PortfolioMetrics, PrivateUser, User } from 'common/user'
import {
coll,
getValue,
getValues,
listenForValue,
listenForValues,
} from './utils'
import { feed } from 'common/feed'
import { CATEGORY_LIST } from 'common/categories'
import { coll, getValues, listenForValue, listenForValues } from './utils'
import { safeLocalStorage } from '../util/local'
import { filterDefined } from 'common/util/array'
import { addUserToGroupViaId } from 'web/lib/firebase/groups'
@ -202,20 +192,6 @@ export async function firebaseLogout() {
await auth.signOut()
}
const storage = getStorage(app)
// Example: uploadData('avatars/ajfi8iejsf.png', data)
export async function uploadData(
path: string,
data: ArrayBuffer | Blob | Uint8Array
) {
const uploadRef = ref(storage, path)
// Uploaded files should be cached for 1 day, then revalidated
// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
const metadata = { cacheControl: 'public, max-age=86400, must-revalidate' }
await uploadBytes(uploadRef, data, metadata)
return await getDownloadURL(uploadRef)
}
export async function listUsers(userIds: string[]) {
if (userIds.length > 10) {
throw new Error('Too many users requested at once; Firestore limits to 10')
@ -263,25 +239,6 @@ export function getUsers() {
return getValues<User>(users)
}
export async function getUserFeed(userId: string) {
const feedDoc = doc(privateUsers, userId, 'cache', 'feed')
const userFeed = await getValue<{
feed: feed
}>(feedDoc)
return userFeed?.feed ?? []
}
export async function getCategoryFeeds(userId: string) {
const cacheCollection = collection(privateUsers, userId, 'cache')
const feedData = await Promise.all(
CATEGORY_LIST.map((category) =>
getValue<{ feed: feed }>(doc(cacheCollection, `feed-${category}`))
)
)
const feeds = feedData.map((data) => data?.feed ?? [])
return Object.fromEntries(zip(CATEGORY_LIST, feeds) as [string, feed][])
}
export async function follow(userId: string, followedUserId: string) {
const followDoc = doc(collection(users, userId, 'follows'), followedUserId)
await setDoc(followDoc, {

View File

@ -1,24 +0,0 @@
import { useRouter } from 'next/router'
import { useUser } from 'web/hooks/use-user'
import { ContractSearch } from '../../components/contract-search'
import { Page } from '../../components/page'
import { Title } from '../../components/title'
export default function TagPage() {
const router = useRouter()
const user = useUser()
const { tag } = router.query as { tag: string }
if (!router.isReady) return <div />
return (
<Page>
<Title text={`#${tag}`} />
<ContractSearch
user={user}
defaultSort="newest"
defaultFilter="all"
additionalFilter={{ tag }}
/>
</Page>
)
}