Remove code for obsolete feed updater backend jobs (#607)
* Remove code for obsolete feed updater backend jobs * Kill two more obsolete guys
This commit is contained in:
Marshall Polaris
GitHub
parent
218b18254c
commit
18b8758191
|
|
@ -23,7 +23,6 @@
|
|||
"main": "functions/src/index.js",
|
||||
"dependencies": {
|
||||
"@amplitude/node": "1.10.0",
|
||||
"fetch": "1.1.0",
|
||||
"firebase-admin": "10.0.0",
|
||||
"firebase-functions": "3.21.2",
|
||||
"lodash": "4.17.21",
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
import * as admin from 'firebase-admin'
|
||||
|
||||
import fetch from './fetch'
|
||||
|
||||
export const callCloudFunction = (functionName: string, data: unknown = {}) => {
|
||||
const projectId = admin.instanceId().app.options.projectId
|
||||
|
||||
const url = `https://us-central1-${projectId}.cloudfunctions.net/${functionName}`
|
||||
|
||||
return fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ data }),
|
||||
}).then((response) => response.json())
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
let fetchRequest: typeof fetch
|
||||
|
||||
try {
|
||||
fetchRequest = fetch
|
||||
} catch {
|
||||
fetchRequest = require('node-fetch')
|
||||
}
|
||||
|
||||
export default fetchRequest
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
import * as functions from 'firebase-functions'
|
||||
|
||||
import { callCloudFunction } from './call-cloud-function'
|
||||
|
||||
export const keepAwake = functions.pubsub
|
||||
.schedule('every 1 minutes')
|
||||
.onRun(async () => {
|
||||
await Promise.all([
|
||||
callCloudFunction('placeBet'),
|
||||
callCloudFunction('resolveMarket'),
|
||||
callCloudFunction('sellBet'),
|
||||
])
|
||||
|
||||
await sleep(30)
|
||||
|
||||
await Promise.all([
|
||||
callCloudFunction('placeBet'),
|
||||
callCloudFunction('resolveMarket'),
|
||||
callCloudFunction('sellBet'),
|
||||
])
|
||||
})
|
||||
|
||||
const sleep = (seconds: number) => {
|
||||
return new Promise((resolve) => setTimeout(resolve, seconds * 1000))
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
import * as admin from 'firebase-admin'
|
||||
|
||||
import { initAdmin } from './script-init'
|
||||
initAdmin()
|
||||
|
||||
import { getValues } from '../utils'
|
||||
import { User } from '../../../common/user'
|
||||
import { batchedWaitAll } from '../../../common/util/promise'
|
||||
import { Contract } from '../../../common/contract'
|
||||
import { updateWordScores } from '../update-recommendations'
|
||||
import { computeFeed } from '../update-feed'
|
||||
import { getFeedContracts, getTaggedContracts } from '../get-feed-data'
|
||||
import { CATEGORY_LIST } from '../../../common/categories'
|
||||
|
||||
const firestore = admin.firestore()
|
||||
|
||||
async function updateFeed() {
|
||||
console.log('Updating feed')
|
||||
|
||||
const contracts = await getValues<Contract>(firestore.collection('contracts'))
|
||||
const feedContracts = await getFeedContracts()
|
||||
const users = await getValues<User>(
|
||||
firestore.collection('users').where('username', '==', 'JamesGrugett')
|
||||
)
|
||||
|
||||
await batchedWaitAll(
|
||||
users.map((user) => async () => {
|
||||
console.log('Updating recs for', user.username)
|
||||
await updateWordScores(user, contracts)
|
||||
console.log('Updating feed for', user.username)
|
||||
await computeFeed(user, feedContracts)
|
||||
})
|
||||
)
|
||||
|
||||
console.log('Updating feed categories!')
|
||||
|
||||
await batchedWaitAll(
|
||||
users.map((user) => async () => {
|
||||
for (const category of CATEGORY_LIST) {
|
||||
const contracts = await getTaggedContracts(category)
|
||||
const feed = await computeFeed(user, contracts)
|
||||
await firestore
|
||||
.collection(`private-users/${user.id}/cache`)
|
||||
.doc(`feed-${category}`)
|
||||
.set({ feed })
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
updateFeed().then(() => process.exit())
|
||||
}
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
import * as functions from 'firebase-functions'
|
||||
import * as admin from 'firebase-admin'
|
||||
import { flatten, shuffle, sortBy, uniq, zip, zipObject } from 'lodash'
|
||||
|
||||
import { getValue, getValues } from './utils'
|
||||
import { Contract } from '../../common/contract'
|
||||
import { logInterpolation } from '../../common/util/math'
|
||||
import { DAY_MS } from '../../common/util/time'
|
||||
import {
|
||||
getProbability,
|
||||
getOutcomeProbability,
|
||||
getTopAnswer,
|
||||
} from '../../common/calculate'
|
||||
import { User } from '../../common/user'
|
||||
import {
|
||||
getContractScore,
|
||||
MAX_FEED_CONTRACTS,
|
||||
} from '../../common/recommended-contracts'
|
||||
import { callCloudFunction } from './call-cloud-function'
|
||||
import {
|
||||
getFeedContracts,
|
||||
getRecentBetsAndComments,
|
||||
getTaggedContracts,
|
||||
} from './get-feed-data'
|
||||
import { CATEGORY_LIST } from '../../common/categories'
|
||||
|
||||
const firestore = admin.firestore()
|
||||
|
||||
const BATCH_SIZE = 30
|
||||
const MAX_BATCHES = 50
|
||||
|
||||
const getUserBatches = async () => {
|
||||
const users = shuffle(await getValues<User>(firestore.collection('users')))
|
||||
const userBatches: User[][] = []
|
||||
for (let i = 0; i < users.length; i += BATCH_SIZE) {
|
||||
userBatches.push(users.slice(i, i + BATCH_SIZE))
|
||||
}
|
||||
|
||||
console.log('updating feed batches', MAX_BATCHES, 'of', userBatches.length)
|
||||
|
||||
return userBatches.slice(0, MAX_BATCHES)
|
||||
}
|
||||
|
||||
export const updateFeed = functions.pubsub
|
||||
.schedule('every 60 minutes')
|
||||
.onRun(async () => {
|
||||
const userBatches = await getUserBatches()
|
||||
|
||||
await Promise.all(
|
||||
userBatches.map((users) =>
|
||||
callCloudFunction('updateFeedBatch', { users })
|
||||
)
|
||||
)
|
||||
|
||||
console.log('updating category feed')
|
||||
|
||||
await Promise.all(
|
||||
CATEGORY_LIST.map((category) =>
|
||||
callCloudFunction('updateCategoryFeed', {
|
||||
category,
|
||||
})
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
export const updateFeedBatch = functions.https.onCall(
|
||||
async (data: { users: User[] }) => {
|
||||
const { users } = data
|
||||
const contracts = await getFeedContracts()
|
||||
const feeds = await getNewFeeds(users, contracts)
|
||||
await Promise.all(
|
||||
zip(users, feeds).map(([user, feed]) =>
|
||||
/* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
|
||||
getUserCacheCollection(user!).doc('feed').set({ feed })
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
export const updateCategoryFeed = functions.https.onCall(
|
||||
async (data: { category: string }) => {
|
||||
const { category } = data
|
||||
const userBatches = await getUserBatches()
|
||||
|
||||
await Promise.all(
|
||||
userBatches.map(async (users) => {
|
||||
await callCloudFunction('updateCategoryFeedBatch', {
|
||||
users,
|
||||
category,
|
||||
})
|
||||
})
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
export const updateCategoryFeedBatch = functions.https.onCall(
|
||||
async (data: { users: User[]; category: string }) => {
|
||||
const { users, category } = data
|
||||
const contracts = await getTaggedContracts(category)
|
||||
const feeds = await getNewFeeds(users, contracts)
|
||||
await Promise.all(
|
||||
zip(users, feeds).map(([user, feed]) =>
|
||||
/* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
|
||||
getUserCacheCollection(user!).doc(`feed-${category}`).set({ feed })
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
const getNewFeeds = async (users: User[], contracts: Contract[]) => {
|
||||
const feeds = await Promise.all(users.map((u) => computeFeed(u, contracts)))
|
||||
const contractIds = uniq(flatten(feeds).map((c) => c.id))
|
||||
const data = await Promise.all(contractIds.map(getRecentBetsAndComments))
|
||||
const dataByContractId = zipObject(contractIds, data)
|
||||
return feeds.map((feed) =>
|
||||
feed.map((contract) => {
|
||||
return { contract, ...dataByContractId[contract.id] }
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
const getUserCacheCollection = (user: User) =>
|
||||
firestore.collection(`private-users/${user.id}/cache`)
|
||||
|
||||
export const computeFeed = async (user: User, contracts: Contract[]) => {
|
||||
const userCacheCollection = getUserCacheCollection(user)
|
||||
|
||||
const [wordScores, lastViewedTime] = await Promise.all([
|
||||
getValue<{ [word: string]: number }>(userCacheCollection.doc('wordScores')),
|
||||
getValue<{ [contractId: string]: number }>(
|
||||
userCacheCollection.doc('lastViewTime')
|
||||
),
|
||||
]).then((dicts) => dicts.map((dict) => dict ?? {}))
|
||||
|
||||
const scoredContracts = contracts.map((contract) => {
|
||||
const score = scoreContract(
|
||||
contract,
|
||||
wordScores,
|
||||
lastViewedTime[contract.id]
|
||||
)
|
||||
return [contract, score] as [Contract, number]
|
||||
})
|
||||
|
||||
const sortedContracts = sortBy(
|
||||
scoredContracts,
|
||||
([_, score]) => score
|
||||
).reverse()
|
||||
|
||||
// console.log(sortedContracts.map(([c, score]) => c.question + ': ' + score))
|
||||
|
||||
return sortedContracts.slice(0, MAX_FEED_CONTRACTS).map(([c]) => c)
|
||||
}
|
||||
|
||||
function scoreContract(
|
||||
contract: Contract,
|
||||
wordScores: { [word: string]: number },
|
||||
viewTime: number | undefined
|
||||
) {
|
||||
const recommendationScore = getContractScore(contract, wordScores)
|
||||
const activityScore = getActivityScore(contract, viewTime)
|
||||
// const lastViewedScore = getLastViewedScore(viewTime)
|
||||
return recommendationScore * activityScore
|
||||
}
|
||||
|
||||
function getActivityScore(contract: Contract, viewTime: number | undefined) {
|
||||
const { createdTime, lastBetTime, lastCommentTime, outcomeType } = contract
|
||||
const hasNewComments =
|
||||
lastCommentTime && (!viewTime || lastCommentTime > viewTime)
|
||||
const newCommentScore = hasNewComments ? 1 : 0.5
|
||||
|
||||
const timeSinceLastComment = Date.now() - (lastCommentTime ?? createdTime)
|
||||
const commentDaysAgo = timeSinceLastComment / DAY_MS
|
||||
const commentTimeScore =
|
||||
0.25 + 0.75 * (1 - logInterpolation(0, 3, commentDaysAgo))
|
||||
|
||||
const timeSinceLastBet = Date.now() - (lastBetTime ?? createdTime)
|
||||
const betDaysAgo = timeSinceLastBet / DAY_MS
|
||||
const betTimeScore = 0.5 + 0.5 * (1 - logInterpolation(0, 3, betDaysAgo))
|
||||
|
||||
let prob = 0.5
|
||||
if (outcomeType === 'BINARY') {
|
||||
prob = getProbability(contract)
|
||||
} else if (outcomeType === 'FREE_RESPONSE') {
|
||||
const topAnswer = getTopAnswer(contract)
|
||||
if (topAnswer)
|
||||
prob = Math.max(0.5, getOutcomeProbability(contract, topAnswer.id))
|
||||
}
|
||||
const frac = 1 - Math.abs(prob - 0.5) ** 2 / 0.25
|
||||
const probScore = 0.5 + frac * 0.5
|
||||
|
||||
const { volume24Hours, volume7Days } = contract
|
||||
const combinedVolume = Math.log(volume24Hours + 1) + Math.log(volume7Days + 1)
|
||||
const volumeScore = 0.5 + 0.5 * logInterpolation(4, 20, combinedVolume)
|
||||
|
||||
const score =
|
||||
newCommentScore * commentTimeScore * betTimeScore * probScore * volumeScore
|
||||
|
||||
// Map score to [0.5, 1] since no recent activty is not a deal breaker.
|
||||
const mappedScore = 0.5 + 0.5 * score
|
||||
const newMappedScore = 0.7 + 0.3 * score
|
||||
|
||||
const isNew = Date.now() < contract.createdTime + DAY_MS
|
||||
return isNew ? newMappedScore : mappedScore
|
||||
}
|
||||
|
||||
// function getLastViewedScore(viewTime: number | undefined) {
|
||||
// if (viewTime === undefined) {
|
||||
// return 1
|
||||
// }
|
||||
|
||||
// const daysAgo = (Date.now() - viewTime) / DAY_MS
|
||||
|
||||
// if (daysAgo < 0.5) {
|
||||
// const frac = logInterpolation(0, 0.5, daysAgo)
|
||||
// return 0.5 + 0.25 * frac
|
||||
// }
|
||||
|
||||
// const frac = logInterpolation(0.5, 14, daysAgo)
|
||||
// return 0.75 + 0.25 * frac
|
||||
// }
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
import * as functions from 'firebase-functions'
|
||||
import * as admin from 'firebase-admin'
|
||||
|
||||
import { getValue, getValues } from './utils'
|
||||
import { Contract } from '../../common/contract'
|
||||
import { Bet } from '../../common/bet'
|
||||
import { User } from '../../common/user'
|
||||
import { ClickEvent } from '../../common/tracking'
|
||||
import { getWordScores } from '../../common/recommended-contracts'
|
||||
import { batchedWaitAll } from '../../common/util/promise'
|
||||
import { callCloudFunction } from './call-cloud-function'
|
||||
|
||||
const firestore = admin.firestore()
|
||||
|
||||
export const updateRecommendations = functions.pubsub
|
||||
.schedule('every 24 hours')
|
||||
.onRun(async () => {
|
||||
const users = await getValues<User>(firestore.collection('users'))
|
||||
|
||||
const batchSize = 100
|
||||
const userBatches: User[][] = []
|
||||
for (let i = 0; i < users.length; i += batchSize) {
|
||||
userBatches.push(users.slice(i, i + batchSize))
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
userBatches.map((batch) =>
|
||||
callCloudFunction('updateRecommendationsBatch', { users: batch })
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
export const updateRecommendationsBatch = functions.https.onCall(
|
||||
async (data: { users: User[] }) => {
|
||||
const { users } = data
|
||||
|
||||
const contracts = await getValues<Contract>(
|
||||
firestore.collection('contracts')
|
||||
)
|
||||
|
||||
await batchedWaitAll(
|
||||
users.map((user) => () => updateWordScores(user, contracts))
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
export const updateWordScores = async (user: User, contracts: Contract[]) => {
|
||||
const [bets, viewCounts, clicks] = await Promise.all([
|
||||
getValues<Bet>(
|
||||
firestore.collectionGroup('bets').where('userId', '==', user.id)
|
||||
),
|
||||
|
||||
getValue<{ [contractId: string]: number }>(
|
||||
firestore.doc(`private-users/${user.id}/cache/viewCounts`)
|
||||
),
|
||||
|
||||
getValues<ClickEvent>(
|
||||
firestore
|
||||
.collection(`private-users/${user.id}/events`)
|
||||
.where('type', '==', 'click')
|
||||
),
|
||||
])
|
||||
|
||||
const wordScores = getWordScores(contracts, viewCounts ?? {}, clicks, bets)
|
||||
|
||||
const cachedCollection = firestore.collection(
|
||||
`private-users/${user.id}/cache`
|
||||
)
|
||||
await cachedCollection.doc('wordScores').set(wordScores)
|
||||
}
|
||||
29
yarn.lock
29
yarn.lock
|
|
@ -3875,13 +3875,6 @@ binary-extensions@^2.0.0:
|
|||
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
|
||||
integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
|
||||
|
||||
biskviit@1.0.1:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/biskviit/-/biskviit-1.0.1.tgz#037a0cd4b71b9e331fd90a1122de17dc49e420a7"
|
||||
integrity sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w==
|
||||
dependencies:
|
||||
psl "^1.1.7"
|
||||
|
||||
bluebird@^3.7.1:
|
||||
version "3.7.2"
|
||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
|
||||
|
|
@ -5237,13 +5230,6 @@ encodeurl@~1.0.2:
|
|||
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
|
||||
integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=
|
||||
|
||||
encoding@0.1.12:
|
||||
version "0.1.12"
|
||||
resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb"
|
||||
integrity sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=
|
||||
dependencies:
|
||||
iconv-lite "~0.4.13"
|
||||
|
||||
end-of-stream@^1.1.0, end-of-stream@^1.4.1:
|
||||
version "1.4.4"
|
||||
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
|
||||
|
|
@ -5817,14 +5803,6 @@ fetch-blob@^3.1.2, fetch-blob@^3.1.4:
|
|||
node-domexception "^1.0.0"
|
||||
web-streams-polyfill "^3.0.3"
|
||||
|
||||
fetch@1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/fetch/-/fetch-1.1.0.tgz#0a8279f06be37f9f0ebb567560a30a480da59a2e"
|
||||
integrity sha1-CoJ58Gvjf58Ou1Z1YKMKSA2lmi4=
|
||||
dependencies:
|
||||
biskviit "1.0.1"
|
||||
encoding "0.1.12"
|
||||
|
||||
file-entry-cache@^6.0.1:
|
||||
version "6.0.1"
|
||||
resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
|
||||
|
|
@ -6782,7 +6760,7 @@ human-signals@^2.1.0:
|
|||
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
|
||||
integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==
|
||||
|
||||
iconv-lite@0.4.24, iconv-lite@~0.4.13:
|
||||
iconv-lite@0.4.24:
|
||||
version "0.4.24"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
|
||||
integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
|
||||
|
|
@ -9151,11 +9129,6 @@ pseudomap@^1.0.1:
|
|||
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
|
||||
integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
|
||||
|
||||
psl@^1.1.7:
|
||||
version "1.8.0"
|
||||
resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
|
||||
integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
|
||||
|
||||
pump@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user