Remove code for obsolete feed updater backend jobs (#607)

* Remove code for obsolete feed updater backend jobs

* Kill two more obsolete guys
This commit is contained in:
Marshall Polaris 2022-07-02 13:26:42 -07:00 committed by GitHub
parent 218b18254c
commit 18b8758191
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1 additions and 423 deletions

View File

@ -23,7 +23,6 @@
"main": "functions/src/index.js", "main": "functions/src/index.js",
"dependencies": { "dependencies": {
"@amplitude/node": "1.10.0", "@amplitude/node": "1.10.0",
"fetch": "1.1.0",
"firebase-admin": "10.0.0", "firebase-admin": "10.0.0",
"firebase-functions": "3.21.2", "firebase-functions": "3.21.2",
"lodash": "4.17.21", "lodash": "4.17.21",

View File

@ -1,17 +0,0 @@
import * as admin from 'firebase-admin'
import fetch from './fetch'
export const callCloudFunction = (functionName: string, data: unknown = {}) => {
const projectId = admin.instanceId().app.options.projectId
const url = `https://us-central1-${projectId}.cloudfunctions.net/${functionName}`
return fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ data }),
}).then((response) => response.json())
}

View File

@ -1,9 +0,0 @@
let fetchRequest: typeof fetch
try {
fetchRequest = fetch
} catch {
fetchRequest = require('node-fetch')
}
export default fetchRequest

View File

@ -1,25 +0,0 @@
import * as functions from 'firebase-functions'
import { callCloudFunction } from './call-cloud-function'
export const keepAwake = functions.pubsub
.schedule('every 1 minutes')
.onRun(async () => {
await Promise.all([
callCloudFunction('placeBet'),
callCloudFunction('resolveMarket'),
callCloudFunction('sellBet'),
])
await sleep(30)
await Promise.all([
callCloudFunction('placeBet'),
callCloudFunction('resolveMarket'),
callCloudFunction('sellBet'),
])
})
const sleep = (seconds: number) => {
return new Promise((resolve) => setTimeout(resolve, seconds * 1000))
}

View File

@ -1,53 +0,0 @@
import * as admin from 'firebase-admin'
import { initAdmin } from './script-init'
initAdmin()
import { getValues } from '../utils'
import { User } from '../../../common/user'
import { batchedWaitAll } from '../../../common/util/promise'
import { Contract } from '../../../common/contract'
import { updateWordScores } from '../update-recommendations'
import { computeFeed } from '../update-feed'
import { getFeedContracts, getTaggedContracts } from '../get-feed-data'
import { CATEGORY_LIST } from '../../../common/categories'
const firestore = admin.firestore()
async function updateFeed() {
console.log('Updating feed')
const contracts = await getValues<Contract>(firestore.collection('contracts'))
const feedContracts = await getFeedContracts()
const users = await getValues<User>(
firestore.collection('users').where('username', '==', 'JamesGrugett')
)
await batchedWaitAll(
users.map((user) => async () => {
console.log('Updating recs for', user.username)
await updateWordScores(user, contracts)
console.log('Updating feed for', user.username)
await computeFeed(user, feedContracts)
})
)
console.log('Updating feed categories!')
await batchedWaitAll(
users.map((user) => async () => {
for (const category of CATEGORY_LIST) {
const contracts = await getTaggedContracts(category)
const feed = await computeFeed(user, contracts)
await firestore
.collection(`private-users/${user.id}/cache`)
.doc(`feed-${category}`)
.set({ feed })
}
})
)
}
if (require.main === module) {
updateFeed().then(() => process.exit())
}

View File

@ -1,220 +0,0 @@
import * as functions from 'firebase-functions'
import * as admin from 'firebase-admin'
import { flatten, shuffle, sortBy, uniq, zip, zipObject } from 'lodash'
import { getValue, getValues } from './utils'
import { Contract } from '../../common/contract'
import { logInterpolation } from '../../common/util/math'
import { DAY_MS } from '../../common/util/time'
import {
getProbability,
getOutcomeProbability,
getTopAnswer,
} from '../../common/calculate'
import { User } from '../../common/user'
import {
getContractScore,
MAX_FEED_CONTRACTS,
} from '../../common/recommended-contracts'
import { callCloudFunction } from './call-cloud-function'
import {
getFeedContracts,
getRecentBetsAndComments,
getTaggedContracts,
} from './get-feed-data'
import { CATEGORY_LIST } from '../../common/categories'
const firestore = admin.firestore()
const BATCH_SIZE = 30
const MAX_BATCHES = 50
const getUserBatches = async () => {
const users = shuffle(await getValues<User>(firestore.collection('users')))
const userBatches: User[][] = []
for (let i = 0; i < users.length; i += BATCH_SIZE) {
userBatches.push(users.slice(i, i + BATCH_SIZE))
}
console.log('updating feed batches', MAX_BATCHES, 'of', userBatches.length)
return userBatches.slice(0, MAX_BATCHES)
}
export const updateFeed = functions.pubsub
.schedule('every 60 minutes')
.onRun(async () => {
const userBatches = await getUserBatches()
await Promise.all(
userBatches.map((users) =>
callCloudFunction('updateFeedBatch', { users })
)
)
console.log('updating category feed')
await Promise.all(
CATEGORY_LIST.map((category) =>
callCloudFunction('updateCategoryFeed', {
category,
})
)
)
})
export const updateFeedBatch = functions.https.onCall(
async (data: { users: User[] }) => {
const { users } = data
const contracts = await getFeedContracts()
const feeds = await getNewFeeds(users, contracts)
await Promise.all(
zip(users, feeds).map(([user, feed]) =>
/* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
getUserCacheCollection(user!).doc('feed').set({ feed })
)
)
}
)
export const updateCategoryFeed = functions.https.onCall(
async (data: { category: string }) => {
const { category } = data
const userBatches = await getUserBatches()
await Promise.all(
userBatches.map(async (users) => {
await callCloudFunction('updateCategoryFeedBatch', {
users,
category,
})
})
)
}
)
export const updateCategoryFeedBatch = functions.https.onCall(
async (data: { users: User[]; category: string }) => {
const { users, category } = data
const contracts = await getTaggedContracts(category)
const feeds = await getNewFeeds(users, contracts)
await Promise.all(
zip(users, feeds).map(([user, feed]) =>
/* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
getUserCacheCollection(user!).doc(`feed-${category}`).set({ feed })
)
)
}
)
const getNewFeeds = async (users: User[], contracts: Contract[]) => {
const feeds = await Promise.all(users.map((u) => computeFeed(u, contracts)))
const contractIds = uniq(flatten(feeds).map((c) => c.id))
const data = await Promise.all(contractIds.map(getRecentBetsAndComments))
const dataByContractId = zipObject(contractIds, data)
return feeds.map((feed) =>
feed.map((contract) => {
return { contract, ...dataByContractId[contract.id] }
})
)
}
const getUserCacheCollection = (user: User) =>
firestore.collection(`private-users/${user.id}/cache`)
export const computeFeed = async (user: User, contracts: Contract[]) => {
const userCacheCollection = getUserCacheCollection(user)
const [wordScores, lastViewedTime] = await Promise.all([
getValue<{ [word: string]: number }>(userCacheCollection.doc('wordScores')),
getValue<{ [contractId: string]: number }>(
userCacheCollection.doc('lastViewTime')
),
]).then((dicts) => dicts.map((dict) => dict ?? {}))
const scoredContracts = contracts.map((contract) => {
const score = scoreContract(
contract,
wordScores,
lastViewedTime[contract.id]
)
return [contract, score] as [Contract, number]
})
const sortedContracts = sortBy(
scoredContracts,
([_, score]) => score
).reverse()
// console.log(sortedContracts.map(([c, score]) => c.question + ': ' + score))
return sortedContracts.slice(0, MAX_FEED_CONTRACTS).map(([c]) => c)
}
function scoreContract(
contract: Contract,
wordScores: { [word: string]: number },
viewTime: number | undefined
) {
const recommendationScore = getContractScore(contract, wordScores)
const activityScore = getActivityScore(contract, viewTime)
// const lastViewedScore = getLastViewedScore(viewTime)
return recommendationScore * activityScore
}
function getActivityScore(contract: Contract, viewTime: number | undefined) {
const { createdTime, lastBetTime, lastCommentTime, outcomeType } = contract
const hasNewComments =
lastCommentTime && (!viewTime || lastCommentTime > viewTime)
const newCommentScore = hasNewComments ? 1 : 0.5
const timeSinceLastComment = Date.now() - (lastCommentTime ?? createdTime)
const commentDaysAgo = timeSinceLastComment / DAY_MS
const commentTimeScore =
0.25 + 0.75 * (1 - logInterpolation(0, 3, commentDaysAgo))
const timeSinceLastBet = Date.now() - (lastBetTime ?? createdTime)
const betDaysAgo = timeSinceLastBet / DAY_MS
const betTimeScore = 0.5 + 0.5 * (1 - logInterpolation(0, 3, betDaysAgo))
let prob = 0.5
if (outcomeType === 'BINARY') {
prob = getProbability(contract)
} else if (outcomeType === 'FREE_RESPONSE') {
const topAnswer = getTopAnswer(contract)
if (topAnswer)
prob = Math.max(0.5, getOutcomeProbability(contract, topAnswer.id))
}
const frac = 1 - Math.abs(prob - 0.5) ** 2 / 0.25
const probScore = 0.5 + frac * 0.5
const { volume24Hours, volume7Days } = contract
const combinedVolume = Math.log(volume24Hours + 1) + Math.log(volume7Days + 1)
const volumeScore = 0.5 + 0.5 * logInterpolation(4, 20, combinedVolume)
const score =
newCommentScore * commentTimeScore * betTimeScore * probScore * volumeScore
// Map score to [0.5, 1] since no recent activty is not a deal breaker.
const mappedScore = 0.5 + 0.5 * score
const newMappedScore = 0.7 + 0.3 * score
const isNew = Date.now() < contract.createdTime + DAY_MS
return isNew ? newMappedScore : mappedScore
}
// function getLastViewedScore(viewTime: number | undefined) {
// if (viewTime === undefined) {
// return 1
// }
// const daysAgo = (Date.now() - viewTime) / DAY_MS
// if (daysAgo < 0.5) {
// const frac = logInterpolation(0, 0.5, daysAgo)
// return 0.5 + 0.25 * frac
// }
// const frac = logInterpolation(0.5, 14, daysAgo)
// return 0.75 + 0.25 * frac
// }

View File

@ -1,70 +0,0 @@
import * as functions from 'firebase-functions'
import * as admin from 'firebase-admin'
import { getValue, getValues } from './utils'
import { Contract } from '../../common/contract'
import { Bet } from '../../common/bet'
import { User } from '../../common/user'
import { ClickEvent } from '../../common/tracking'
import { getWordScores } from '../../common/recommended-contracts'
import { batchedWaitAll } from '../../common/util/promise'
import { callCloudFunction } from './call-cloud-function'
const firestore = admin.firestore()
export const updateRecommendations = functions.pubsub
.schedule('every 24 hours')
.onRun(async () => {
const users = await getValues<User>(firestore.collection('users'))
const batchSize = 100
const userBatches: User[][] = []
for (let i = 0; i < users.length; i += batchSize) {
userBatches.push(users.slice(i, i + batchSize))
}
await Promise.all(
userBatches.map((batch) =>
callCloudFunction('updateRecommendationsBatch', { users: batch })
)
)
})
export const updateRecommendationsBatch = functions.https.onCall(
async (data: { users: User[] }) => {
const { users } = data
const contracts = await getValues<Contract>(
firestore.collection('contracts')
)
await batchedWaitAll(
users.map((user) => () => updateWordScores(user, contracts))
)
}
)
export const updateWordScores = async (user: User, contracts: Contract[]) => {
const [bets, viewCounts, clicks] = await Promise.all([
getValues<Bet>(
firestore.collectionGroup('bets').where('userId', '==', user.id)
),
getValue<{ [contractId: string]: number }>(
firestore.doc(`private-users/${user.id}/cache/viewCounts`)
),
getValues<ClickEvent>(
firestore
.collection(`private-users/${user.id}/events`)
.where('type', '==', 'click')
),
])
const wordScores = getWordScores(contracts, viewCounts ?? {}, clicks, bets)
const cachedCollection = firestore.collection(
`private-users/${user.id}/cache`
)
await cachedCollection.doc('wordScores').set(wordScores)
}

View File

@ -3875,13 +3875,6 @@ binary-extensions@^2.0.0:
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
biskviit@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/biskviit/-/biskviit-1.0.1.tgz#037a0cd4b71b9e331fd90a1122de17dc49e420a7"
integrity sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w==
dependencies:
psl "^1.1.7"
bluebird@^3.7.1: bluebird@^3.7.1:
version "3.7.2" version "3.7.2"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@ -5237,13 +5230,6 @@ encodeurl@~1.0.2:
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=
encoding@0.1.12:
version "0.1.12"
resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb"
integrity sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=
dependencies:
iconv-lite "~0.4.13"
end-of-stream@^1.1.0, end-of-stream@^1.4.1: end-of-stream@^1.1.0, end-of-stream@^1.4.1:
version "1.4.4" version "1.4.4"
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
@ -5817,14 +5803,6 @@ fetch-blob@^3.1.2, fetch-blob@^3.1.4:
node-domexception "^1.0.0" node-domexception "^1.0.0"
web-streams-polyfill "^3.0.3" web-streams-polyfill "^3.0.3"
fetch@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/fetch/-/fetch-1.1.0.tgz#0a8279f06be37f9f0ebb567560a30a480da59a2e"
integrity sha1-CoJ58Gvjf58Ou1Z1YKMKSA2lmi4=
dependencies:
biskviit "1.0.1"
encoding "0.1.12"
file-entry-cache@^6.0.1: file-entry-cache@^6.0.1:
version "6.0.1" version "6.0.1"
resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
@ -6782,7 +6760,7 @@ human-signals@^2.1.0:
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==
iconv-lite@0.4.24, iconv-lite@~0.4.13: iconv-lite@0.4.24:
version "0.4.24" version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
@ -9151,11 +9129,6 @@ pseudomap@^1.0.1:
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
psl@^1.1.7:
version "1.8.0"
resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
pump@^3.0.0: pump@^3.0.0:
version "3.0.0" version "3.0.0"
resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"