Remove code for obsolete feed updater backend jobs
This commit is contained in:
Marshall Polaris
parent
b0b8c6e98b
commit
e10d21d778
|
|
@ -23,7 +23,6 @@
|
|||
"main": "functions/src/index.js",
|
||||
"dependencies": {
|
||||
"@amplitude/node": "1.10.0",
|
||||
"fetch": "1.1.0",
|
||||
"firebase-admin": "10.0.0",
|
||||
"firebase-functions": "3.21.2",
|
||||
"lodash": "4.17.21",
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
import * as admin from 'firebase-admin'
|
||||
|
||||
import fetch from './fetch'
|
||||
|
||||
export const callCloudFunction = (functionName: string, data: unknown = {}) => {
|
||||
const projectId = admin.instanceId().app.options.projectId
|
||||
|
||||
const url = `https://us-central1-${projectId}.cloudfunctions.net/${functionName}`
|
||||
|
||||
return fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ data }),
|
||||
}).then((response) => response.json())
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
let fetchRequest: typeof fetch
|
||||
|
||||
try {
|
||||
fetchRequest = fetch
|
||||
} catch {
|
||||
fetchRequest = require('node-fetch')
|
||||
}
|
||||
|
||||
export default fetchRequest
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
import * as admin from 'firebase-admin'
|
||||
|
||||
import { initAdmin } from './script-init'
|
||||
initAdmin()
|
||||
|
||||
import { getValues } from '../utils'
|
||||
import { User } from '../../../common/user'
|
||||
import { batchedWaitAll } from '../../../common/util/promise'
|
||||
import { Contract } from '../../../common/contract'
|
||||
import { updateWordScores } from '../update-recommendations'
|
||||
import { computeFeed } from '../update-feed'
|
||||
import { getFeedContracts, getTaggedContracts } from '../get-feed-data'
|
||||
import { CATEGORY_LIST } from '../../../common/categories'
|
||||
|
||||
const firestore = admin.firestore()
|
||||
|
||||
async function updateFeed() {
|
||||
console.log('Updating feed')
|
||||
|
||||
const contracts = await getValues<Contract>(firestore.collection('contracts'))
|
||||
const feedContracts = await getFeedContracts()
|
||||
const users = await getValues<User>(
|
||||
firestore.collection('users').where('username', '==', 'JamesGrugett')
|
||||
)
|
||||
|
||||
await batchedWaitAll(
|
||||
users.map((user) => async () => {
|
||||
console.log('Updating recs for', user.username)
|
||||
await updateWordScores(user, contracts)
|
||||
console.log('Updating feed for', user.username)
|
||||
await computeFeed(user, feedContracts)
|
||||
})
|
||||
)
|
||||
|
||||
console.log('Updating feed categories!')
|
||||
|
||||
await batchedWaitAll(
|
||||
users.map((user) => async () => {
|
||||
for (const category of CATEGORY_LIST) {
|
||||
const contracts = await getTaggedContracts(category)
|
||||
const feed = await computeFeed(user, contracts)
|
||||
await firestore
|
||||
.collection(`private-users/${user.id}/cache`)
|
||||
.doc(`feed-${category}`)
|
||||
.set({ feed })
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
updateFeed().then(() => process.exit())
|
||||
}
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
import * as functions from 'firebase-functions'
|
||||
import * as admin from 'firebase-admin'
|
||||
|
||||
import { getValue, getValues } from './utils'
|
||||
import { Contract } from '../../common/contract'
|
||||
import { Bet } from '../../common/bet'
|
||||
import { User } from '../../common/user'
|
||||
import { ClickEvent } from '../../common/tracking'
|
||||
import { getWordScores } from '../../common/recommended-contracts'
|
||||
import { batchedWaitAll } from '../../common/util/promise'
|
||||
import { callCloudFunction } from './call-cloud-function'
|
||||
|
||||
const firestore = admin.firestore()
|
||||
|
||||
export const updateRecommendations = functions.pubsub
|
||||
.schedule('every 24 hours')
|
||||
.onRun(async () => {
|
||||
const users = await getValues<User>(firestore.collection('users'))
|
||||
|
||||
const batchSize = 100
|
||||
const userBatches: User[][] = []
|
||||
for (let i = 0; i < users.length; i += batchSize) {
|
||||
userBatches.push(users.slice(i, i + batchSize))
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
userBatches.map((batch) =>
|
||||
callCloudFunction('updateRecommendationsBatch', { users: batch })
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
export const updateRecommendationsBatch = functions.https.onCall(
|
||||
async (data: { users: User[] }) => {
|
||||
const { users } = data
|
||||
|
||||
const contracts = await getValues<Contract>(
|
||||
firestore.collection('contracts')
|
||||
)
|
||||
|
||||
await batchedWaitAll(
|
||||
users.map((user) => () => updateWordScores(user, contracts))
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
export const updateWordScores = async (user: User, contracts: Contract[]) => {
|
||||
const [bets, viewCounts, clicks] = await Promise.all([
|
||||
getValues<Bet>(
|
||||
firestore.collectionGroup('bets').where('userId', '==', user.id)
|
||||
),
|
||||
|
||||
getValue<{ [contractId: string]: number }>(
|
||||
firestore.doc(`private-users/${user.id}/cache/viewCounts`)
|
||||
),
|
||||
|
||||
getValues<ClickEvent>(
|
||||
firestore
|
||||
.collection(`private-users/${user.id}/events`)
|
||||
.where('type', '==', 'click')
|
||||
),
|
||||
])
|
||||
|
||||
const wordScores = getWordScores(contracts, viewCounts ?? {}, clicks, bets)
|
||||
|
||||
const cachedCollection = firestore.collection(
|
||||
`private-users/${user.id}/cache`
|
||||
)
|
||||
await cachedCollection.doc('wordScores').set(wordScores)
|
||||
}
|
||||
29
yarn.lock
29
yarn.lock
|
|
@ -3875,13 +3875,6 @@ binary-extensions@^2.0.0:
|
|||
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
|
||||
integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
|
||||
|
||||
biskviit@1.0.1:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/biskviit/-/biskviit-1.0.1.tgz#037a0cd4b71b9e331fd90a1122de17dc49e420a7"
|
||||
integrity sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w==
|
||||
dependencies:
|
||||
psl "^1.1.7"
|
||||
|
||||
bluebird@^3.7.1:
|
||||
version "3.7.2"
|
||||
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
|
||||
|
|
@ -5237,13 +5230,6 @@ encodeurl@~1.0.2:
|
|||
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
|
||||
integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=
|
||||
|
||||
encoding@0.1.12:
|
||||
version "0.1.12"
|
||||
resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb"
|
||||
integrity sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=
|
||||
dependencies:
|
||||
iconv-lite "~0.4.13"
|
||||
|
||||
end-of-stream@^1.1.0, end-of-stream@^1.4.1:
|
||||
version "1.4.4"
|
||||
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
|
||||
|
|
@ -5817,14 +5803,6 @@ fetch-blob@^3.1.2, fetch-blob@^3.1.4:
|
|||
node-domexception "^1.0.0"
|
||||
web-streams-polyfill "^3.0.3"
|
||||
|
||||
fetch@1.1.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/fetch/-/fetch-1.1.0.tgz#0a8279f06be37f9f0ebb567560a30a480da59a2e"
|
||||
integrity sha1-CoJ58Gvjf58Ou1Z1YKMKSA2lmi4=
|
||||
dependencies:
|
||||
biskviit "1.0.1"
|
||||
encoding "0.1.12"
|
||||
|
||||
file-entry-cache@^6.0.1:
|
||||
version "6.0.1"
|
||||
resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
|
||||
|
|
@ -6782,7 +6760,7 @@ human-signals@^2.1.0:
|
|||
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
|
||||
integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==
|
||||
|
||||
iconv-lite@0.4.24, iconv-lite@~0.4.13:
|
||||
iconv-lite@0.4.24:
|
||||
version "0.4.24"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
|
||||
integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
|
||||
|
|
@ -9151,11 +9129,6 @@ pseudomap@^1.0.1:
|
|||
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
|
||||
integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
|
||||
|
||||
psl@^1.1.7:
|
||||
version "1.8.0"
|
||||
resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
|
||||
integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
|
||||
|
||||
pump@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user