diff --git a/functions/package.json b/functions/package.json index eb6c7151..ed12b4e7 100644 --- a/functions/package.json +++ b/functions/package.json @@ -23,7 +23,6 @@ "main": "functions/src/index.js", "dependencies": { "@amplitude/node": "1.10.0", - "fetch": "1.1.0", "firebase-admin": "10.0.0", "firebase-functions": "3.21.2", "lodash": "4.17.21", diff --git a/functions/src/call-cloud-function.ts b/functions/src/call-cloud-function.ts deleted file mode 100644 index 35191343..00000000 --- a/functions/src/call-cloud-function.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as admin from 'firebase-admin' - -import fetch from './fetch' - -export const callCloudFunction = (functionName: string, data: unknown = {}) => { - const projectId = admin.instanceId().app.options.projectId - - const url = `https://us-central1-${projectId}.cloudfunctions.net/${functionName}` - - return fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ data }), - }).then((response) => response.json()) -} diff --git a/functions/src/fetch.ts b/functions/src/fetch.ts deleted file mode 100644 index 1b54dc6c..00000000 --- a/functions/src/fetch.ts +++ /dev/null @@ -1,9 +0,0 @@ -let fetchRequest: typeof fetch - -try { - fetchRequest = fetch -} catch { - fetchRequest = require('node-fetch') -} - -export default fetchRequest diff --git a/functions/src/keep-awake.ts b/functions/src/keep-awake.ts deleted file mode 100644 index 00799e65..00000000 --- a/functions/src/keep-awake.ts +++ /dev/null @@ -1,25 +0,0 @@ -import * as functions from 'firebase-functions' - -import { callCloudFunction } from './call-cloud-function' - -export const keepAwake = functions.pubsub - .schedule('every 1 minutes') - .onRun(async () => { - await Promise.all([ - callCloudFunction('placeBet'), - callCloudFunction('resolveMarket'), - callCloudFunction('sellBet'), - ]) - - await sleep(30) - - await Promise.all([ - callCloudFunction('placeBet'), - callCloudFunction('resolveMarket'), - callCloudFunction('sellBet'), - ]) - }) - -const sleep = (seconds: number) => { - return new Promise((resolve) => setTimeout(resolve, seconds * 1000)) -} diff --git a/functions/src/scripts/update-feed.ts b/functions/src/scripts/update-feed.ts deleted file mode 100644 index c5cba142..00000000 --- a/functions/src/scripts/update-feed.ts +++ /dev/null @@ -1,53 +0,0 @@ -import * as admin from 'firebase-admin' - -import { initAdmin } from './script-init' -initAdmin() - -import { getValues } from '../utils' -import { User } from '../../../common/user' -import { batchedWaitAll } from '../../../common/util/promise' -import { Contract } from '../../../common/contract' -import { updateWordScores } from '../update-recommendations' -import { computeFeed } from '../update-feed' -import { getFeedContracts, getTaggedContracts } from '../get-feed-data' -import { CATEGORY_LIST } from '../../../common/categories' - -const firestore = admin.firestore() - -async function updateFeed() { - console.log('Updating feed') - - const contracts = await getValues(firestore.collection('contracts')) - const feedContracts = await getFeedContracts() - const users = await getValues( - firestore.collection('users').where('username', '==', 'JamesGrugett') - ) - - await batchedWaitAll( - users.map((user) => async () => { - console.log('Updating recs for', user.username) - await updateWordScores(user, contracts) - console.log('Updating feed for', user.username) - await computeFeed(user, feedContracts) - }) - ) - - console.log('Updating feed categories!') - - await batchedWaitAll( - users.map((user) => async () => { - for (const category of CATEGORY_LIST) { - const contracts = await getTaggedContracts(category) - const feed = await computeFeed(user, contracts) - await firestore - .collection(`private-users/${user.id}/cache`) - .doc(`feed-${category}`) - .set({ feed }) - } - }) - ) -} - -if (require.main === module) { - updateFeed().then(() => process.exit()) -} diff --git a/functions/src/update-feed.ts b/functions/src/update-feed.ts deleted file mode 100644 index f19fda92..00000000 --- a/functions/src/update-feed.ts +++ /dev/null @@ -1,220 +0,0 @@ -import * as functions from 'firebase-functions' -import * as admin from 'firebase-admin' -import { flatten, shuffle, sortBy, uniq, zip, zipObject } from 'lodash' - -import { getValue, getValues } from './utils' -import { Contract } from '../../common/contract' -import { logInterpolation } from '../../common/util/math' -import { DAY_MS } from '../../common/util/time' -import { - getProbability, - getOutcomeProbability, - getTopAnswer, -} from '../../common/calculate' -import { User } from '../../common/user' -import { - getContractScore, - MAX_FEED_CONTRACTS, -} from '../../common/recommended-contracts' -import { callCloudFunction } from './call-cloud-function' -import { - getFeedContracts, - getRecentBetsAndComments, - getTaggedContracts, -} from './get-feed-data' -import { CATEGORY_LIST } from '../../common/categories' - -const firestore = admin.firestore() - -const BATCH_SIZE = 30 -const MAX_BATCHES = 50 - -const getUserBatches = async () => { - const users = shuffle(await getValues(firestore.collection('users'))) - const userBatches: User[][] = [] - for (let i = 0; i < users.length; i += BATCH_SIZE) { - userBatches.push(users.slice(i, i + BATCH_SIZE)) - } - - console.log('updating feed batches', MAX_BATCHES, 'of', userBatches.length) - - return userBatches.slice(0, MAX_BATCHES) -} - -export const updateFeed = functions.pubsub - .schedule('every 60 minutes') - .onRun(async () => { - const userBatches = await getUserBatches() - - await Promise.all( - userBatches.map((users) => - callCloudFunction('updateFeedBatch', { users }) - ) - ) - - console.log('updating category feed') - - await Promise.all( - CATEGORY_LIST.map((category) => - callCloudFunction('updateCategoryFeed', { - category, - }) - ) - ) - }) - -export const updateFeedBatch = functions.https.onCall( - async (data: { users: User[] }) => { - const { users } = data - const contracts = await getFeedContracts() - const feeds = await getNewFeeds(users, contracts) - await Promise.all( - zip(users, feeds).map(([user, feed]) => - /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */ - getUserCacheCollection(user!).doc('feed').set({ feed }) - ) - ) - } -) - -export const updateCategoryFeed = functions.https.onCall( - async (data: { category: string }) => { - const { category } = data - const userBatches = await getUserBatches() - - await Promise.all( - userBatches.map(async (users) => { - await callCloudFunction('updateCategoryFeedBatch', { - users, - category, - }) - }) - ) - } -) - -export const updateCategoryFeedBatch = functions.https.onCall( - async (data: { users: User[]; category: string }) => { - const { users, category } = data - const contracts = await getTaggedContracts(category) - const feeds = await getNewFeeds(users, contracts) - await Promise.all( - zip(users, feeds).map(([user, feed]) => - /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */ - getUserCacheCollection(user!).doc(`feed-${category}`).set({ feed }) - ) - ) - } -) - -const getNewFeeds = async (users: User[], contracts: Contract[]) => { - const feeds = await Promise.all(users.map((u) => computeFeed(u, contracts))) - const contractIds = uniq(flatten(feeds).map((c) => c.id)) - const data = await Promise.all(contractIds.map(getRecentBetsAndComments)) - const dataByContractId = zipObject(contractIds, data) - return feeds.map((feed) => - feed.map((contract) => { - return { contract, ...dataByContractId[contract.id] } - }) - ) -} - -const getUserCacheCollection = (user: User) => - firestore.collection(`private-users/${user.id}/cache`) - -export const computeFeed = async (user: User, contracts: Contract[]) => { - const userCacheCollection = getUserCacheCollection(user) - - const [wordScores, lastViewedTime] = await Promise.all([ - getValue<{ [word: string]: number }>(userCacheCollection.doc('wordScores')), - getValue<{ [contractId: string]: number }>( - userCacheCollection.doc('lastViewTime') - ), - ]).then((dicts) => dicts.map((dict) => dict ?? {})) - - const scoredContracts = contracts.map((contract) => { - const score = scoreContract( - contract, - wordScores, - lastViewedTime[contract.id] - ) - return [contract, score] as [Contract, number] - }) - - const sortedContracts = sortBy( - scoredContracts, - ([_, score]) => score - ).reverse() - - // console.log(sortedContracts.map(([c, score]) => c.question + ': ' + score)) - - return sortedContracts.slice(0, MAX_FEED_CONTRACTS).map(([c]) => c) -} - -function scoreContract( - contract: Contract, - wordScores: { [word: string]: number }, - viewTime: number | undefined -) { - const recommendationScore = getContractScore(contract, wordScores) - const activityScore = getActivityScore(contract, viewTime) - // const lastViewedScore = getLastViewedScore(viewTime) - return recommendationScore * activityScore -} - -function getActivityScore(contract: Contract, viewTime: number | undefined) { - const { createdTime, lastBetTime, lastCommentTime, outcomeType } = contract - const hasNewComments = - lastCommentTime && (!viewTime || lastCommentTime > viewTime) - const newCommentScore = hasNewComments ? 1 : 0.5 - - const timeSinceLastComment = Date.now() - (lastCommentTime ?? createdTime) - const commentDaysAgo = timeSinceLastComment / DAY_MS - const commentTimeScore = - 0.25 + 0.75 * (1 - logInterpolation(0, 3, commentDaysAgo)) - - const timeSinceLastBet = Date.now() - (lastBetTime ?? createdTime) - const betDaysAgo = timeSinceLastBet / DAY_MS - const betTimeScore = 0.5 + 0.5 * (1 - logInterpolation(0, 3, betDaysAgo)) - - let prob = 0.5 - if (outcomeType === 'BINARY') { - prob = getProbability(contract) - } else if (outcomeType === 'FREE_RESPONSE') { - const topAnswer = getTopAnswer(contract) - if (topAnswer) - prob = Math.max(0.5, getOutcomeProbability(contract, topAnswer.id)) - } - const frac = 1 - Math.abs(prob - 0.5) ** 2 / 0.25 - const probScore = 0.5 + frac * 0.5 - - const { volume24Hours, volume7Days } = contract - const combinedVolume = Math.log(volume24Hours + 1) + Math.log(volume7Days + 1) - const volumeScore = 0.5 + 0.5 * logInterpolation(4, 20, combinedVolume) - - const score = - newCommentScore * commentTimeScore * betTimeScore * probScore * volumeScore - - // Map score to [0.5, 1] since no recent activty is not a deal breaker. - const mappedScore = 0.5 + 0.5 * score - const newMappedScore = 0.7 + 0.3 * score - - const isNew = Date.now() < contract.createdTime + DAY_MS - return isNew ? newMappedScore : mappedScore -} - -// function getLastViewedScore(viewTime: number | undefined) { -// if (viewTime === undefined) { -// return 1 -// } - -// const daysAgo = (Date.now() - viewTime) / DAY_MS - -// if (daysAgo < 0.5) { -// const frac = logInterpolation(0, 0.5, daysAgo) -// return 0.5 + 0.25 * frac -// } - -// const frac = logInterpolation(0.5, 14, daysAgo) -// return 0.75 + 0.25 * frac -// } diff --git a/functions/src/update-recommendations.ts b/functions/src/update-recommendations.ts deleted file mode 100644 index bc82291c..00000000 --- a/functions/src/update-recommendations.ts +++ /dev/null @@ -1,70 +0,0 @@ -import * as functions from 'firebase-functions' -import * as admin from 'firebase-admin' - -import { getValue, getValues } from './utils' -import { Contract } from '../../common/contract' -import { Bet } from '../../common/bet' -import { User } from '../../common/user' -import { ClickEvent } from '../../common/tracking' -import { getWordScores } from '../../common/recommended-contracts' -import { batchedWaitAll } from '../../common/util/promise' -import { callCloudFunction } from './call-cloud-function' - -const firestore = admin.firestore() - -export const updateRecommendations = functions.pubsub - .schedule('every 24 hours') - .onRun(async () => { - const users = await getValues(firestore.collection('users')) - - const batchSize = 100 - const userBatches: User[][] = [] - for (let i = 0; i < users.length; i += batchSize) { - userBatches.push(users.slice(i, i + batchSize)) - } - - await Promise.all( - userBatches.map((batch) => - callCloudFunction('updateRecommendationsBatch', { users: batch }) - ) - ) - }) - -export const updateRecommendationsBatch = functions.https.onCall( - async (data: { users: User[] }) => { - const { users } = data - - const contracts = await getValues( - firestore.collection('contracts') - ) - - await batchedWaitAll( - users.map((user) => () => updateWordScores(user, contracts)) - ) - } -) - -export const updateWordScores = async (user: User, contracts: Contract[]) => { - const [bets, viewCounts, clicks] = await Promise.all([ - getValues( - firestore.collectionGroup('bets').where('userId', '==', user.id) - ), - - getValue<{ [contractId: string]: number }>( - firestore.doc(`private-users/${user.id}/cache/viewCounts`) - ), - - getValues( - firestore - .collection(`private-users/${user.id}/events`) - .where('type', '==', 'click') - ), - ]) - - const wordScores = getWordScores(contracts, viewCounts ?? {}, clicks, bets) - - const cachedCollection = firestore.collection( - `private-users/${user.id}/cache` - ) - await cachedCollection.doc('wordScores').set(wordScores) -} diff --git a/yarn.lock b/yarn.lock index 15cd3c51..c07d548f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3875,13 +3875,6 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d" integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA== -biskviit@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/biskviit/-/biskviit-1.0.1.tgz#037a0cd4b71b9e331fd90a1122de17dc49e420a7" - integrity sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w== - dependencies: - psl "^1.1.7" - bluebird@^3.7.1: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" @@ -5237,13 +5230,6 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= -encoding@0.1.12: - version "0.1.12" - resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.12.tgz#538b66f3ee62cd1ab51ec323829d1f9480c74beb" - integrity sha1-U4tm8+5izRq1HsMjgp0flIDHS+s= - dependencies: - iconv-lite "~0.4.13" - end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -5817,14 +5803,6 @@ fetch-blob@^3.1.2, fetch-blob@^3.1.4: node-domexception "^1.0.0" web-streams-polyfill "^3.0.3" -fetch@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/fetch/-/fetch-1.1.0.tgz#0a8279f06be37f9f0ebb567560a30a480da59a2e" - integrity sha1-CoJ58Gvjf58Ou1Z1YKMKSA2lmi4= - dependencies: - biskviit "1.0.1" - encoding "0.1.12" - file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -6782,7 +6760,7 @@ human-signals@^2.1.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== -iconv-lite@0.4.24, iconv-lite@~0.4.13: +iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -9151,11 +9129,6 @@ pseudomap@^1.0.1: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= -psl@^1.1.7: - version "1.8.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" - integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== - pump@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"