Compute stats in Firebase instead of Vercel (#584)
* Add stats updating cloud function * Read stats from database on client instead of computing them * Improve logging for stats updater * Tidying up
This commit is contained in:
parent
2e5d852a77
commit
0067bee94b
23
common/stats.ts
Normal file
23
common/stats.ts
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
export type Stats = {
|
||||||
|
startDate: number
|
||||||
|
dailyActiveUsers: number[]
|
||||||
|
weeklyActiveUsers: number[]
|
||||||
|
monthlyActiveUsers: number[]
|
||||||
|
dailyBetCounts: number[]
|
||||||
|
dailyContractCounts: number[]
|
||||||
|
dailyCommentCounts: number[]
|
||||||
|
dailySignups: number[]
|
||||||
|
weekOnWeekRetention: number[]
|
||||||
|
monthlyRetention: number[]
|
||||||
|
weeklyActivationRate: number[]
|
||||||
|
topTenthActions: {
|
||||||
|
daily: number[]
|
||||||
|
weekly: number[]
|
||||||
|
monthly: number[]
|
||||||
|
}
|
||||||
|
manaBet: {
|
||||||
|
daily: number[]
|
||||||
|
weekly: number[]
|
||||||
|
monthly: number[]
|
||||||
|
}
|
||||||
|
}
|
|
@ -12,6 +12,10 @@ service cloud.firestore {
|
||||||
|| request.auth.uid == 'IPTOzEqrpkWmEzh6hwvAyY9PqFb2' // Manifold
|
|| request.auth.uid == 'IPTOzEqrpkWmEzh6hwvAyY9PqFb2' // Manifold
|
||||||
}
|
}
|
||||||
|
|
||||||
|
match /stats/stats {
|
||||||
|
allow read;
|
||||||
|
}
|
||||||
|
|
||||||
match /users/{userId} {
|
match /users/{userId} {
|
||||||
allow read;
|
allow read;
|
||||||
allow update: if resource.data.id == request.auth.uid
|
allow update: if resource.data.id == request.auth.uid
|
||||||
|
|
|
@ -15,6 +15,7 @@ export * from './on-create-comment'
|
||||||
export * from './on-view'
|
export * from './on-view'
|
||||||
export * from './unsubscribe'
|
export * from './unsubscribe'
|
||||||
export * from './update-metrics'
|
export * from './update-metrics'
|
||||||
|
export * from './update-stats'
|
||||||
export * from './backup-db'
|
export * from './backup-db'
|
||||||
export * from './change-user-info'
|
export * from './change-user-info'
|
||||||
export * from './market-close-notifications'
|
export * from './market-close-notifications'
|
||||||
|
|
15
functions/src/scripts/update-stats.ts
Normal file
15
functions/src/scripts/update-stats.ts
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
import { initAdmin } from './script-init'
|
||||||
|
initAdmin()
|
||||||
|
|
||||||
|
import { log, logMemory } from '../utils'
|
||||||
|
import { updateStatsCore } from '../update-stats'
|
||||||
|
|
||||||
|
async function updateStats() {
|
||||||
|
logMemory()
|
||||||
|
log('Updating stats...')
|
||||||
|
await updateStatsCore()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
updateStats().then(() => process.exit())
|
||||||
|
}
|
316
functions/src/update-stats.ts
Normal file
316
functions/src/update-stats.ts
Normal file
|
@ -0,0 +1,316 @@
|
||||||
|
import * as functions from 'firebase-functions'
|
||||||
|
import * as admin from 'firebase-admin'
|
||||||
|
import { concat, countBy, sortBy, range, zip, uniq, sum, sumBy } from 'lodash'
|
||||||
|
import { getValues, log, logMemory } from './utils'
|
||||||
|
import { Bet } from '../../common/bet'
|
||||||
|
import { Contract } from '../../common/contract'
|
||||||
|
import { Comment } from '../../common/comment'
|
||||||
|
import { User } from '../../common/user'
|
||||||
|
import { DAY_MS } from '../../common/util/time'
|
||||||
|
import { average } from '../../common/util/math'
|
||||||
|
|
||||||
|
const firestore = admin.firestore()
|
||||||
|
|
||||||
|
const numberOfDays = 90
|
||||||
|
|
||||||
|
const getBetsQuery = (startTime: number, endTime: number) =>
|
||||||
|
firestore
|
||||||
|
.collectionGroup('bets')
|
||||||
|
.where('createdTime', '>=', startTime)
|
||||||
|
.where('createdTime', '<', endTime)
|
||||||
|
.orderBy('createdTime', 'asc')
|
||||||
|
|
||||||
|
export async function getDailyBets(startTime: number, numberOfDays: number) {
|
||||||
|
const query = getBetsQuery(startTime, startTime + DAY_MS * numberOfDays)
|
||||||
|
const bets = await getValues<Bet>(query)
|
||||||
|
|
||||||
|
const betsByDay = range(0, numberOfDays).map(() => [] as Bet[])
|
||||||
|
for (const bet of bets) {
|
||||||
|
const dayIndex = Math.floor((bet.createdTime - startTime) / DAY_MS)
|
||||||
|
betsByDay[dayIndex].push(bet)
|
||||||
|
}
|
||||||
|
|
||||||
|
return betsByDay
|
||||||
|
}
|
||||||
|
|
||||||
|
const getCommentsQuery = (startTime: number, endTime: number) =>
|
||||||
|
firestore
|
||||||
|
.collectionGroup('comments')
|
||||||
|
.where('createdTime', '>=', startTime)
|
||||||
|
.where('createdTime', '<', endTime)
|
||||||
|
.orderBy('createdTime', 'asc')
|
||||||
|
|
||||||
|
export async function getDailyComments(
|
||||||
|
startTime: number,
|
||||||
|
numberOfDays: number
|
||||||
|
) {
|
||||||
|
const query = getCommentsQuery(startTime, startTime + DAY_MS * numberOfDays)
|
||||||
|
const comments = await getValues<Comment>(query)
|
||||||
|
|
||||||
|
const commentsByDay = range(0, numberOfDays).map(() => [] as Comment[])
|
||||||
|
for (const comment of comments) {
|
||||||
|
const dayIndex = Math.floor((comment.createdTime - startTime) / DAY_MS)
|
||||||
|
commentsByDay[dayIndex].push(comment)
|
||||||
|
}
|
||||||
|
|
||||||
|
return commentsByDay
|
||||||
|
}
|
||||||
|
|
||||||
|
const getContractsQuery = (startTime: number, endTime: number) =>
|
||||||
|
firestore
|
||||||
|
.collection('contracts')
|
||||||
|
.where('createdTime', '>=', startTime)
|
||||||
|
.where('createdTime', '<', endTime)
|
||||||
|
.orderBy('createdTime', 'asc')
|
||||||
|
|
||||||
|
export async function getDailyContracts(
|
||||||
|
startTime: number,
|
||||||
|
numberOfDays: number
|
||||||
|
) {
|
||||||
|
const query = getContractsQuery(startTime, startTime + DAY_MS * numberOfDays)
|
||||||
|
const contracts = await getValues<Contract>(query)
|
||||||
|
|
||||||
|
const contractsByDay = range(0, numberOfDays).map(() => [] as Contract[])
|
||||||
|
for (const contract of contracts) {
|
||||||
|
const dayIndex = Math.floor((contract.createdTime - startTime) / DAY_MS)
|
||||||
|
contractsByDay[dayIndex].push(contract)
|
||||||
|
}
|
||||||
|
|
||||||
|
return contractsByDay
|
||||||
|
}
|
||||||
|
|
||||||
|
const getUsersQuery = (startTime: number, endTime: number) =>
|
||||||
|
firestore
|
||||||
|
.collection('users')
|
||||||
|
.where('createdTime', '>=', startTime)
|
||||||
|
.where('createdTime', '<', endTime)
|
||||||
|
.orderBy('createdTime', 'asc')
|
||||||
|
|
||||||
|
export async function getDailyNewUsers(
|
||||||
|
startTime: number,
|
||||||
|
numberOfDays: number
|
||||||
|
) {
|
||||||
|
const query = getUsersQuery(startTime, startTime + DAY_MS * numberOfDays)
|
||||||
|
const users = await getValues<User>(query)
|
||||||
|
|
||||||
|
const usersByDay = range(0, numberOfDays).map(() => [] as User[])
|
||||||
|
for (const user of users) {
|
||||||
|
const dayIndex = Math.floor((user.createdTime - startTime) / DAY_MS)
|
||||||
|
usersByDay[dayIndex].push(user)
|
||||||
|
}
|
||||||
|
|
||||||
|
return usersByDay
|
||||||
|
}
|
||||||
|
|
||||||
|
export const updateStatsCore = async () => {
|
||||||
|
const today = Date.now()
|
||||||
|
const startDate = today - numberOfDays * DAY_MS
|
||||||
|
|
||||||
|
log('Fetching data for stats update...')
|
||||||
|
const [dailyBets, dailyContracts, dailyComments, dailyNewUsers] =
|
||||||
|
await Promise.all([
|
||||||
|
getDailyBets(startDate.valueOf(), numberOfDays),
|
||||||
|
getDailyContracts(startDate.valueOf(), numberOfDays),
|
||||||
|
getDailyComments(startDate.valueOf(), numberOfDays),
|
||||||
|
getDailyNewUsers(startDate.valueOf(), numberOfDays),
|
||||||
|
])
|
||||||
|
logMemory()
|
||||||
|
|
||||||
|
const dailyBetCounts = dailyBets.map((bets) => bets.length)
|
||||||
|
const dailyContractCounts = dailyContracts.map(
|
||||||
|
(contracts) => contracts.length
|
||||||
|
)
|
||||||
|
const dailyCommentCounts = dailyComments.map((comments) => comments.length)
|
||||||
|
|
||||||
|
const dailyUserIds = zip(dailyContracts, dailyBets, dailyComments).map(
|
||||||
|
([contracts, bets, comments]) => {
|
||||||
|
const creatorIds = (contracts ?? []).map((c) => c.creatorId)
|
||||||
|
const betUserIds = (bets ?? []).map((bet) => bet.userId)
|
||||||
|
const commentUserIds = (comments ?? []).map((comment) => comment.userId)
|
||||||
|
return uniq([...creatorIds, ...betUserIds, ...commentUserIds])
|
||||||
|
}
|
||||||
|
)
|
||||||
|
log(
|
||||||
|
`Fetched ${sum(dailyBetCounts)} bets, ${sum(
|
||||||
|
dailyContractCounts
|
||||||
|
)} contracts, ${sum(dailyComments)} comments, from ${sum(
|
||||||
|
dailyNewUsers
|
||||||
|
)} unique users.`
|
||||||
|
)
|
||||||
|
|
||||||
|
const dailyActiveUsers = dailyUserIds.map((userIds) => userIds.length)
|
||||||
|
|
||||||
|
const weeklyActiveUsers = dailyUserIds.map((_, i) => {
|
||||||
|
const start = Math.max(0, i - 6)
|
||||||
|
const end = i
|
||||||
|
const uniques = new Set<string>()
|
||||||
|
for (let j = start; j <= end; j++)
|
||||||
|
dailyUserIds[j].forEach((userId) => uniques.add(userId))
|
||||||
|
return uniques.size
|
||||||
|
})
|
||||||
|
|
||||||
|
const monthlyActiveUsers = dailyUserIds.map((_, i) => {
|
||||||
|
const start = Math.max(0, i - 29)
|
||||||
|
const end = i
|
||||||
|
const uniques = new Set<string>()
|
||||||
|
for (let j = start; j <= end; j++)
|
||||||
|
dailyUserIds[j].forEach((userId) => uniques.add(userId))
|
||||||
|
return uniques.size
|
||||||
|
})
|
||||||
|
|
||||||
|
const weekOnWeekRetention = dailyUserIds.map((_userId, i) => {
|
||||||
|
const twoWeeksAgo = {
|
||||||
|
start: Math.max(0, i - 13),
|
||||||
|
end: Math.max(0, i - 7),
|
||||||
|
}
|
||||||
|
const lastWeek = {
|
||||||
|
start: Math.max(0, i - 6),
|
||||||
|
end: i,
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeTwoWeeksAgo = new Set<string>()
|
||||||
|
for (let j = twoWeeksAgo.start; j <= twoWeeksAgo.end; j++) {
|
||||||
|
dailyUserIds[j].forEach((userId) => activeTwoWeeksAgo.add(userId))
|
||||||
|
}
|
||||||
|
const activeLastWeek = new Set<string>()
|
||||||
|
for (let j = lastWeek.start; j <= lastWeek.end; j++) {
|
||||||
|
dailyUserIds[j].forEach((userId) => activeLastWeek.add(userId))
|
||||||
|
}
|
||||||
|
const retainedCount = sumBy(Array.from(activeTwoWeeksAgo), (userId) =>
|
||||||
|
activeLastWeek.has(userId) ? 1 : 0
|
||||||
|
)
|
||||||
|
const retainedFrac = retainedCount / activeTwoWeeksAgo.size
|
||||||
|
return Math.round(retainedFrac * 100 * 100) / 100
|
||||||
|
})
|
||||||
|
|
||||||
|
const monthlyRetention = dailyUserIds.map((_userId, i) => {
|
||||||
|
const twoMonthsAgo = {
|
||||||
|
start: Math.max(0, i - 60),
|
||||||
|
end: Math.max(0, i - 30),
|
||||||
|
}
|
||||||
|
const lastMonth = {
|
||||||
|
start: Math.max(0, i - 30),
|
||||||
|
end: i,
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeTwoMonthsAgo = new Set<string>()
|
||||||
|
for (let j = twoMonthsAgo.start; j <= twoMonthsAgo.end; j++) {
|
||||||
|
dailyUserIds[j].forEach((userId) => activeTwoMonthsAgo.add(userId))
|
||||||
|
}
|
||||||
|
const activeLastMonth = new Set<string>()
|
||||||
|
for (let j = lastMonth.start; j <= lastMonth.end; j++) {
|
||||||
|
dailyUserIds[j].forEach((userId) => activeLastMonth.add(userId))
|
||||||
|
}
|
||||||
|
const retainedCount = sumBy(Array.from(activeTwoMonthsAgo), (userId) =>
|
||||||
|
activeLastMonth.has(userId) ? 1 : 0
|
||||||
|
)
|
||||||
|
const retainedFrac = retainedCount / activeTwoMonthsAgo.size
|
||||||
|
return Math.round(retainedFrac * 100 * 100) / 100
|
||||||
|
})
|
||||||
|
|
||||||
|
const firstBetDict: { [userId: string]: number } = {}
|
||||||
|
for (let i = 0; i < dailyBets.length; i++) {
|
||||||
|
const bets = dailyBets[i]
|
||||||
|
for (const bet of bets) {
|
||||||
|
if (bet.userId in firstBetDict) continue
|
||||||
|
firstBetDict[bet.userId] = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const weeklyActivationRate = dailyNewUsers.map((_, i) => {
|
||||||
|
const start = Math.max(0, i - 6)
|
||||||
|
const end = i
|
||||||
|
let activatedCount = 0
|
||||||
|
let newUsers = 0
|
||||||
|
for (let j = start; j <= end; j++) {
|
||||||
|
const userIds = dailyNewUsers[j].map((user) => user.id)
|
||||||
|
newUsers += userIds.length
|
||||||
|
for (const userId of userIds) {
|
||||||
|
const dayIndex = firstBetDict[userId]
|
||||||
|
if (dayIndex !== undefined && dayIndex <= end) {
|
||||||
|
activatedCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const frac = activatedCount / (newUsers || 1)
|
||||||
|
return Math.round(frac * 100 * 100) / 100
|
||||||
|
})
|
||||||
|
const dailySignups = dailyNewUsers.map((users) => users.length)
|
||||||
|
|
||||||
|
const dailyTopTenthActions = zip(
|
||||||
|
dailyContracts,
|
||||||
|
dailyBets,
|
||||||
|
dailyComments
|
||||||
|
).map(([contracts, bets, comments]) => {
|
||||||
|
const userIds = concat(
|
||||||
|
contracts?.map((c) => c.creatorId) ?? [],
|
||||||
|
bets?.map((b) => b.userId) ?? [],
|
||||||
|
comments?.map((c) => c.userId) ?? []
|
||||||
|
)
|
||||||
|
const counts = Object.values(countBy(userIds))
|
||||||
|
const sortedCounts = sortBy(counts, (count) => count).reverse()
|
||||||
|
if (sortedCounts.length === 0) return 0
|
||||||
|
const tenthPercentile = sortedCounts[Math.floor(sortedCounts.length * 0.1)]
|
||||||
|
return tenthPercentile
|
||||||
|
})
|
||||||
|
const weeklyTopTenthActions = dailyTopTenthActions.map((_, i) => {
|
||||||
|
const start = Math.max(0, i - 6)
|
||||||
|
const end = i
|
||||||
|
return average(dailyTopTenthActions.slice(start, end))
|
||||||
|
})
|
||||||
|
const monthlyTopTenthActions = dailyTopTenthActions.map((_, i) => {
|
||||||
|
const start = Math.max(0, i - 29)
|
||||||
|
const end = i
|
||||||
|
return average(dailyTopTenthActions.slice(start, end))
|
||||||
|
})
|
||||||
|
|
||||||
|
// Total mana divided by 100.
|
||||||
|
const dailyManaBet = dailyBets.map((bets) => {
|
||||||
|
return Math.round(sumBy(bets, (bet) => bet.amount) / 100)
|
||||||
|
})
|
||||||
|
const weeklyManaBet = dailyManaBet.map((_, i) => {
|
||||||
|
const start = Math.max(0, i - 6)
|
||||||
|
const end = i
|
||||||
|
const total = sum(dailyManaBet.slice(start, end))
|
||||||
|
if (end - start < 7) return (total * 7) / (end - start)
|
||||||
|
return total
|
||||||
|
})
|
||||||
|
const monthlyManaBet = dailyManaBet.map((_, i) => {
|
||||||
|
const start = Math.max(0, i - 29)
|
||||||
|
const end = i
|
||||||
|
const total = sum(dailyManaBet.slice(start, end))
|
||||||
|
const range = end - start + 1
|
||||||
|
if (range < 30) return (total * 30) / range
|
||||||
|
return total
|
||||||
|
})
|
||||||
|
|
||||||
|
const statsData = {
|
||||||
|
startDate: startDate.valueOf(),
|
||||||
|
dailyActiveUsers,
|
||||||
|
weeklyActiveUsers,
|
||||||
|
monthlyActiveUsers,
|
||||||
|
dailyBetCounts,
|
||||||
|
dailyContractCounts,
|
||||||
|
dailyCommentCounts,
|
||||||
|
dailySignups,
|
||||||
|
weekOnWeekRetention,
|
||||||
|
weeklyActivationRate,
|
||||||
|
monthlyRetention,
|
||||||
|
topTenthActions: {
|
||||||
|
daily: dailyTopTenthActions,
|
||||||
|
weekly: weeklyTopTenthActions,
|
||||||
|
monthly: monthlyTopTenthActions,
|
||||||
|
},
|
||||||
|
manaBet: {
|
||||||
|
daily: dailyManaBet,
|
||||||
|
weekly: weeklyManaBet,
|
||||||
|
monthly: monthlyManaBet,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
log('Computed stats: ', statsData)
|
||||||
|
await firestore.doc('stats/stats').set(statsData)
|
||||||
|
}
|
||||||
|
|
||||||
|
export const updateStats = functions
|
||||||
|
.runWith({ memory: '1GB', timeoutSeconds: 540 })
|
||||||
|
.pubsub.schedule('every 60 minutes')
|
||||||
|
.onRun(updateStatsCore)
|
|
@ -5,7 +5,7 @@ import {
|
||||||
where,
|
where,
|
||||||
orderBy,
|
orderBy,
|
||||||
} from 'firebase/firestore'
|
} from 'firebase/firestore'
|
||||||
import { range, uniq } from 'lodash'
|
import { uniq } from 'lodash'
|
||||||
|
|
||||||
import { db } from './init'
|
import { db } from './init'
|
||||||
import { Bet } from 'common/bet'
|
import { Bet } from 'common/bet'
|
||||||
|
@ -136,24 +136,3 @@ export function withoutAnteBets(contract: Contract, bets?: Bet[]) {
|
||||||
|
|
||||||
return bets?.filter((bet) => !bet.isAnte) ?? []
|
return bets?.filter((bet) => !bet.isAnte) ?? []
|
||||||
}
|
}
|
||||||
|
|
||||||
const getBetsQuery = (startTime: number, endTime: number) =>
|
|
||||||
query(
|
|
||||||
collectionGroup(db, 'bets'),
|
|
||||||
where('createdTime', '>=', startTime),
|
|
||||||
where('createdTime', '<', endTime),
|
|
||||||
orderBy('createdTime', 'asc')
|
|
||||||
)
|
|
||||||
|
|
||||||
export async function getDailyBets(startTime: number, numberOfDays: number) {
|
|
||||||
const query = getBetsQuery(startTime, startTime + DAY_IN_MS * numberOfDays)
|
|
||||||
const bets = await getValues<Bet>(query)
|
|
||||||
|
|
||||||
const betsByDay = range(0, numberOfDays).map(() => [] as Bet[])
|
|
||||||
for (const bet of bets) {
|
|
||||||
const dayIndex = Math.floor((bet.createdTime - startTime) / DAY_IN_MS)
|
|
||||||
betsByDay[dayIndex].push(bet)
|
|
||||||
}
|
|
||||||
|
|
||||||
return betsByDay
|
|
||||||
}
|
|
||||||
|
|
|
@ -7,7 +7,6 @@ import {
|
||||||
setDoc,
|
setDoc,
|
||||||
where,
|
where,
|
||||||
} from 'firebase/firestore'
|
} from 'firebase/firestore'
|
||||||
import { range } from 'lodash'
|
|
||||||
|
|
||||||
import { getValues, listenForValues } from './utils'
|
import { getValues, listenForValues } from './utils'
|
||||||
import { db } from './init'
|
import { db } from './init'
|
||||||
|
@ -136,33 +135,6 @@ export function listenForRecentComments(
|
||||||
return listenForValues<Comment>(recentCommentsQuery, setComments)
|
return listenForValues<Comment>(recentCommentsQuery, setComments)
|
||||||
}
|
}
|
||||||
|
|
||||||
const getCommentsQuery = (startTime: number, endTime: number) =>
|
|
||||||
query(
|
|
||||||
collectionGroup(db, 'comments'),
|
|
||||||
where('createdTime', '>=', startTime),
|
|
||||||
where('createdTime', '<', endTime),
|
|
||||||
orderBy('createdTime', 'asc')
|
|
||||||
)
|
|
||||||
|
|
||||||
export async function getDailyComments(
|
|
||||||
startTime: number,
|
|
||||||
numberOfDays: number
|
|
||||||
) {
|
|
||||||
const query = getCommentsQuery(
|
|
||||||
startTime,
|
|
||||||
startTime + DAY_IN_MS * numberOfDays
|
|
||||||
)
|
|
||||||
const comments = await getValues<Comment>(query)
|
|
||||||
|
|
||||||
const commentsByDay = range(0, numberOfDays).map(() => [] as Comment[])
|
|
||||||
for (const comment of comments) {
|
|
||||||
const dayIndex = Math.floor((comment.createdTime - startTime) / DAY_IN_MS)
|
|
||||||
commentsByDay[dayIndex].push(comment)
|
|
||||||
}
|
|
||||||
|
|
||||||
return commentsByDay
|
|
||||||
}
|
|
||||||
|
|
||||||
const getUsersCommentsQuery = (userId: string) =>
|
const getUsersCommentsQuery = (userId: string) =>
|
||||||
query(
|
query(
|
||||||
collectionGroup(db, 'comments'),
|
collectionGroup(db, 'comments'),
|
||||||
|
|
|
@ -14,7 +14,7 @@ import {
|
||||||
limit,
|
limit,
|
||||||
startAfter,
|
startAfter,
|
||||||
} from 'firebase/firestore'
|
} from 'firebase/firestore'
|
||||||
import { range, sortBy, sum } from 'lodash'
|
import { sortBy, sum } from 'lodash'
|
||||||
|
|
||||||
import { app } from './init'
|
import { app } from './init'
|
||||||
import { getValues, listenForValue, listenForValues } from './utils'
|
import { getValues, listenForValue, listenForValues } from './utils'
|
||||||
|
@ -303,35 +303,6 @@ export async function getClosingSoonContracts() {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
const getContractsQuery = (startTime: number, endTime: number) =>
|
|
||||||
query(
|
|
||||||
collection(db, 'contracts'),
|
|
||||||
where('createdTime', '>=', startTime),
|
|
||||||
where('createdTime', '<', endTime),
|
|
||||||
orderBy('createdTime', 'asc')
|
|
||||||
)
|
|
||||||
|
|
||||||
const DAY_IN_MS = 24 * 60 * 60 * 1000
|
|
||||||
|
|
||||||
export async function getDailyContracts(
|
|
||||||
startTime: number,
|
|
||||||
numberOfDays: number
|
|
||||||
) {
|
|
||||||
const query = getContractsQuery(
|
|
||||||
startTime,
|
|
||||||
startTime + DAY_IN_MS * numberOfDays
|
|
||||||
)
|
|
||||||
const contracts = await getValues<Contract>(query)
|
|
||||||
|
|
||||||
const contractsByDay = range(0, numberOfDays).map(() => [] as Contract[])
|
|
||||||
for (const contract of contracts) {
|
|
||||||
const dayIndex = Math.floor((contract.createdTime - startTime) / DAY_IN_MS)
|
|
||||||
contractsByDay[dayIndex].push(contract)
|
|
||||||
}
|
|
||||||
|
|
||||||
return contractsByDay
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getRecentBetsAndComments(contract: Contract) {
|
export async function getRecentBetsAndComments(contract: Contract) {
|
||||||
const contractDoc = doc(db, 'contracts', contract.id)
|
const contractDoc = doc(db, 'contracts', contract.id)
|
||||||
|
|
||||||
|
|
15
web/lib/firebase/stats.ts
Normal file
15
web/lib/firebase/stats.ts
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
import {
|
||||||
|
CollectionReference,
|
||||||
|
doc,
|
||||||
|
collection,
|
||||||
|
getDoc,
|
||||||
|
} from 'firebase/firestore'
|
||||||
|
import { db } from 'web/lib/firebase/init'
|
||||||
|
import { Stats } from 'common/stats'
|
||||||
|
|
||||||
|
const statsCollection = collection(db, 'stats') as CollectionReference<Stats>
|
||||||
|
const statsDoc = doc(statsCollection, 'stats')
|
||||||
|
|
||||||
|
export const getStats = async () => {
|
||||||
|
return (await getDoc(statsDoc)).data()
|
||||||
|
}
|
|
@ -21,13 +21,12 @@ import {
|
||||||
GoogleAuthProvider,
|
GoogleAuthProvider,
|
||||||
signInWithPopup,
|
signInWithPopup,
|
||||||
} from 'firebase/auth'
|
} from 'firebase/auth'
|
||||||
import { range, throttle, zip } from 'lodash'
|
import { throttle, zip } from 'lodash'
|
||||||
|
|
||||||
import { app } from './init'
|
import { app } from './init'
|
||||||
import { PortfolioMetrics, PrivateUser, User } from 'common/user'
|
import { PortfolioMetrics, PrivateUser, User } from 'common/user'
|
||||||
import { createUser } from './fn-call'
|
import { createUser } from './fn-call'
|
||||||
import { getValue, getValues, listenForValue, listenForValues } from './utils'
|
import { getValue, getValues, listenForValue, listenForValues } from './utils'
|
||||||
import { DAY_MS } from 'common/util/time'
|
|
||||||
import { feed } from 'common/feed'
|
import { feed } from 'common/feed'
|
||||||
import { CATEGORY_LIST } from 'common/categories'
|
import { CATEGORY_LIST } from 'common/categories'
|
||||||
import { safeLocalStorage } from '../util/local'
|
import { safeLocalStorage } from '../util/local'
|
||||||
|
@ -214,30 +213,6 @@ export function getUsers() {
|
||||||
return getValues<User>(collection(db, 'users'))
|
return getValues<User>(collection(db, 'users'))
|
||||||
}
|
}
|
||||||
|
|
||||||
const getUsersQuery = (startTime: number, endTime: number) =>
|
|
||||||
query(
|
|
||||||
collection(db, 'users'),
|
|
||||||
where('createdTime', '>=', startTime),
|
|
||||||
where('createdTime', '<', endTime),
|
|
||||||
orderBy('createdTime', 'asc')
|
|
||||||
)
|
|
||||||
|
|
||||||
export async function getDailyNewUsers(
|
|
||||||
startTime: number,
|
|
||||||
numberOfDays: number
|
|
||||||
) {
|
|
||||||
const query = getUsersQuery(startTime, startTime + DAY_MS * numberOfDays)
|
|
||||||
const users = await getValues<User>(query)
|
|
||||||
|
|
||||||
const usersByDay = range(0, numberOfDays).map(() => [] as User[])
|
|
||||||
for (const user of users) {
|
|
||||||
const dayIndex = Math.floor((user.createdTime - startTime) / DAY_MS)
|
|
||||||
usersByDay[dayIndex].push(user)
|
|
||||||
}
|
|
||||||
|
|
||||||
return usersByDay
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getUserFeed(userId: string) {
|
export async function getUserFeed(userId: string) {
|
||||||
const feedDoc = doc(db, 'private-users', userId, 'cache', 'feed')
|
const feedDoc = doc(db, 'private-users', userId, 'cache', 'feed')
|
||||||
const userFeed = await getValue<{
|
const userFeed = await getValue<{
|
||||||
|
|
|
@ -1,18 +1,21 @@
|
||||||
|
import { useState, useEffect } from 'react'
|
||||||
import { Col } from 'web/components/layout/col'
|
import { Col } from 'web/components/layout/col'
|
||||||
import { Spacer } from 'web/components/layout/spacer'
|
import { Spacer } from 'web/components/layout/spacer'
|
||||||
import { fromPropz } from 'web/hooks/use-propz'
|
import { CustomAnalytics, FirebaseAnalytics } from '../stats'
|
||||||
import Analytics, {
|
import { getStats } from 'web/lib/firebase/stats'
|
||||||
CustomAnalytics,
|
import { Stats } from 'common/stats'
|
||||||
FirebaseAnalytics,
|
|
||||||
getStaticPropz,
|
|
||||||
} from '../stats'
|
|
||||||
|
|
||||||
export const getStaticProps = fromPropz(getStaticPropz)
|
export default function AnalyticsEmbed() {
|
||||||
|
const [stats, setStats] = useState<Stats | undefined>(undefined)
|
||||||
export default function AnalyticsEmbed(props: Parameters<typeof Analytics>[0]) {
|
useEffect(() => {
|
||||||
|
getStats().then(setStats)
|
||||||
|
}, [])
|
||||||
|
if (stats == null) {
|
||||||
|
return <></>
|
||||||
|
}
|
||||||
return (
|
return (
|
||||||
<Col className="w-full bg-white px-2">
|
<Col className="w-full bg-white px-2">
|
||||||
<CustomAnalytics {...props} />
|
<CustomAnalytics {...stats} />
|
||||||
<Spacer h={8} />
|
<Spacer h={8} />
|
||||||
<FirebaseAnalytics />
|
<FirebaseAnalytics />
|
||||||
</Col>
|
</Col>
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
import dayjs from 'dayjs'
|
import dayjs from 'dayjs'
|
||||||
import { zip, uniq, sumBy, concat, countBy, sortBy, sum } from 'lodash'
|
import { useEffect, useState } from 'react'
|
||||||
import {
|
import {
|
||||||
DailyCountChart,
|
DailyCountChart,
|
||||||
DailyPercentChart,
|
DailyPercentChart,
|
||||||
|
@ -9,265 +9,18 @@ import { Spacer } from 'web/components/layout/spacer'
|
||||||
import { Tabs } from 'web/components/layout/tabs'
|
import { Tabs } from 'web/components/layout/tabs'
|
||||||
import { Page } from 'web/components/page'
|
import { Page } from 'web/components/page'
|
||||||
import { Title } from 'web/components/title'
|
import { Title } from 'web/components/title'
|
||||||
import { fromPropz, usePropz } from 'web/hooks/use-propz'
|
|
||||||
import { getDailyBets } from 'web/lib/firebase/bets'
|
|
||||||
import { getDailyComments } from 'web/lib/firebase/comments'
|
|
||||||
import { getDailyContracts } from 'web/lib/firebase/contracts'
|
|
||||||
import { getDailyNewUsers } from 'web/lib/firebase/users'
|
|
||||||
import { SiteLink } from 'web/components/site-link'
|
import { SiteLink } from 'web/components/site-link'
|
||||||
import { Linkify } from 'web/components/linkify'
|
import { Linkify } from 'web/components/linkify'
|
||||||
import { average } from 'common/util/math'
|
import { getStats } from 'web/lib/firebase/stats'
|
||||||
|
import { Stats } from 'common/stats'
|
||||||
|
|
||||||
export const getStaticProps = fromPropz(getStaticPropz)
|
export default function Analytics() {
|
||||||
export async function getStaticPropz() {
|
const [stats, setStats] = useState<Stats | undefined>(undefined)
|
||||||
const numberOfDays = 90
|
useEffect(() => {
|
||||||
const today = dayjs(dayjs().format('YYYY-MM-DD'))
|
getStats().then(setStats)
|
||||||
// Convert from UTC midnight to PT midnight.
|
}, [])
|
||||||
.add(7, 'hours')
|
if (stats == null) {
|
||||||
|
return <></>
|
||||||
const startDate = today.subtract(numberOfDays, 'day')
|
|
||||||
|
|
||||||
const [dailyBets, dailyContracts, dailyComments, dailyNewUsers] =
|
|
||||||
await Promise.all([
|
|
||||||
getDailyBets(startDate.valueOf(), numberOfDays),
|
|
||||||
getDailyContracts(startDate.valueOf(), numberOfDays),
|
|
||||||
getDailyComments(startDate.valueOf(), numberOfDays),
|
|
||||||
getDailyNewUsers(startDate.valueOf(), numberOfDays),
|
|
||||||
])
|
|
||||||
|
|
||||||
const dailyBetCounts = dailyBets.map((bets) => bets.length)
|
|
||||||
const dailyContractCounts = dailyContracts.map(
|
|
||||||
(contracts) => contracts.length
|
|
||||||
)
|
|
||||||
const dailyCommentCounts = dailyComments.map((comments) => comments.length)
|
|
||||||
|
|
||||||
const dailyUserIds = zip(dailyContracts, dailyBets, dailyComments).map(
|
|
||||||
([contracts, bets, comments]) => {
|
|
||||||
const creatorIds = (contracts ?? []).map((c) => c.creatorId)
|
|
||||||
const betUserIds = (bets ?? []).map((bet) => bet.userId)
|
|
||||||
const commentUserIds = (comments ?? []).map((comment) => comment.userId)
|
|
||||||
return uniq([...creatorIds, ...betUserIds, ...commentUserIds])
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
const dailyActiveUsers = dailyUserIds.map((userIds) => userIds.length)
|
|
||||||
|
|
||||||
const weeklyActiveUsers = dailyUserIds.map((_, i) => {
|
|
||||||
const start = Math.max(0, i - 6)
|
|
||||||
const end = i
|
|
||||||
const uniques = new Set<string>()
|
|
||||||
for (let j = start; j <= end; j++)
|
|
||||||
dailyUserIds[j].forEach((userId) => uniques.add(userId))
|
|
||||||
return uniques.size
|
|
||||||
})
|
|
||||||
|
|
||||||
const monthlyActiveUsers = dailyUserIds.map((_, i) => {
|
|
||||||
const start = Math.max(0, i - 29)
|
|
||||||
const end = i
|
|
||||||
const uniques = new Set<string>()
|
|
||||||
for (let j = start; j <= end; j++)
|
|
||||||
dailyUserIds[j].forEach((userId) => uniques.add(userId))
|
|
||||||
return uniques.size
|
|
||||||
})
|
|
||||||
|
|
||||||
const weekOnWeekRetention = dailyUserIds.map((_userId, i) => {
|
|
||||||
const twoWeeksAgo = {
|
|
||||||
start: Math.max(0, i - 13),
|
|
||||||
end: Math.max(0, i - 7),
|
|
||||||
}
|
|
||||||
const lastWeek = {
|
|
||||||
start: Math.max(0, i - 6),
|
|
||||||
end: i,
|
|
||||||
}
|
|
||||||
|
|
||||||
const activeTwoWeeksAgo = new Set<string>()
|
|
||||||
for (let j = twoWeeksAgo.start; j <= twoWeeksAgo.end; j++) {
|
|
||||||
dailyUserIds[j].forEach((userId) => activeTwoWeeksAgo.add(userId))
|
|
||||||
}
|
|
||||||
const activeLastWeek = new Set<string>()
|
|
||||||
for (let j = lastWeek.start; j <= lastWeek.end; j++) {
|
|
||||||
dailyUserIds[j].forEach((userId) => activeLastWeek.add(userId))
|
|
||||||
}
|
|
||||||
const retainedCount = sumBy(Array.from(activeTwoWeeksAgo), (userId) =>
|
|
||||||
activeLastWeek.has(userId) ? 1 : 0
|
|
||||||
)
|
|
||||||
const retainedFrac = retainedCount / activeTwoWeeksAgo.size
|
|
||||||
return Math.round(retainedFrac * 100 * 100) / 100
|
|
||||||
})
|
|
||||||
|
|
||||||
const monthlyRetention = dailyUserIds.map((_userId, i) => {
|
|
||||||
const twoMonthsAgo = {
|
|
||||||
start: Math.max(0, i - 60),
|
|
||||||
end: Math.max(0, i - 30),
|
|
||||||
}
|
|
||||||
const lastMonth = {
|
|
||||||
start: Math.max(0, i - 30),
|
|
||||||
end: i,
|
|
||||||
}
|
|
||||||
|
|
||||||
const activeTwoMonthsAgo = new Set<string>()
|
|
||||||
for (let j = twoMonthsAgo.start; j <= twoMonthsAgo.end; j++) {
|
|
||||||
dailyUserIds[j].forEach((userId) => activeTwoMonthsAgo.add(userId))
|
|
||||||
}
|
|
||||||
const activeLastMonth = new Set<string>()
|
|
||||||
for (let j = lastMonth.start; j <= lastMonth.end; j++) {
|
|
||||||
dailyUserIds[j].forEach((userId) => activeLastMonth.add(userId))
|
|
||||||
}
|
|
||||||
const retainedCount = sumBy(Array.from(activeTwoMonthsAgo), (userId) =>
|
|
||||||
activeLastMonth.has(userId) ? 1 : 0
|
|
||||||
)
|
|
||||||
const retainedFrac = retainedCount / activeTwoMonthsAgo.size
|
|
||||||
return Math.round(retainedFrac * 100 * 100) / 100
|
|
||||||
})
|
|
||||||
|
|
||||||
const firstBetDict: { [userId: string]: number } = {}
|
|
||||||
for (let i = 0; i < dailyBets.length; i++) {
|
|
||||||
const bets = dailyBets[i]
|
|
||||||
for (const bet of bets) {
|
|
||||||
if (bet.userId in firstBetDict) continue
|
|
||||||
firstBetDict[bet.userId] = i
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const weeklyActivationRate = dailyNewUsers.map((_, i) => {
|
|
||||||
const start = Math.max(0, i - 6)
|
|
||||||
const end = i
|
|
||||||
let activatedCount = 0
|
|
||||||
let newUsers = 0
|
|
||||||
for (let j = start; j <= end; j++) {
|
|
||||||
const userIds = dailyNewUsers[j].map((user) => user.id)
|
|
||||||
newUsers += userIds.length
|
|
||||||
for (const userId of userIds) {
|
|
||||||
const dayIndex = firstBetDict[userId]
|
|
||||||
if (dayIndex !== undefined && dayIndex <= end) {
|
|
||||||
activatedCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const frac = activatedCount / (newUsers || 1)
|
|
||||||
return Math.round(frac * 100 * 100) / 100
|
|
||||||
})
|
|
||||||
const dailySignups = dailyNewUsers.map((users) => users.length)
|
|
||||||
|
|
||||||
const dailyTopTenthActions = zip(
|
|
||||||
dailyContracts,
|
|
||||||
dailyBets,
|
|
||||||
dailyComments
|
|
||||||
).map(([contracts, bets, comments]) => {
|
|
||||||
const userIds = concat(
|
|
||||||
contracts?.map((c) => c.creatorId) ?? [],
|
|
||||||
bets?.map((b) => b.userId) ?? [],
|
|
||||||
comments?.map((c) => c.userId) ?? []
|
|
||||||
)
|
|
||||||
const counts = Object.values(countBy(userIds))
|
|
||||||
const sortedCounts = sortBy(counts, (count) => count).reverse()
|
|
||||||
if (sortedCounts.length === 0) return 0
|
|
||||||
const tenthPercentile = sortedCounts[Math.floor(sortedCounts.length * 0.1)]
|
|
||||||
return tenthPercentile
|
|
||||||
})
|
|
||||||
const weeklyTopTenthActions = dailyTopTenthActions.map((_, i) => {
|
|
||||||
const start = Math.max(0, i - 6)
|
|
||||||
const end = i
|
|
||||||
return average(dailyTopTenthActions.slice(start, end))
|
|
||||||
})
|
|
||||||
const monthlyTopTenthActions = dailyTopTenthActions.map((_, i) => {
|
|
||||||
const start = Math.max(0, i - 29)
|
|
||||||
const end = i
|
|
||||||
return average(dailyTopTenthActions.slice(start, end))
|
|
||||||
})
|
|
||||||
|
|
||||||
// Total mana divided by 100.
|
|
||||||
const dailyManaBet = dailyBets.map((bets) => {
|
|
||||||
return Math.round(sumBy(bets, (bet) => bet.amount) / 100)
|
|
||||||
})
|
|
||||||
const weeklyManaBet = dailyManaBet.map((_, i) => {
|
|
||||||
const start = Math.max(0, i - 6)
|
|
||||||
const end = i
|
|
||||||
const total = sum(dailyManaBet.slice(start, end))
|
|
||||||
if (end - start < 7) return (total * 7) / (end - start)
|
|
||||||
return total
|
|
||||||
})
|
|
||||||
const monthlyManaBet = dailyManaBet.map((_, i) => {
|
|
||||||
const start = Math.max(0, i - 29)
|
|
||||||
const end = i
|
|
||||||
const total = sum(dailyManaBet.slice(start, end))
|
|
||||||
const range = end - start + 1
|
|
||||||
if (range < 30) return (total * 30) / range
|
|
||||||
return total
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
props: {
|
|
||||||
startDate: startDate.valueOf(),
|
|
||||||
dailyActiveUsers,
|
|
||||||
weeklyActiveUsers,
|
|
||||||
monthlyActiveUsers,
|
|
||||||
dailyBetCounts,
|
|
||||||
dailyContractCounts,
|
|
||||||
dailyCommentCounts,
|
|
||||||
dailySignups,
|
|
||||||
weekOnWeekRetention,
|
|
||||||
weeklyActivationRate,
|
|
||||||
monthlyRetention,
|
|
||||||
topTenthActions: {
|
|
||||||
daily: dailyTopTenthActions,
|
|
||||||
weekly: weeklyTopTenthActions,
|
|
||||||
monthly: monthlyTopTenthActions,
|
|
||||||
},
|
|
||||||
manaBet: {
|
|
||||||
daily: dailyManaBet,
|
|
||||||
weekly: weeklyManaBet,
|
|
||||||
monthly: monthlyManaBet,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
revalidate: 60 * 60, // Regenerate after an hour
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default function Analytics(props: {
|
|
||||||
startDate: number
|
|
||||||
dailyActiveUsers: number[]
|
|
||||||
weeklyActiveUsers: number[]
|
|
||||||
monthlyActiveUsers: number[]
|
|
||||||
dailyBetCounts: number[]
|
|
||||||
dailyContractCounts: number[]
|
|
||||||
dailyCommentCounts: number[]
|
|
||||||
dailySignups: number[]
|
|
||||||
weekOnWeekRetention: number[]
|
|
||||||
monthlyRetention: number[]
|
|
||||||
weeklyActivationRate: number[]
|
|
||||||
topTenthActions: {
|
|
||||||
daily: number[]
|
|
||||||
weekly: number[]
|
|
||||||
monthly: number[]
|
|
||||||
}
|
|
||||||
manaBet: {
|
|
||||||
daily: number[]
|
|
||||||
weekly: number[]
|
|
||||||
monthly: number[]
|
|
||||||
}
|
|
||||||
}) {
|
|
||||||
props = usePropz(props, getStaticPropz) ?? {
|
|
||||||
startDate: 0,
|
|
||||||
dailyActiveUsers: [],
|
|
||||||
weeklyActiveUsers: [],
|
|
||||||
monthlyActiveUsers: [],
|
|
||||||
dailyBetCounts: [],
|
|
||||||
dailyContractCounts: [],
|
|
||||||
dailyCommentCounts: [],
|
|
||||||
dailySignups: [],
|
|
||||||
weekOnWeekRetention: [],
|
|
||||||
monthlyRetention: [],
|
|
||||||
weeklyActivationRate: [],
|
|
||||||
topTenthActions: {
|
|
||||||
daily: [],
|
|
||||||
weekly: [],
|
|
||||||
monthly: [],
|
|
||||||
},
|
|
||||||
manaBet: {
|
|
||||||
daily: [],
|
|
||||||
weekly: [],
|
|
||||||
monthly: [],
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
return (
|
return (
|
||||||
<Page>
|
<Page>
|
||||||
|
@ -275,7 +28,7 @@ export default function Analytics(props: {
|
||||||
tabs={[
|
tabs={[
|
||||||
{
|
{
|
||||||
title: 'Activity',
|
title: 'Activity',
|
||||||
content: <CustomAnalytics {...props} />,
|
content: <CustomAnalytics {...stats} />,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
title: 'Market Stats',
|
title: 'Market Stats',
|
||||||
|
|
Loading…
Reference in New Issue
Block a user