manifold/functions/src/weekly-markets-emails.ts

import * as functions from 'firebase-functions'
import * as admin from 'firebase-admin'

import { Contract } from '../../common/contract'
import {
  getAllPrivateUsers,
  getGroup,
  getPrivateUser,
  getUser,
  getValues,
  isProd,
  log,
} from './utils'
import { createRNG, shuffle } from '../../common/util/random'
import { DAY_MS, HOUR_MS } from '../../common/util/time'
import { filterDefined } from '../../common/util/array'
import { Follow } from '../../common/follow'
import { countBy, uniq, uniqBy } from 'lodash'
import { sendInterestingMarketsEmail } from './emails'

export const weeklyMarketsEmails = functions
  .runWith({ secrets: ['MAILGUN_KEY'], memory: '4GB' })
  // every minute on Monday for 2 hours starting at 12pm PT (UTC -07:00)
  .pubsub.schedule('* 19-20 * * 1')
  .timeZone('Etc/UTC')
  .onRun(async () => {
    await sendTrendingMarketsEmailsToAllUsers()
  })

const firestore = admin.firestore()

export async function getTrendingContracts() {
  return await getValues<Contract>(
    firestore
      .collection('contracts')
      .where('isResolved', '==', false)
      .where('visibility', '==', 'public')
      // can't use multiple inequality (/orderBy) operators on different fields,
      // so have to filter for closed contracts separately
      .orderBy('popularityScore', 'desc')
      // might as well go big and do a quick filter for closed ones later
      .limit(500)
  )
}

export async function sendTrendingMarketsEmailsToAllUsers() {
  const numContractsToSend = 6
  const privateUsers = isProd()
    ? await getAllPrivateUsers()
    : filterDefined([
        await getPrivateUser('6hHpzvRG0pMq8PNJs7RZj2qlZGn2'), // dev Ian
      ])
  const privateUsersToSendEmailsTo = privateUsers
    // Get all users that haven't unsubscribed from weekly emails
    .filter(
      (user) =>
        user.notificationPreferences.trending_markets.includes('email') &&
        !user.weeklyTrendingEmailSent
    )
    .slice(0, 90) // Send the emails out in batches

  // For testing different users on prod: (only send ian an email though)
  // const privateUsersToSendEmailsTo = filterDefined([
  //   await getPrivateUser('AJwLWoo3xue32XIiAVrL5SyR1WB2'), // prod Ian
  //   // isProd()
  //   await getPrivateUser('FptiiMZZ6dQivihLI8MYFQ6ypSw1'), // prod Mik
  //   //   : await getPrivateUser('6hHpzvRG0pMq8PNJs7RZj2qlZGn2'), // dev Ian
  // ])

  log(
    'Sending weekly trending emails to',
    privateUsersToSendEmailsTo.length,
    'users'
  )
  const trendingContracts = (await getTrendingContracts())
    .filter(
      (contract) =>
        !(
          contract.question.toLowerCase().includes('trump') &&
          contract.question.toLowerCase().includes('president')
        ) &&
        (contract?.closeTime ?? 0) > Date.now() + DAY_MS &&
        !contract.groupSlugs?.includes('manifold-features') &&
        !contract.groupSlugs?.includes('manifold-6748e065087e')
    )
    .slice(0, 50)

  const uniqueTrendingContracts = removeSimilarQuestions(
    trendingContracts,
    trendingContracts,
    true
  ).slice(0, 20)

  await Promise.all(
    privateUsersToSendEmailsTo.map(async (privateUser) => {
      if (!privateUser.email) {
        log(`No email for ${privateUser.username}`)
        return
      }

      const unbetOnFollowedMarkets = await getUserUnBetOnFollowsMarkets(
        privateUser.id
      )
      const unBetOnGroupMarkets = await getUserUnBetOnGroupsMarkets(
        privateUser.id,
        unbetOnFollowedMarkets
      )
      const similarBettorsMarkets = await getSimilarBettorsMarkets(
        privateUser.id,
        unBetOnGroupMarkets
      )

      const marketsAvailableToSend = uniqBy(
        [
          ...chooseRandomSubset(unbetOnFollowedMarkets, 2),
          // // Most people will belong to groups but may not follow other users,
          // so choose more from the other subsets if the followed markets is sparse
          ...chooseRandomSubset(
            unBetOnGroupMarkets,
            unbetOnFollowedMarkets.length < 2 ? 3 : 2
          ),
          ...chooseRandomSubset(
            similarBettorsMarkets,
            unbetOnFollowedMarkets.length < 2 ? 3 : 2
          ),
        ],
        (contract) => contract.id
      )
      // // at least send them trending contracts if nothing else
      if (marketsAvailableToSend.length < numContractsToSend) {
        const trendingMarketsToSend =
          numContractsToSend - marketsAvailableToSend.length
        log(
          `not enough personalized markets, sending ${trendingMarketsToSend} trending`
        )
        marketsAvailableToSend.push(
          ...removeSimilarQuestions(
            uniqueTrendingContracts,
            marketsAvailableToSend,
            false
          )
            .filter(
              (contract) => !contract.uniqueBettorIds?.includes(privateUser.id)
            )
            .slice(0, trendingMarketsToSend)
        )
      }

      if (marketsAvailableToSend.length < numContractsToSend) {
        log(
          'not enough new, unbet-on contracts to send to user',
          privateUser.id
        )
        await firestore.collection('private-users').doc(privateUser.id).update({
          weeklyTrendingEmailSent: true,
        })
        return
      }
      // choose random subset of contracts to send to user
      const contractsToSend = chooseRandomSubset(
        marketsAvailableToSend,
        numContractsToSend
      )

      const user = await getUser(privateUser.id)
      if (!user) return

      log(
        'sending contracts:',
        contractsToSend.map((c) => c.question + ' ' + c.popularityScore)
      )
      // if they don't have enough markets, find user bets and get the other bettor ids who most overlap on those markets, then do the same thing as above for them
      await sendInterestingMarketsEmail(user, privateUser, contractsToSend)
      await firestore.collection('private-users').doc(user.id).update({
        weeklyTrendingEmailSent: true,
      })
    })
  )
}

const MINIMUM_POPULARITY_SCORE = 10

const getUserUnBetOnFollowsMarkets = async (userId: string) => {
  const follows = await getValues<Follow>(
    firestore.collection('users').doc(userId).collection('follows')
  )

  const unBetOnContractsFromFollows = await Promise.all(
    follows.map(async (follow) => {
      const unresolvedContracts = await getValues<Contract>(
        firestore
          .collection('contracts')
          .where('isResolved', '==', false)
          .where('visibility', '==', 'public')
          .where('creatorId', '==', follow.userId)
          // can't use multiple inequality (/orderBy) operators on different fields,
          // so have to filter for closed contracts separately
          .orderBy('popularityScore', 'desc')
          .limit(50)
      )
      // filter out contracts that have close times less than 6 hours from now
      const openContracts = unresolvedContracts.filter(
        (contract) => (contract?.closeTime ?? 0) > Date.now() + 6 * HOUR_MS
      )

      return openContracts.filter(
        (contract) => !contract.uniqueBettorIds?.includes(userId)
      )
    })
  )

  const sortedMarkets = uniqBy(
    unBetOnContractsFromFollows.flat(),
    (contract) => contract.id
  )
    .filter(
      (contract) =>
        contract.popularityScore !== undefined &&
        contract.popularityScore > MINIMUM_POPULARITY_SCORE
    )
    .sort((a, b) => (b.popularityScore ?? 0) - (a.popularityScore ?? 0))

  const uniqueSortedMarkets = removeSimilarQuestions(
    sortedMarkets,
    sortedMarkets,
    true
  )

  const topSortedMarkets = uniqueSortedMarkets.slice(0, 10)
  // log(
  //   'top 10 sorted markets by followed users',
  //   topSortedMarkets.map((c) => c.question + ' ' + c.popularityScore)
  // )
  return topSortedMarkets
}

const getUserUnBetOnGroupsMarkets = async (
  userId: string,
  differentThanTheseContracts: Contract[]
) => {
  const snap = await firestore
    .collectionGroup('groupMembers')
    .where('userId', '==', userId)
    .get()

  const groupIds = filterDefined(
    snap.docs.map((doc) => doc.ref.parent.parent?.id)
  )
  const groups = filterDefined(
    await Promise.all(groupIds.map(async (groupId) => await getGroup(groupId)))
  )
  if (groups.length === 0) return []

  const unBetOnContractsFromGroups = await Promise.all(
    groups.map(async (group) => {
      const unresolvedContracts = await getValues<Contract>(
        firestore
          .collection('contracts')
          .where('isResolved', '==', false)
          .where('visibility', '==', 'public')
          .where('groupSlugs', 'array-contains', group.slug)
          // can't use multiple inequality (/orderBy) operators on different fields,
          // so have to filter for closed contracts separately
          .orderBy('popularityScore', 'desc')
          .limit(50)
      )
      // filter out contracts that have close times less than 6 hours from now
      const openContracts = unresolvedContracts.filter(
        (contract) => (contract?.closeTime ?? 0) > Date.now() + 6 * HOUR_MS
      )

      return openContracts.filter(
        (contract) => !contract.uniqueBettorIds?.includes(userId)
      )
    })
  )

  const sortedMarkets = uniqBy(
    unBetOnContractsFromGroups.flat(),
    (contract) => contract.id
  )
    .filter(
      (contract) =>
        contract.popularityScore !== undefined &&
        contract.popularityScore > MINIMUM_POPULARITY_SCORE
    )
    .sort((a, b) => (b.popularityScore ?? 0) - (a.popularityScore ?? 0))

  const uniqueSortedMarkets = removeSimilarQuestions(
    sortedMarkets,
    sortedMarkets,
    true
  )
  const topSortedMarkets = removeSimilarQuestions(
    uniqueSortedMarkets,
    differentThanTheseContracts,
    false
  ).slice(0, 10)

  // log(
  //   'top 10 sorted group markets',
  //   topSortedMarkets.map((c) => c.question + ' ' + c.popularityScore)
  // )
  return topSortedMarkets
}

// Gets markets followed by similar bettors and bet on by similar bettors
const getSimilarBettorsMarkets = async (
  userId: string,
  differentThanTheseContracts: Contract[]
) => {
  // get contracts with unique bettor ids with this user
  const contractsUserHasBetOn = await getValues<Contract>(
    firestore
      .collection('contracts')
      .where('uniqueBettorIds', 'array-contains', userId)
  )
  if (contractsUserHasBetOn.length === 0) return []
  // count the number of times each unique bettor id appears on those contracts
  const bettorIdsToCounts = countBy(
    contractsUserHasBetOn.map((contract) => contract.uniqueBettorIds).flat(),
    (bettorId) => bettorId
  )

  // sort by number of times they appear with at least 2 appearances
  const sortedBettorIds = Object.entries(bettorIdsToCounts)
    .sort((a, b) => b[1] - a[1])
    .filter((bettorId) => bettorId[1] > 2)
    .map((entry) => entry[0])
    .filter((bettorId) => bettorId !== userId)

  // get the top 10 most similar bettors (excluding this user)
  const similarBettorIds = sortedBettorIds.slice(0, 10)
  if (similarBettorIds.length === 0) return []

  // get contracts with unique bettor ids with this user
  const contractsSimilarBettorsHaveBetOn = uniqBy(
    (
      await getValues<Contract>(
        firestore
          .collection('contracts')
          .where(
            'uniqueBettorIds',
            'array-contains-any',
            similarBettorIds.slice(0, 10)
          )
          .orderBy('popularityScore', 'desc')
          .limit(200)
      )
    ).filter(
      (contract) =>
        !contract.uniqueBettorIds?.includes(userId) &&
        (contract.popularityScore ?? 0) > MINIMUM_POPULARITY_SCORE
    ),
    (contract) => contract.id
  )

  // sort the contracts by how many times similar bettor ids are in their unique bettor ids array
  const sortedContractsInSimilarBettorsBets = contractsSimilarBettorsHaveBetOn
    .map((contract) => {
      const appearances = contract.uniqueBettorIds?.filter((bettorId) =>
        similarBettorIds.includes(bettorId)
      ).length
      return [contract, appearances] as [Contract, number]
    })
    .sort((a, b) => b[1] - a[1])
    .map((entry) => entry[0])

  const uniqueSortedContractsInSimilarBettorsBets = removeSimilarQuestions(
    sortedContractsInSimilarBettorsBets,
    sortedContractsInSimilarBettorsBets,
    true
  )

  const topMostSimilarContracts = removeSimilarQuestions(
    uniqueSortedContractsInSimilarBettorsBets,
    differentThanTheseContracts,
    false
  ).slice(0, 10)

  // log(
  //   'top 10 sorted contracts other similar bettors have bet on',
  //   topMostSimilarContracts.map((c) => c.question)
  // )

  return topMostSimilarContracts
}

// search contract array by question and remove contracts with 3 matching words in the question
const removeSimilarQuestions = (
  contractsToFilter: Contract[],
  byContracts: Contract[],
  allowExactSameContracts: boolean
) => {
  // log(
  //   'contracts to filter by',
  //   byContracts.map((c) => c.question + ' ' + c.popularityScore)
  // )
  let contractsToRemove: Contract[] = []
  byContracts.length > 0 &&
    byContracts.forEach((contract) => {
      const contractQuestion = stripNonAlphaChars(
        contract.question.toLowerCase()
      )
      const contractQuestionWords = uniq(contractQuestion.split(' ')).filter(
        (w) => !IGNORE_WORDS.includes(w)
      )
      contractsToRemove = contractsToRemove.concat(
        contractsToFilter.filter(
          // Remove contracts with more than 2 matching (uncommon) words and a lower popularity score
          (c2) => {
            const significantOverlap =
              // TODO: we should probably use a library for comparing strings/sentiments
              uniq(
                stripNonAlphaChars(c2.question.toLowerCase()).split(' ')
              ).filter((word) => contractQuestionWords.includes(word)).length >
              2
            const lessPopular =
              (c2.popularityScore ?? 0) < (contract.popularityScore ?? 0)
            return (
              (significantOverlap && lessPopular) ||
              (allowExactSameContracts ? false : c2.id === contract.id)
            )
          }
        )
      )
    })
  // log(
  //   'contracts to filter out',
  //   contractsToRemove.map((c) => c.question)
  // )

  const returnContracts = contractsToFilter.filter(
    (cf) => !contractsToRemove.map((c) => c.id).includes(cf.id)
  )

  return returnContracts
}

const fiveMinutes = 5 * 60 * 1000
const seed = Math.round(Date.now() / fiveMinutes).toString()
const rng = createRNG(seed)

function chooseRandomSubset(contracts: Contract[], count: number) {
  shuffle(contracts, rng)
  return contracts.slice(0, count)
}

function stripNonAlphaChars(str: string) {
  return str.replace(/[^\w\s']|_/g, '').replace(/\s+/g, ' ')
}

const IGNORE_WORDS = [
  'the',
  'a',
  'an',
  'and',
  'or',
  'of',
  'to',
  'in',
  'on',
  'will',
  'be',
  'is',
  'are',
  'for',
  'by',
  'at',
  'from',
  'what',
  'when',
  'which',
  'that',
  'it',
  'as',
  'if',
  'then',
  'than',
  'but',
  'have',
  'has',
  'had',
]