Denormalize some contract comment fields (#760)

* Make `groupConsecutive` more capable

* Put denormalized `contractQuestion` and `contractSlug` on comments

* Update user profile UI to use new denormalized fields

* `/Austin` -> `/market`

common/comment.ts

@@ -20,4 +20,6 @@ export type Comment = {
   userName: string
   userUsername: string
   userAvatarUrl?: string
+  contractSlug?: string
+  contractQuestion?: string
 }

common/util/array.ts

@@ -1,3 +1,5 @@
+import { isEqual } from 'lodash'
+
 export function filterDefined<T>(array: (T | null | undefined)[]) {
   return array.filter((item) => item !== null && item !== undefined) as T[]
 }
@@ -26,7 +28,7 @@ export function groupConsecutive<T, U>(xs: T[], key: (x: T) => U) {
   let curr = { key: key(xs[0]), items: [xs[0]] }
   for (const x of xs.slice(1)) {
     const k = key(x)
-    if (k !== curr.key) {
+    if (!isEqual(key, curr.key)) {
       result.push(curr)
       curr = { key: k, items: [x] }
     } else {

firestore.indexes.json

@@ -496,6 +496,28 @@
         }
       ]
     },
+    {
+      "collectionGroup": "comments",
+      "fieldPath": "contractId",
+      "indexes": [
+        {
+          "order": "ASCENDING",
+          "queryScope": "COLLECTION"
+        },
+        {
+          "order": "DESCENDING",
+          "queryScope": "COLLECTION"
+        },
+        {
+          "arrayConfig": "CONTAINS",
+          "queryScope": "COLLECTION"
+        },
+        {
+          "order": "ASCENDING",
+          "queryScope": "COLLECTION_GROUP"
+        }
+      ]
+    },
     {
       "collectionGroup": "comments",
       "fieldPath": "createdTime",

functions/src/on-create-comment-on-contract.ts

@@ -24,6 +24,11 @@ export const onCreateCommentOnContract = functions
     if (!contract)
       throw new Error('Could not find contract corresponding with comment')
 
+    await change.ref.update({
+      contractSlug: contract.slug,
+      contractQuestion: contract.question,
+    })
+
     const comment = change.data() as Comment
     const lastCommentTime = comment.createdTime
 

functions/src/scripts/denormalize-comment-contract-data.ts

@@ -0,0 +1,70 @@
+// Filling in the contract-based fields on comments.
+
+import * as admin from 'firebase-admin'
+import { initAdmin } from './script-init'
+import {
+  DocumentCorrespondence,
+  findDiffs,
+  describeDiff,
+  applyDiff,
+} from './denormalize'
+import { DocumentSnapshot, Transaction } from 'firebase-admin/firestore'
+
+initAdmin()
+const firestore = admin.firestore()
+
+async function getContractsById(transaction: Transaction) {
+  const contracts = await transaction.get(firestore.collection('contracts'))
+  const results = Object.fromEntries(contracts.docs.map((doc) => [doc.id, doc]))
+  console.log(`Found ${contracts.size} contracts.`)
+  return results
+}
+
+async function getCommentsByContractId(transaction: Transaction) {
+  const comments = await transaction.get(
+    firestore.collectionGroup('comments').where('contractId', '!=', null)
+  )
+  const results = new Map<string, DocumentSnapshot[]>()
+  comments.forEach((doc) => {
+    const contractId = doc.get('contractId')
+    const contractComments = results.get(contractId) || []
+    contractComments.push(doc)
+    results.set(contractId, contractComments)
+  })
+  console.log(`Found ${comments.size} comments on ${results.size} contracts.`)
+  return results
+}
+
+async function denormalize() {
+  let hasMore = true
+  while (hasMore) {
+    hasMore = await admin.firestore().runTransaction(async (transaction) => {
+      const [contractsById, commentsByContractId] = await Promise.all([
+        getContractsById(transaction),
+        getCommentsByContractId(transaction),
+      ])
+      const mapping = Object.entries(contractsById).map(
+        ([id, doc]): DocumentCorrespondence => {
+          return [doc, commentsByContractId.get(id) || []]
+        }
+      )
+      const slugDiffs = findDiffs(mapping, 'slug', 'contractSlug')
+      const qDiffs = findDiffs(mapping, 'question', 'contractQuestion')
+      console.log(`Found ${slugDiffs.length} comments with mismatched slugs.`)
+      console.log(`Found ${qDiffs.length} comments with mismatched questions.`)
+      const diffs = slugDiffs.concat(qDiffs)
+      diffs.slice(0, 500).forEach((d) => {
+        console.log(describeDiff(d))
+        applyDiff(transaction, d)
+      })
+      if (diffs.length > 500) {
+        console.log(`Applying first 500 because of Firestore limit...`)
+      }
+      return diffs.length > 500
+    })
+  }
+}
+
+if (require.main === module) {
+  denormalize().catch((e) => console.error(e))
+}

web/components/comments-list.tsx

@@ -1,12 +1,8 @@
 import { useEffect, useState } from 'react'
-import { Dictionary, keyBy, uniq } from 'lodash'
 
 import { Comment } from 'common/comment'
-import { Contract } from 'common/contract'
+import { groupConsecutive } from 'common/util/array'
-import { filterDefined, groupConsecutive } from 'common/util/array'
-import { contractPath } from 'web/lib/firebase/contracts'
 import { getUsersComments } from 'web/lib/firebase/comments'
-import { getContractFromId } from 'web/lib/firebase/contracts'
 import { SiteLink } from './site-link'
 import { Row } from './layout/row'
 import { Avatar } from './avatar'
@@ -20,12 +16,21 @@ import { LoadingIndicator } from './loading-indicator'
 
 const COMMENTS_PER_PAGE = 50
 
-type ContractComment = Comment & { contractId: string }
+type ContractComment = Comment & {
+  contractId: string
+  contractSlug: string
+  contractQuestion: string
+}
+
+function contractPath(slug: string) {
+  // by convention this includes the contract creator username, but we don't
+  // have that handy, so we just put /market/
+  return `/market/${slug}`
+}
 
 export function UserCommentsList(props: { user: User }) {
   const { user } = props
   const [comments, setComments] = useState<ContractComment[] | undefined>()
-  const [contracts, setContracts] = useState<Dictionary<Contract> | undefined>()
   const [page, setPage] = useState(0)
   const start = page * COMMENTS_PER_PAGE
   const end = start + COMMENTS_PER_PAGE
@@ -37,34 +42,23 @@ export function UserCommentsList(props: { user: User }) {
     })
   }, [user.id])
 
-  useEffect(() => {
+  if (comments == null) {
-    if (comments) {
-      const contractIds = uniq(comments.map((c) => c.contractId))
-      Promise.all(contractIds.map(getContractFromId)).then((contracts) => {
-        setContracts(keyBy(filterDefined(contracts), 'id'))
-      })
-    }
-  }, [comments])
-
-  if (comments == null || contracts == null) {
     return <LoadingIndicator />
   }
 
-  const pageComments = groupConsecutive(
+  const pageComments = groupConsecutive(comments.slice(start, end), (c) => {
-    comments.slice(start, end),
+    return { question: c.contractQuestion, slug: c.contractSlug }
-    (c) => c.contractId
+  })
-  )
   return (
     <Col className={'bg-white'}>
       {pageComments.map(({ key, items }, i) => {
-        const contract = contracts[key]
         return (
           <div key={start + i} className="border-b p-5">
             <SiteLink
               className="mb-2 block pb-2 font-medium text-indigo-700"
-              href={contractPath(contract)}
+              href={contractPath(key.slug)}
             >
-              {contract.question}
+              {key.question}
             </SiteLink>
             <Col className="gap-6">
               {items.map((comment) => (