Write migration script to fix up old contract/post/comment text

common/comment.ts

@@ -1,5 +1,3 @@
-import type { JSONContent } from '@tiptap/core'
-
 export type AnyCommentType = OnContract | OnGroup | OnPost
 
 // Currently, comments are created after the bet, not atomically with the bet.
@@ -8,10 +6,7 @@ export type Comment<T extends AnyCommentType = AnyCommentType> = {
   id: string
   replyToCommentId?: string
   userId: string
-
+  content: string
-  /** @deprecated - content now stored as JSON in content*/
-  text?: string
-  content: JSONContent
   createdTime: number
 
   // Denormalized, for rendering comments

common/contract.ts

@@ -1,6 +1,5 @@
 import { Answer } from './answer'
 import { Fees } from './fees'
-import { JSONContent } from '@tiptap/core'
 import { GroupLink } from 'common/group'
 
 export type AnyMechanism = DPM | CPMM
@@ -28,7 +27,7 @@ export type Contract<T extends AnyContractType = AnyContractType> = {
   creatorAvatarUrl?: string
 
   question: string
-  description: string | JSONContent // More info about what the contract is about
+  description: string // More info about what the contract is about
   tags: string[]
   lowercaseTags: string[]
   visibility: visibility

common/new-contract.ts

@@ -69,7 +69,7 @@ export function getNewContract(
     creatorAvatarUrl: creator.avatarUrl,
 
     question: question.trim(),
-    description,
+    description: JSON.stringify(description),
     tags,
     lowercaseTags,
     visibility,

common/post.ts

@@ -1,9 +1,7 @@
-import { JSONContent } from '@tiptap/core'
-
 export type Post = {
   id: string
   title: string
-  content: JSONContent
+  content: string
   creatorId: string // User id
   createdTime: number
   slug: string

common/util/parse.ts

@@ -79,6 +79,18 @@ export function parseMentions(data: JSONContent): string[] {
   return uniq(mentions)
 }
 
+export const plainTextToProseMirror = (text: string): JSONContent => {
+  return {
+    type: 'doc',
+    content: [
+      {
+        type: 'paragraph',
+        content: [{ type: 'text', text }],
+      },
+    ],
+  }
+}
+
 // can't just do [StarterKit, Image...] because it doesn't work with cjs imports
 export const exhibitExts = [
   Blockquote,

functions/src/create-market.ts

@@ -29,6 +29,7 @@ import {
 } from '../../common/antes'
 import { Answer, getNoneAnswer } from '../../common/answer'
 import { getNewContract } from '../../common/new-contract'
+import { plainTextToProseMirror } from '../../common/util/parse'
 import { NUMERIC_BUCKET_COUNT } from '../../common/numeric-constants'
 import { User } from '../../common/user'
 import { Group, GroupLink, MAX_ID_LENGTH } from '../../common/group'
@@ -187,15 +188,7 @@ export const createmarket = newEndpoint({}, async (req, auth) => {
   // convert string descriptions into JSONContent
   const newDescription =
     typeof description === 'string'
-      ? {
+      ? plainTextToProseMirror(description)
-          type: 'doc',
-          content: [
-            {
-              type: 'paragraph',
-              content: [{ type: 'text', text: description }],
-            },
-          ],
-        }
       : description ?? {}
 
   const contract = getNewContract(

functions/src/create-post.ts

@@ -57,7 +57,7 @@ export const createpost = newEndpoint({}, async (req, auth) => {
     slug,
     title,
     createdTime: Date.now(),
-    content: content,
+    content: JSON.stringify(content),
   }
 
   await postRef.create(post)

functions/src/on-create-comment-on-contract.ts

@@ -174,7 +174,8 @@ export const onCreateCommentOnContract = functions
       ? comments.find((c) => c.id === comment.replyToCommentId)?.userId
       : answer?.userId
 
-    const mentionedUsers = compact(parseMentions(comment.content))
+    const parsedContent = JSON.parse(comment.content)
+    const mentionedUsers = compact(parseMentions(parsedContent))
     const repliedUsers: replied_users_info = {}
 
     // The parent of the reply chain could be a comment or an answer
@@ -210,7 +211,7 @@ export const onCreateCommentOnContract = functions
       'created',
       commentCreator,
       eventId,
-      richTextToString(comment.content),
+      richTextToString(parsedContent),
       contract,
       {
         repliedUsersInfo: repliedUsers,

functions/src/on-create-contract.ts

@@ -17,7 +17,7 @@ export const onCreateContract = functions
     const contractCreator = await getUser(contract.creatorId)
     if (!contractCreator) throw new Error('Could not find contract creator')
 
-    const desc = contract.description as JSONContent
+    const desc = JSON.parse(contract.description) as JSONContent
     const mentioned = parseMentions(desc)
     await addUserToContractFollowers(contract.id, contractCreator.id)
 

functions/src/scripts/migrate-rich-text.ts

@@ -0,0 +1,113 @@
+// We have three kinds of rich text:
+// - Contract descriptions.
+// - Comment text.
+// - Post contents.
+// These are stored in two different ways:
+// - As plaintext strings.
+// - As structured ProseMirror JSON in Firestore.
+// We want to make all of these into:
+// - Strings containing serialized ProseMirror JSON.
+
+import * as admin from 'firebase-admin'
+import { FieldValue } from 'firebase-admin/firestore'
+import { initAdmin } from './script-init'
+import { log, writeAsync } from '../utils'
+import { filterDefined } from '../../../common/util/array'
+import { plainTextToProseMirror } from '../../../common/util/parse'
+
+initAdmin()
+const firestore = admin.firestore()
+
+const isValidJSON = (s: string) => {
+  try {
+    JSON.parse(s)
+    return true
+  } catch {
+    return false
+  }
+}
+
+const migrateContractDescriptions = async () => {
+  const contractQ = await firestore.collection('contracts').get()
+  console.log(`Loaded ${contractQ.size} contracts.`)
+  const updates = filterDefined(
+    contractQ.docs.map((doc) => {
+      const fields: { [k: string]: unknown } = {}
+      const oldDescription = doc.get('description')
+      if (typeof oldDescription === 'string') {
+        if (isValidJSON(oldDescription)) {
+          // this one is already good
+          return null
+        } else {
+          fields.description = JSON.stringify(
+            plainTextToProseMirror(oldDescription)
+          )
+        }
+      } else if (oldDescription != null) {
+        // already JSON, just need to serialize into string
+        fields.description = JSON.stringify(oldDescription)
+      } else {
+        throw new Error('Content had null description for some reason.')
+      }
+      return { doc: doc.ref, fields }
+    })
+  )
+  log(`Found ${updates.length} contracts with old format descriptions.`)
+  await writeAsync(firestore, updates)
+}
+
+const migrateCommentContents = async () => {
+  const commentQ = await firestore.collectionGroup('comments').get()
+  console.log(`Loaded ${commentQ.size} comments.`)
+  const updates = filterDefined(
+    commentQ.docs.map((doc) => {
+      const fields: { [k: string]: unknown } = {}
+      const oldText = doc.get('text')
+      const oldContent = doc.get('content')
+      if (typeof oldContent === 'string') {
+        // this one is already good
+        return null
+      } else if (oldContent != null) {
+        // already JSON, just need to serialize into string
+        fields.content = JSON.stringify(oldContent)
+      } else if (oldText != null) {
+        fields.text = FieldValue.delete()
+        fields.content = JSON.stringify(plainTextToProseMirror(oldText))
+      } else {
+        throw new Error('Comment mysteriously had neither text nor content.')
+      }
+      return { doc: doc.ref, fields }
+    })
+  )
+  log(`Found ${updates.length} comments with old format content.`)
+  await writeAsync(firestore, updates)
+}
+
+const migratePostContents = async () => {
+  const postQ = await firestore.collection('posts').get()
+  console.log(`Loaded ${postQ.size} posts.`)
+  const updates = filterDefined(
+    postQ.docs.map((doc) => {
+      const fields: { [k: string]: unknown } = {}
+      const oldContent = doc.get('content')
+      if (typeof oldContent === 'string') {
+        // this one is already good
+        return null
+      } else if (oldContent != null) {
+        // already JSON, just need to serialize into string
+        fields.content = JSON.stringify(oldContent)
+      } else {
+        throw new Error('Post had null content for some reason.')
+      }
+      return { doc: doc.ref, fields }
+    })
+  )
+  log(`Found ${updates.length} posts with old format content.`)
+  await writeAsync(firestore, updates)
+}
+
+if (require.main === module) {
+  migrateContractDescriptions()
+  migrateCommentContents()
+  migratePostContents()
+}

web/components/comments-list.tsx

@@ -8,7 +8,7 @@ import { Avatar } from './avatar'
 import { RelativeTimestamp } from './relative-timestamp'
 import { User } from 'common/user'
 import { Col } from './layout/col'
-import { Content } from './editor'
+import { RichContent } from './editor'
 import { LoadingIndicator } from './loading-indicator'
 import { UserLink } from 'web/components/user-link'
 import { PaginationNextPrev } from 'web/components/pagination'
@@ -99,7 +99,7 @@ function ProfileCommentGroup(props: {
 
 function ProfileComment(props: { comment: ContractComment }) {
   const { comment } = props
-  const { text, content, userUsername, userName, userAvatarUrl, createdTime } =
+  const { content, userUsername, userName, userAvatarUrl, createdTime } =
     comment
   // TODO: find and attach relevant bets by comment betId at some point
   return (
@@ -114,7 +114,7 @@ function ProfileComment(props: { comment: ContractComment }) {
           />{' '}
           <RelativeTimestamp time={createdTime} />
         </p>
-        <Content content={content || text} smallImage />
+        <RichContent content={JSON.parse(content)} smallImage />
       </div>
     </Row>
   )

web/components/contract/contract-description.tsx

@@ -9,7 +9,7 @@ import { useAdmin } from 'web/hooks/use-admin'
 import { useUser } from 'web/hooks/use-user'
 import { updateContract } from 'web/lib/firebase/contracts'
 import { Row } from '../layout/row'
-import { Content } from '../editor'
+import { RichContent } from '../editor'
 import { TextEditor, useTextEditor } from 'web/components/editor'
 import { Button } from '../button'
 import { Spacer } from '../layout/spacer'
@@ -29,7 +29,7 @@ export function ContractDescription(props: {
       {isCreator || isAdmin ? (
         <RichEditContract contract={contract} isAdmin={isAdmin && !isCreator} />
       ) : (
-        <Content content={contract.description} />
+        <RichContent content={JSON.parse(contract.description)} />
       )}
     </div>
   )
@@ -60,7 +60,7 @@ function RichEditContract(props: { contract: Contract; isAdmin?: boolean }) {
     const lowercaseTags = tags.map((tag) => tag.toLowerCase())
 
     await updateContract(contract.id, {
-      description: editor.getJSON(),
+      description: JSON.stringify(editor.getJSON()),
       tags,
       lowercaseTags,
     })
@@ -88,7 +88,7 @@ function RichEditContract(props: { contract: Contract; isAdmin?: boolean }) {
     </>
   ) : (
     <>
-      <Content content={contract.description} />
+      <RichContent content={JSON.parse(contract.description)} />
       <Spacer h={2} />
       <Row className="items-center gap-2">
         {isAdmin && 'Admin: '}
@@ -139,9 +139,11 @@ function EditQuestion(props: {
     setEditing(false)
     await updateContract(contract.id, {
       question: newText,
-      description: joinContent(
+      description: JSON.stringify(
-        contract.description,
+        joinContent(
-        questionChanged(contract.question, newText)
+          JSON.parse(contract.description),
+          questionChanged(contract.question, newText)
+        )
       ),
     })
   }

web/components/contract/contract-details.tsx

@@ -380,7 +380,7 @@ function EditableCloseDate(props: {
 
       updateContract(contract.id, {
         closeTime: newCloseTime,
-        description: editor.getJSON(),
+        description: JSON.stringify(editor.getJSON()),
       })
 
       setIsEditingCloseTime(false)

web/components/editor.tsx

@@ -14,7 +14,6 @@ import { Image } from '@tiptap/extension-image'
 import { Link } from '@tiptap/extension-link'
 import clsx from 'clsx'
 import { useEffect, useState } from 'react'
-import { Linkify } from './linkify'
 import { uploadImage } from 'web/lib/firebase/storage'
 import { useMutation } from 'react-query'
 import { FileUploadButton } from './file-upload-button'
@@ -316,6 +315,7 @@ export function RichContent(props: {
   smallImage?: boolean
 }) {
   const { className, content, smallImage } = props
+
   const editor = useEditor({
     editorProps: { attributes: { class: proseClass } },
     extensions: [
@@ -341,23 +341,3 @@ export function RichContent(props: {
 
   return <EditorContent className={className} editor={editor} />
 }
-
-// backwards compatibility: we used to store content as strings
-export function Content(props: {
-  content: JSONContent | string
-  className?: string
-  smallImage?: boolean
-}) {
-  const { className, content } = props
-  return typeof content === 'string' ? (
-    <Linkify
-      className={clsx(
-        className,
-        'whitespace-pre-line font-light leading-relaxed'
-      )}
-      text={content}
-    />
-  ) : (
-    <RichContent {...props} />
-  )
-}

web/components/feed/feed-comments.tsx

@@ -15,7 +15,7 @@ import { Col } from 'web/components/layout/col'
 import { track } from 'web/lib/service/analytics'
 import { Tipper } from '../tipper'
 import { CommentTipMap, CommentTips } from 'web/hooks/use-tip-txns'
-import { Content } from '../editor'
+import { RichContent } from '../editor'
 import { Editor } from '@tiptap/react'
 import { UserLink } from 'web/components/user-link'
 import { CommentInput } from '../comment-input'
@@ -76,7 +76,6 @@ export function FeedComment(props: {
 }) {
   const { contract, comment, tips, indent, onReplyClick } = props
   const {
-    text,
     content,
     userUsername,
     userName,
@@ -163,9 +162,9 @@ export function FeedComment(props: {
             elementId={comment.id}
           />
         </div>
-        <Content
+        <RichContent
           className="mt-2 text-[15px] text-gray-700"
-          content={content || text}
+          content={JSON.parse(content)}
           smallImage
         />
         <Row className="mt-2 items-center gap-6 text-xs text-gray-500">

web/components/groups/group-about-post.tsx

@@ -1,5 +1,5 @@
 import { Row } from '../layout/row'
-import { Content } from '../editor'
+import { RichContent } from '../editor'
 import { TextEditor, useTextEditor } from 'web/components/editor'
 import { Button } from '../button'
 import { Spacer } from '../layout/spacer'
@@ -24,7 +24,9 @@ export function GroupAboutPost(props: {
   return (
     <div className="rounded-md bg-white p-4 ">
       {isEditable && <RichEditGroupAboutPost group={group} post={post} />}
-      {!isEditable && post && <Content content={post.content} />}
+      {!isEditable && post && (
+        <RichContent content={JSON.parse(post.content)} />
+      )}
     </div>
   )
 }
@@ -56,7 +58,7 @@ function RichEditGroupAboutPost(props: { group: Group; post: Post | null }) {
       })
     } else {
       await updatePost(post, {
-        content: newPost.content,
+        content: JSON.stringify(newPost.content),
       })
     }
   }
@@ -124,7 +126,7 @@ function RichEditGroupAboutPost(props: { group: Group; post: Post | null }) {
             </Button>
           </div>
 
-          <Content content={post.content} />
+          <RichContent content={JSON.parse(post.content)} />
           <Spacer h={2} />
         </div>
       )}

web/lib/firebase/comments.ts

@@ -100,7 +100,7 @@ async function createComment(
   const comment = removeUndefinedProps({
     id: ref.id,
     userId: user.id,
-    content: content,
+    content: JSON.stringify(content),
     createdTime: Date.now(),
     userName: user.name,
     userUsername: user.username,

web/pages/api/v0/_types.ts

@@ -53,7 +53,7 @@ export type FullMarket = LiteMarket & {
   bets: Bet[]
   comments: Comment[]
   answers?: ApiAnswer[]
-  description: string | JSONContent
+  description: JSONContent
   textDescription: string // string version of description
 }
 
@@ -155,18 +155,15 @@ export function toFullMarket(
         )
       : undefined
 
-  const { description } = contract
+  const parsedDescription = JSON.parse(contract.description)
 
   return {
     ...liteMarket,
     answers,
     comments,
     bets,
-    description,
+    description: parsedDescription,
-    textDescription:
+    textDescription: richTextToString(parsedDescription),
-      typeof description === 'string'
-        ? description
-        : richTextToString(description),
   }
 }
 

web/pages/post/[...slugs]/index.tsx

@@ -4,7 +4,7 @@ import { postPath, getPostBySlug, updatePost } from 'web/lib/firebase/posts'
 import { Post } from 'common/post'
 import { Title } from 'web/components/title'
 import { Spacer } from 'web/components/layout/spacer'
-import { Content, TextEditor, useTextEditor } from 'web/components/editor'
+import { RichContent, TextEditor, useTextEditor } from 'web/components/editor'
 import { getUser, User } from 'web/lib/firebase/users'
 import { PencilIcon, ShareIcon } from '@heroicons/react/solid'
 import clsx from 'clsx'
@@ -110,7 +110,7 @@ export default function PostPage(props: {
             {user && user.id === post.creatorId ? (
               <RichEditPost post={post} />
             ) : (
-              <Content content={post.content} />
+              <RichContent content={JSON.parse(post.content)} />
             )}
           </div>
         </div>
@@ -178,7 +178,7 @@ function RichEditPost(props: { post: Post }) {
     if (!editor) return
 
     await updatePost(post, {
-      content: editor.getJSON(),
+      content: JSON.stringify(editor.getJSON()),
     })
   }
 
@@ -219,7 +219,7 @@ function RichEditPost(props: { post: Post }) {
           </Button>
         </div>
 
-        <Content content={post.content} />
+        <RichContent content={JSON.parse(post.content)} />
         <Spacer h={2} />
       </div>
     </>

web/posts/post-comments.tsx

@@ -9,7 +9,7 @@ import { useRouter } from 'next/router'
 import { useEffect, useState } from 'react'
 import { Avatar } from 'web/components/avatar'
 import { CommentInput } from 'web/components/comment-input'
-import { Content } from 'web/components/editor'
+import { RichContent } from 'web/components/editor'
 import { CopyLinkDateTimeComponent } from 'web/components/feed/copy-link-date-time'
 import { Col } from 'web/components/layout/col'
 import { Row } from 'web/components/layout/row'
@@ -108,7 +108,7 @@ export function PostComment(props: {
   onReplyClick?: (comment: PostComment) => void
 }) {
   const { post, comment, tips, indent, onReplyClick } = props
-  const { text, content, userUsername, userName, userAvatarUrl, createdTime } =
+  const { content, userUsername, userName, userAvatarUrl, createdTime } =
     comment
 
   const [highlighted, setHighlighted] = useState(false)
@@ -150,9 +150,9 @@ export function PostComment(props: {
             elementId={comment.id}
           />
         </div>
-        <Content
+        <RichContent
           className="mt-2 text-[15px] text-gray-700"
-          content={content || text}
+          content={JSON.parse(content)}
           smallImage
         />
         <Row className="mt-2 items-center gap-6 text-xs text-gray-500">