From d309a4f31ba2602cf33d4dbd5d5a825d2a9ed273 Mon Sep 17 00:00:00 2001 From: Marshall Polaris Date: Thu, 22 Sep 2022 21:44:48 -0700 Subject: [PATCH 1/3] Extract utility method --- common/util/parse.ts | 12 ++++++++++++ functions/src/create-market.ts | 11 ++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/common/util/parse.ts b/common/util/parse.ts index 0bbd5cd9..bc5dc4d2 100644 --- a/common/util/parse.ts +++ b/common/util/parse.ts @@ -79,6 +79,18 @@ export function parseMentions(data: JSONContent): string[] { return uniq(mentions) } +export const plainTextToProseMirror = (text: string): JSONContent => { + return { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text }], + }, + ], + } +} + // can't just do [StarterKit, Image...] because it doesn't work with cjs imports export const exhibitExts = [ Blockquote, diff --git a/functions/src/create-market.ts b/functions/src/create-market.ts index 300d91f2..098080ea 100644 --- a/functions/src/create-market.ts +++ b/functions/src/create-market.ts @@ -29,6 +29,7 @@ import { } from '../../common/antes' import { Answer, getNoneAnswer } from '../../common/answer' import { getNewContract } from '../../common/new-contract' +import { plainTextToProseMirror } from '../../common/util/parse' import { NUMERIC_BUCKET_COUNT } from '../../common/numeric-constants' import { User } from '../../common/user' import { Group, GroupLink, MAX_ID_LENGTH } from '../../common/group' @@ -187,15 +188,7 @@ export const createmarket = newEndpoint({}, async (req, auth) => { // convert string descriptions into JSONContent const newDescription = typeof description === 'string' - ? { - type: 'doc', - content: [ - { - type: 'paragraph', - content: [{ type: 'text', text: description }], - }, - ], - } + ? plainTextToProseMirror(description) : description ?? {} const contract = getNewContract( From 5cda037f6b64b68d4ef99ffa1e24766d755cf2ed Mon Sep 17 00:00:00 2001 From: Marshall Polaris Date: Thu, 22 Sep 2022 21:58:42 -0700 Subject: [PATCH 2/3] Store all text content as JSON-serialized ProseMirror docs --- common/comment.ts | 7 +----- common/contract.ts | 3 +-- common/new-contract.ts | 2 +- common/post.ts | 4 +--- functions/src/create-post.ts | 2 +- .../src/on-create-comment-on-contract.ts | 5 +++-- functions/src/on-create-contract.ts | 2 +- web/components/comments-list.tsx | 6 ++--- .../contract/contract-description.tsx | 16 ++++++++------ web/components/contract/contract-details.tsx | 2 +- web/components/editor.tsx | 22 +------------------ web/components/feed/feed-comments.tsx | 7 +++--- web/components/groups/group-about-post.tsx | 10 +++++---- web/lib/firebase/comments.ts | 2 +- web/pages/api/v0/_types.ts | 11 ++++------ web/pages/post/[...slugs]/index.tsx | 8 +++---- web/posts/post-comments.tsx | 8 +++---- 17 files changed, 45 insertions(+), 72 deletions(-) diff --git a/common/comment.ts b/common/comment.ts index cdb62fd3..6ace444a 100644 --- a/common/comment.ts +++ b/common/comment.ts @@ -1,5 +1,3 @@ -import type { JSONContent } from '@tiptap/core' - export type AnyCommentType = OnContract | OnGroup | OnPost // Currently, comments are created after the bet, not atomically with the bet. @@ -8,10 +6,7 @@ export type Comment = { id: string replyToCommentId?: string userId: string - - /** @deprecated - content now stored as JSON in content*/ - text?: string - content: JSONContent + content: string createdTime: number // Denormalized, for rendering comments diff --git a/common/contract.ts b/common/contract.ts index 248c9745..eed907be 100644 --- a/common/contract.ts +++ b/common/contract.ts @@ -1,6 +1,5 @@ import { Answer } from './answer' import { Fees } from './fees' -import { JSONContent } from '@tiptap/core' import { GroupLink } from 'common/group' export type AnyMechanism = DPM | CPMM @@ -28,7 +27,7 @@ export type Contract = { creatorAvatarUrl?: string question: string - description: string | JSONContent // More info about what the contract is about + description: string // More info about what the contract is about tags: string[] lowercaseTags: string[] visibility: visibility diff --git a/common/new-contract.ts b/common/new-contract.ts index 431f435e..d54ae3cb 100644 --- a/common/new-contract.ts +++ b/common/new-contract.ts @@ -69,7 +69,7 @@ export function getNewContract( creatorAvatarUrl: creator.avatarUrl, question: question.trim(), - description, + description: JSON.stringify(description), tags, lowercaseTags, visibility, diff --git a/common/post.ts b/common/post.ts index 05eab685..94fc58ff 100644 --- a/common/post.ts +++ b/common/post.ts @@ -1,9 +1,7 @@ -import { JSONContent } from '@tiptap/core' - export type Post = { id: string title: string - content: JSONContent + content: string creatorId: string // User id createdTime: number slug: string diff --git a/functions/src/create-post.ts b/functions/src/create-post.ts index 113a34bd..52901efb 100644 --- a/functions/src/create-post.ts +++ b/functions/src/create-post.ts @@ -57,7 +57,7 @@ export const createpost = newEndpoint({}, async (req, auth) => { slug, title, createdTime: Date.now(), - content: content, + content: JSON.stringify(content), } await postRef.create(post) diff --git a/functions/src/on-create-comment-on-contract.ts b/functions/src/on-create-comment-on-contract.ts index d1f0a503..648c1949 100644 --- a/functions/src/on-create-comment-on-contract.ts +++ b/functions/src/on-create-comment-on-contract.ts @@ -174,7 +174,8 @@ export const onCreateCommentOnContract = functions ? comments.find((c) => c.id === comment.replyToCommentId)?.userId : answer?.userId - const mentionedUsers = compact(parseMentions(comment.content)) + const parsedContent = JSON.parse(comment.content) + const mentionedUsers = compact(parseMentions(parsedContent)) const repliedUsers: replied_users_info = {} // The parent of the reply chain could be a comment or an answer @@ -210,7 +211,7 @@ export const onCreateCommentOnContract = functions 'created', commentCreator, eventId, - richTextToString(comment.content), + richTextToString(parsedContent), contract, { repliedUsersInfo: repliedUsers, diff --git a/functions/src/on-create-contract.ts b/functions/src/on-create-contract.ts index b613142b..6260c0ed 100644 --- a/functions/src/on-create-contract.ts +++ b/functions/src/on-create-contract.ts @@ -17,7 +17,7 @@ export const onCreateContract = functions const contractCreator = await getUser(contract.creatorId) if (!contractCreator) throw new Error('Could not find contract creator') - const desc = contract.description as JSONContent + const desc = JSON.parse(contract.description) as JSONContent const mentioned = parseMentions(desc) await addUserToContractFollowers(contract.id, contractCreator.id) diff --git a/web/components/comments-list.tsx b/web/components/comments-list.tsx index 0b1c3843..89d12ab8 100644 --- a/web/components/comments-list.tsx +++ b/web/components/comments-list.tsx @@ -8,7 +8,7 @@ import { Avatar } from './avatar' import { RelativeTimestamp } from './relative-timestamp' import { User } from 'common/user' import { Col } from './layout/col' -import { Content } from './editor' +import { RichContent } from './editor' import { LoadingIndicator } from './loading-indicator' import { UserLink } from 'web/components/user-link' import { PaginationNextPrev } from 'web/components/pagination' @@ -99,7 +99,7 @@ function ProfileCommentGroup(props: { function ProfileComment(props: { comment: ContractComment }) { const { comment } = props - const { text, content, userUsername, userName, userAvatarUrl, createdTime } = + const { content, userUsername, userName, userAvatarUrl, createdTime } = comment // TODO: find and attach relevant bets by comment betId at some point return ( @@ -114,7 +114,7 @@ function ProfileComment(props: { comment: ContractComment }) { />{' '}

- + ) diff --git a/web/components/contract/contract-description.tsx b/web/components/contract/contract-description.tsx index 53557305..df797871 100644 --- a/web/components/contract/contract-description.tsx +++ b/web/components/contract/contract-description.tsx @@ -9,7 +9,7 @@ import { useAdmin } from 'web/hooks/use-admin' import { useUser } from 'web/hooks/use-user' import { updateContract } from 'web/lib/firebase/contracts' import { Row } from '../layout/row' -import { Content } from '../editor' +import { RichContent } from '../editor' import { TextEditor, useTextEditor } from 'web/components/editor' import { Button } from '../button' import { Spacer } from '../layout/spacer' @@ -29,7 +29,7 @@ export function ContractDescription(props: { {isCreator || isAdmin ? ( ) : ( - + )} ) @@ -60,7 +60,7 @@ function RichEditContract(props: { contract: Contract; isAdmin?: boolean }) { const lowercaseTags = tags.map((tag) => tag.toLowerCase()) await updateContract(contract.id, { - description: editor.getJSON(), + description: JSON.stringify(editor.getJSON()), tags, lowercaseTags, }) @@ -88,7 +88,7 @@ function RichEditContract(props: { contract: Contract; isAdmin?: boolean }) { ) : ( <> - + {isAdmin && 'Admin: '} @@ -139,9 +139,11 @@ function EditQuestion(props: { setEditing(false) await updateContract(contract.id, { question: newText, - description: joinContent( - contract.description, - questionChanged(contract.question, newText) + description: JSON.stringify( + joinContent( + JSON.parse(contract.description), + questionChanged(contract.question, newText) + ) ), }) } diff --git a/web/components/contract/contract-details.tsx b/web/components/contract/contract-details.tsx index 3525b9f9..ec94f733 100644 --- a/web/components/contract/contract-details.tsx +++ b/web/components/contract/contract-details.tsx @@ -380,7 +380,7 @@ function EditableCloseDate(props: { updateContract(contract.id, { closeTime: newCloseTime, - description: editor.getJSON(), + description: JSON.stringify(editor.getJSON()), }) setIsEditingCloseTime(false) diff --git a/web/components/editor.tsx b/web/components/editor.tsx index 95f18b3f..2c364734 100644 --- a/web/components/editor.tsx +++ b/web/components/editor.tsx @@ -14,7 +14,6 @@ import { Image } from '@tiptap/extension-image' import { Link } from '@tiptap/extension-link' import clsx from 'clsx' import { useEffect, useState } from 'react' -import { Linkify } from './linkify' import { uploadImage } from 'web/lib/firebase/storage' import { useMutation } from 'react-query' import { FileUploadButton } from './file-upload-button' @@ -316,6 +315,7 @@ export function RichContent(props: { smallImage?: boolean }) { const { className, content, smallImage } = props + const editor = useEditor({ editorProps: { attributes: { class: proseClass } }, extensions: [ @@ -341,23 +341,3 @@ export function RichContent(props: { return } - -// backwards compatibility: we used to store content as strings -export function Content(props: { - content: JSONContent | string - className?: string - smallImage?: boolean -}) { - const { className, content } = props - return typeof content === 'string' ? ( - - ) : ( - - ) -} diff --git a/web/components/feed/feed-comments.tsx b/web/components/feed/feed-comments.tsx index 1b62690b..a3cd6c32 100644 --- a/web/components/feed/feed-comments.tsx +++ b/web/components/feed/feed-comments.tsx @@ -15,7 +15,7 @@ import { Col } from 'web/components/layout/col' import { track } from 'web/lib/service/analytics' import { Tipper } from '../tipper' import { CommentTipMap, CommentTips } from 'web/hooks/use-tip-txns' -import { Content } from '../editor' +import { RichContent } from '../editor' import { Editor } from '@tiptap/react' import { UserLink } from 'web/components/user-link' import { CommentInput } from '../comment-input' @@ -76,7 +76,6 @@ export function FeedComment(props: { }) { const { contract, comment, tips, indent, onReplyClick } = props const { - text, content, userUsername, userName, @@ -163,9 +162,9 @@ export function FeedComment(props: { elementId={comment.id} /> - diff --git a/web/components/groups/group-about-post.tsx b/web/components/groups/group-about-post.tsx index 4d3046e9..7dcf8543 100644 --- a/web/components/groups/group-about-post.tsx +++ b/web/components/groups/group-about-post.tsx @@ -1,5 +1,5 @@ import { Row } from '../layout/row' -import { Content } from '../editor' +import { RichContent } from '../editor' import { TextEditor, useTextEditor } from 'web/components/editor' import { Button } from '../button' import { Spacer } from '../layout/spacer' @@ -24,7 +24,9 @@ export function GroupAboutPost(props: { return (
{isEditable && } - {!isEditable && post && } + {!isEditable && post && ( + + )}
) } @@ -56,7 +58,7 @@ function RichEditGroupAboutPost(props: { group: Group; post: Post | null }) { }) } else { await updatePost(post, { - content: newPost.content, + content: JSON.stringify(newPost.content), }) } } @@ -124,7 +126,7 @@ function RichEditGroupAboutPost(props: { group: Group; post: Post | null }) { - + )} diff --git a/web/lib/firebase/comments.ts b/web/lib/firebase/comments.ts index db4e8ede..22128f15 100644 --- a/web/lib/firebase/comments.ts +++ b/web/lib/firebase/comments.ts @@ -100,7 +100,7 @@ async function createComment( const comment = removeUndefinedProps({ id: ref.id, userId: user.id, - content: content, + content: JSON.stringify(content), createdTime: Date.now(), userName: user.name, userUsername: user.username, diff --git a/web/pages/api/v0/_types.ts b/web/pages/api/v0/_types.ts index ea2f053b..adf6bb59 100644 --- a/web/pages/api/v0/_types.ts +++ b/web/pages/api/v0/_types.ts @@ -53,7 +53,7 @@ export type FullMarket = LiteMarket & { bets: Bet[] comments: Comment[] answers?: ApiAnswer[] - description: string | JSONContent + description: JSONContent textDescription: string // string version of description } @@ -155,18 +155,15 @@ export function toFullMarket( ) : undefined - const { description } = contract + const parsedDescription = JSON.parse(contract.description) return { ...liteMarket, answers, comments, bets, - description, - textDescription: - typeof description === 'string' - ? description - : richTextToString(description), + description: parsedDescription, + textDescription: richTextToString(parsedDescription), } } diff --git a/web/pages/post/[...slugs]/index.tsx b/web/pages/post/[...slugs]/index.tsx index 6cd4408f..62089d1f 100644 --- a/web/pages/post/[...slugs]/index.tsx +++ b/web/pages/post/[...slugs]/index.tsx @@ -4,7 +4,7 @@ import { postPath, getPostBySlug, updatePost } from 'web/lib/firebase/posts' import { Post } from 'common/post' import { Title } from 'web/components/title' import { Spacer } from 'web/components/layout/spacer' -import { Content, TextEditor, useTextEditor } from 'web/components/editor' +import { RichContent, TextEditor, useTextEditor } from 'web/components/editor' import { getUser, User } from 'web/lib/firebase/users' import { PencilIcon, ShareIcon } from '@heroicons/react/solid' import clsx from 'clsx' @@ -110,7 +110,7 @@ export default function PostPage(props: { {user && user.id === post.creatorId ? ( ) : ( - + )} @@ -178,7 +178,7 @@ function RichEditPost(props: { post: Post }) { if (!editor) return await updatePost(post, { - content: editor.getJSON(), + content: JSON.stringify(editor.getJSON()), }) } @@ -219,7 +219,7 @@ function RichEditPost(props: { post: Post }) { - + diff --git a/web/posts/post-comments.tsx b/web/posts/post-comments.tsx index b98887bb..16b48be0 100644 --- a/web/posts/post-comments.tsx +++ b/web/posts/post-comments.tsx @@ -9,7 +9,7 @@ import { useRouter } from 'next/router' import { useEffect, useState } from 'react' import { Avatar } from 'web/components/avatar' import { CommentInput } from 'web/components/comment-input' -import { Content } from 'web/components/editor' +import { RichContent } from 'web/components/editor' import { CopyLinkDateTimeComponent } from 'web/components/feed/copy-link-date-time' import { Col } from 'web/components/layout/col' import { Row } from 'web/components/layout/row' @@ -108,7 +108,7 @@ export function PostComment(props: { onReplyClick?: (comment: PostComment) => void }) { const { post, comment, tips, indent, onReplyClick } = props - const { text, content, userUsername, userName, userAvatarUrl, createdTime } = + const { content, userUsername, userName, userAvatarUrl, createdTime } = comment const [highlighted, setHighlighted] = useState(false) @@ -150,9 +150,9 @@ export function PostComment(props: { elementId={comment.id} /> - From 99978118a5c35378db04862c8612d14d70449b25 Mon Sep 17 00:00:00 2001 From: Marshall Polaris Date: Thu, 22 Sep 2022 22:20:31 -0700 Subject: [PATCH 3/3] Write migration script to fix up old contract/post/comment text --- functions/src/scripts/migrate-rich-text.ts | 113 +++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 functions/src/scripts/migrate-rich-text.ts diff --git a/functions/src/scripts/migrate-rich-text.ts b/functions/src/scripts/migrate-rich-text.ts new file mode 100644 index 00000000..d1d9d22c --- /dev/null +++ b/functions/src/scripts/migrate-rich-text.ts @@ -0,0 +1,113 @@ +// We have three kinds of rich text: +// - Contract descriptions. +// - Comment text. +// - Post contents. +// These are stored in two different ways: +// - As plaintext strings. +// - As structured ProseMirror JSON in Firestore. +// We want to make all of these into: +// - Strings containing serialized ProseMirror JSON. + +import * as admin from 'firebase-admin' +import { FieldValue } from 'firebase-admin/firestore' +import { initAdmin } from './script-init' +import { log, writeAsync } from '../utils' +import { filterDefined } from '../../../common/util/array' +import { plainTextToProseMirror } from '../../../common/util/parse' + +initAdmin() +const firestore = admin.firestore() + +const isValidJSON = (s: string) => { + try { + JSON.parse(s) + return true + } catch { + return false + } +} + +const migrateContractDescriptions = async () => { + const contractQ = await firestore.collection('contracts').get() + console.log(`Loaded ${contractQ.size} contracts.`) + const updates = filterDefined( + contractQ.docs.map((doc) => { + const fields: { [k: string]: unknown } = {} + const oldDescription = doc.get('description') + if (typeof oldDescription === 'string') { + if (isValidJSON(oldDescription)) { + // this one is already good + return null + } else { + fields.description = JSON.stringify( + plainTextToProseMirror(oldDescription) + ) + } + } else if (oldDescription != null) { + // already JSON, just need to serialize into string + fields.description = JSON.stringify(oldDescription) + } else { + throw new Error('Content had null description for some reason.') + } + return { doc: doc.ref, fields } + }) + ) + log(`Found ${updates.length} contracts with old format descriptions.`) + await writeAsync(firestore, updates) +} + +const migrateCommentContents = async () => { + const commentQ = await firestore.collectionGroup('comments').get() + console.log(`Loaded ${commentQ.size} comments.`) + const updates = filterDefined( + commentQ.docs.map((doc) => { + const fields: { [k: string]: unknown } = {} + const oldText = doc.get('text') + const oldContent = doc.get('content') + if (typeof oldContent === 'string') { + // this one is already good + return null + } else if (oldContent != null) { + // already JSON, just need to serialize into string + fields.content = JSON.stringify(oldContent) + } else if (oldText != null) { + fields.text = FieldValue.delete() + fields.content = JSON.stringify(plainTextToProseMirror(oldText)) + } else { + throw new Error('Comment mysteriously had neither text nor content.') + } + return { doc: doc.ref, fields } + }) + ) + log(`Found ${updates.length} comments with old format content.`) + await writeAsync(firestore, updates) +} + +const migratePostContents = async () => { + const postQ = await firestore.collection('posts').get() + console.log(`Loaded ${postQ.size} posts.`) + const updates = filterDefined( + postQ.docs.map((doc) => { + const fields: { [k: string]: unknown } = {} + const oldContent = doc.get('content') + if (typeof oldContent === 'string') { + // this one is already good + return null + } else if (oldContent != null) { + // already JSON, just need to serialize into string + fields.content = JSON.stringify(oldContent) + } else { + throw new Error('Post had null content for some reason.') + } + return { doc: doc.ref, fields } + }) + ) + log(`Found ${updates.length} posts with old format content.`) + await writeAsync(firestore, updates) +} + +if (require.main === module) { + migrateContractDescriptions() + migrateCommentContents() + migratePostContents() +}