From 33ddd86addbd51fbbeb3e15ccc3185201be1904e Mon Sep 17 00:00:00 2001 From: Marshall Polaris Date: Sat, 4 Jun 2022 14:39:25 -0700 Subject: [PATCH] Switch to Google Secret Manager for function secrets (#418) * Upgrade firebase-functions 3.16.0 -> 3.21.2 * Use Secret Manager instead of config * Small refactoring on new stripe/mailgun initialization * Teach README about new secrets workflow --- functions/README.md | 32 ++--- functions/package.json | 2 +- functions/src/create-answer.ts | 184 +++++++++++++++-------------- functions/src/create-user.ts | 2 +- functions/src/on-create-comment.ts | 5 +- functions/src/resolve-market.ts | 2 +- functions/src/send-email.ts | 10 +- functions/src/stripe.ts | 19 +-- yarn.lock | 9 +- 9 files changed, 138 insertions(+), 127 deletions(-) diff --git a/functions/README.md b/functions/README.md index e8debc6e..7fd312c3 100644 --- a/functions/README.md +++ b/functions/README.md @@ -22,25 +22,20 @@ Adapted from https://firebase.google.com/docs/functions/get-started ### For local development -0. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev -1. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI -2. If you don't have java (or see the error `Error: Process java -version has exited with code 1. Please make sure Java is installed and on your system PATH.`): - 1. `$ brew install java` - 2. `$ sudo ln -sfn /opt/homebrew/opt/openjdk/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk.jdk` -3. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud -4. `$ gcloud config set project ` to choose the project (`$ gcloud projects list` to see options) -5. `$ mkdir firestore_export` to create a folder to store the exported database -6. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local - 1. TODO: this won't work when open source, we'll have to point to the public db +0. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI +1. If you don't have java (or see the error `Error: Process java -version has exited with code 1. Please make sure Java is installed and on your system PATH.`): 0. `$ brew install java` + 1. `$ sudo ln -sfn /opt/homebrew/opt/openjdk/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk.jdk` +2. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud +3. `$ gcloud config set project ` to choose the project (`$ gcloud projects list` to see options) +4. `$ mkdir firestore_export` to create a folder to store the exported database +5. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local 0. TODO: this won't work when open source, we'll have to point to the public db ## Developing locally 0. `$ firebase use dev` if you haven't already -1. `$ yarn serve` to spin up the emulators - 1. The Emulator UI is at http://localhost:4000; the functions are hosted on :5001. - Note: You have to kill and restart emulators when you change code; no hot reload =( -2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend - 1. Note: emulated database is cleared after every shutdown +1. `$ yarn serve` to spin up the emulators 0. The Emulator UI is at http://localhost:4000; the functions are hosted on :5001. + Note: You have to kill and restart emulators when you change code; no hot reload =( +2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend 0. Note: emulated database is cleared after every shutdown ## Firestore Commands @@ -62,8 +57,7 @@ Adapted from https://firebase.google.com/docs/functions/get-started ## Secrets management -Secrets are strings that shouldn't be checked into Git (eg API keys, passwords). We store these using [environment config on Firebase Functions](https://firebase.google.com/docs/functions/config-env). Some useful workflows: +Secrets are strings that shouldn't be checked into Git (eg API keys, passwords). We store these using [Google Secret Manager](https://console.cloud.google.com/security/secret-manager), which provides them as environment variables to functions that require them. Some useful workflows: -- Set a secret: `$ firebase functions:config:set stripe.test_secret="THE-API-KEY"` -- Preview all secrets: `$ firebase functions:config:get` -- Cache for local dev:`$ firebase functions:config:get > .runtimeconfig.json` +- Set a secret: `$ firebase functions:secrets:set stripe.test_secret="THE-API-KEY"` +- Read a secret: `$ firebase functions:secrets:access STRIPE_APIKEY` diff --git a/functions/package.json b/functions/package.json index 0b012401..3f3315f3 100644 --- a/functions/package.json +++ b/functions/package.json @@ -24,7 +24,7 @@ "cors": "2.8.5", "fetch": "1.1.0", "firebase-admin": "10.0.0", - "firebase-functions": "3.16.0", + "firebase-functions": "3.21.2", "lodash": "4.17.21", "mailgun-js": "0.22.0", "module-alias": "2.2.2", diff --git a/functions/src/create-answer.ts b/functions/src/create-answer.ts index 7becfc7f..cf3867b0 100644 --- a/functions/src/create-answer.ts +++ b/functions/src/create-answer.ts @@ -8,113 +8,121 @@ import { Answer, MAX_ANSWER_LENGTH } from '../../common/answer' import { getContract, getValues } from './utils' import { sendNewAnswerEmail } from './emails' -export const createAnswer = functions.runWith({ minInstances: 1 }).https.onCall( - async ( - data: { - contractId: string - amount: number - text: string - }, - context - ) => { - const userId = context?.auth?.uid - if (!userId) return { status: 'error', message: 'Not authorized' } +export const createAnswer = functions + .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] }) + .https.onCall( + async ( + data: { + contractId: string + amount: number + text: string + }, + context + ) => { + const userId = context?.auth?.uid + if (!userId) return { status: 'error', message: 'Not authorized' } - const { contractId, amount, text } = data + const { contractId, amount, text } = data - if (amount <= 0 || isNaN(amount) || !isFinite(amount)) - return { status: 'error', message: 'Invalid amount' } + if (amount <= 0 || isNaN(amount) || !isFinite(amount)) + return { status: 'error', message: 'Invalid amount' } - if (!text || typeof text !== 'string' || text.length > MAX_ANSWER_LENGTH) - return { status: 'error', message: 'Invalid text' } + if (!text || typeof text !== 'string' || text.length > MAX_ANSWER_LENGTH) + return { status: 'error', message: 'Invalid text' } - // Run as transaction to prevent race conditions. - const result = await firestore.runTransaction(async (transaction) => { - const userDoc = firestore.doc(`users/${userId}`) - const userSnap = await transaction.get(userDoc) - if (!userSnap.exists) - return { status: 'error', message: 'User not found' } - const user = userSnap.data() as User + // Run as transaction to prevent race conditions. + const result = await firestore.runTransaction(async (transaction) => { + const userDoc = firestore.doc(`users/${userId}`) + const userSnap = await transaction.get(userDoc) + if (!userSnap.exists) + return { status: 'error', message: 'User not found' } + const user = userSnap.data() as User - if (user.balance < amount) - return { status: 'error', message: 'Insufficient balance' } + if (user.balance < amount) + return { status: 'error', message: 'Insufficient balance' } - const contractDoc = firestore.doc(`contracts/${contractId}`) - const contractSnap = await transaction.get(contractDoc) - if (!contractSnap.exists) - return { status: 'error', message: 'Invalid contract' } - const contract = contractSnap.data() as Contract + const contractDoc = firestore.doc(`contracts/${contractId}`) + const contractSnap = await transaction.get(contractDoc) + if (!contractSnap.exists) + return { status: 'error', message: 'Invalid contract' } + const contract = contractSnap.data() as Contract - if (contract.outcomeType !== 'FREE_RESPONSE') - return { - status: 'error', - message: 'Requires a free response contract', - } + if (contract.outcomeType !== 'FREE_RESPONSE') + return { + status: 'error', + message: 'Requires a free response contract', + } - const { closeTime, volume } = contract - if (closeTime && Date.now() > closeTime) - return { status: 'error', message: 'Trading is closed' } + const { closeTime, volume } = contract + if (closeTime && Date.now() > closeTime) + return { status: 'error', message: 'Trading is closed' } - const [lastAnswer] = await getValues( - firestore + const [lastAnswer] = await getValues( + firestore + .collection(`contracts/${contractId}/answers`) + .orderBy('number', 'desc') + .limit(1) + ) + + if (!lastAnswer) + return { status: 'error', message: 'Could not fetch last answer' } + + const number = lastAnswer.number + 1 + const id = `${number}` + + const newAnswerDoc = firestore .collection(`contracts/${contractId}/answers`) - .orderBy('number', 'desc') - .limit(1) - ) + .doc(id) - if (!lastAnswer) - return { status: 'error', message: 'Could not fetch last answer' } + const answerId = newAnswerDoc.id + const { username, name, avatarUrl } = user - const number = lastAnswer.number + 1 - const id = `${number}` + const answer: Answer = { + id, + number, + contractId, + createdTime: Date.now(), + userId: user.id, + username, + name, + avatarUrl, + text, + } + transaction.create(newAnswerDoc, answer) - const newAnswerDoc = firestore - .collection(`contracts/${contractId}/answers`) - .doc(id) + const loanAmount = 0 - const answerId = newAnswerDoc.id - const { username, name, avatarUrl } = user + const { newBet, newPool, newTotalShares, newTotalBets } = + getNewMultiBetInfo(answerId, amount, contract, loanAmount) - const answer: Answer = { - id, - number, - contractId, - createdTime: Date.now(), - userId: user.id, - username, - name, - avatarUrl, - text, - } - transaction.create(newAnswerDoc, answer) + const newBalance = user.balance - amount + const betDoc = firestore + .collection(`contracts/${contractId}/bets`) + .doc() + transaction.create(betDoc, { + id: betDoc.id, + userId: user.id, + ...newBet, + }) + transaction.update(userDoc, { balance: newBalance }) + transaction.update(contractDoc, { + pool: newPool, + totalShares: newTotalShares, + totalBets: newTotalBets, + answers: [...(contract.answers ?? []), answer], + volume: volume + amount, + }) - const loanAmount = 0 - - const { newBet, newPool, newTotalShares, newTotalBets } = - getNewMultiBetInfo(answerId, amount, contract, loanAmount) - - const newBalance = user.balance - amount - const betDoc = firestore.collection(`contracts/${contractId}/bets`).doc() - transaction.create(betDoc, { id: betDoc.id, userId: user.id, ...newBet }) - transaction.update(userDoc, { balance: newBalance }) - transaction.update(contractDoc, { - pool: newPool, - totalShares: newTotalShares, - totalBets: newTotalBets, - answers: [...(contract.answers ?? []), answer], - volume: volume + amount, + return { status: 'success', answerId, betId: betDoc.id, answer } }) - return { status: 'success', answerId, betId: betDoc.id, answer } - }) + const { answer } = result + const contract = await getContract(contractId) - const { answer } = result - const contract = await getContract(contractId) + if (answer && contract) await sendNewAnswerEmail(answer, contract) - if (answer && contract) await sendNewAnswerEmail(answer, contract) - - return result - } -) + return result + } + ) const firestore = admin.firestore() diff --git a/functions/src/create-user.ts b/functions/src/create-user.ts index f73b868b..51e73c36 100644 --- a/functions/src/create-user.ts +++ b/functions/src/create-user.ts @@ -17,7 +17,7 @@ import { sendWelcomeEmail } from './emails' import { isWhitelisted } from '../../common/envs/constants' export const createUser = functions - .runWith({ minInstances: 1 }) + .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] }) .https.onCall(async (data: { deviceToken?: string }, context) => { const userId = context?.auth?.uid if (!userId) return { status: 'error', message: 'Not authorized' } diff --git a/functions/src/on-create-comment.ts b/functions/src/on-create-comment.ts index 43676615..bf1dacd5 100644 --- a/functions/src/on-create-comment.ts +++ b/functions/src/on-create-comment.ts @@ -11,8 +11,9 @@ import { createNotification } from './create-notification' const firestore = admin.firestore() -export const onCreateComment = functions.firestore - .document('contracts/{contractId}/comments/{commentId}') +export const onCreateComment = functions + .runWith({ secrets: ['MAILGUN_KEY'] }) + .firestore.document('contracts/{contractId}/comments/{commentId}') .onCreate(async (change, context) => { const { contractId } = context.params as { contractId: string diff --git a/functions/src/resolve-market.ts b/functions/src/resolve-market.ts index cf8c018f..894f1492 100644 --- a/functions/src/resolve-market.ts +++ b/functions/src/resolve-market.ts @@ -17,7 +17,7 @@ import { removeUndefinedProps } from '../../common/util/object' import { LiquidityProvision } from '../../common/liquidity-provision' export const resolveMarket = functions - .runWith({ minInstances: 1 }) + .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] }) .https.onCall( async ( data: { diff --git a/functions/src/send-email.ts b/functions/src/send-email.ts index b0ac58a6..f97234f6 100644 --- a/functions/src/send-email.ts +++ b/functions/src/send-email.ts @@ -1,8 +1,9 @@ import * as mailgun from 'mailgun-js' -import * as functions from 'firebase-functions' -const DOMAIN = 'mg.manifold.markets' -const mg = mailgun({ apiKey: functions.config().mailgun.key, domain: DOMAIN }) +const initMailgun = () => { + const apiKey = process.env.MAILGUN_KEY as string + return mailgun({ apiKey, domain: 'mg.manifold.markets' }) +} export const sendTextEmail = (to: string, subject: string, text: string) => { const data: mailgun.messages.SendData = { @@ -13,7 +14,7 @@ export const sendTextEmail = (to: string, subject: string, text: string) => { // Don't rewrite urls in plaintext emails 'o:tracking-clicks': 'htmlonly', } - + const mg = initMailgun() return mg.messages().send(data, (error) => { if (error) console.log('Error sending email', error) else console.log('Sent text email', to, subject) @@ -34,6 +35,7 @@ export const sendTemplateEmail = ( template: templateId, 'h:X-Mailgun-Variables': JSON.stringify(templateData), } + const mg = initMailgun() return mg.messages().send(data, (error) => { if (error) console.log('Error sending email', error) else console.log('Sent template email', templateId, to, subject) diff --git a/functions/src/stripe.ts b/functions/src/stripe.ts index ac3e2a19..a5d1482f 100644 --- a/functions/src/stripe.ts +++ b/functions/src/stripe.ts @@ -21,10 +21,10 @@ export type StripeTransaction = { timestamp: number } -const stripe = new Stripe(functions.config().stripe.apikey, { - apiVersion: '2020-08-27', - typescript: true, -}) +const initStripe = () => { + const apiKey = process.env.STRIPE_APIKEY as string + return new Stripe(apiKey, { apiVersion: '2020-08-27', typescript: true }) +} // manage at https://dashboard.stripe.com/test/products?active=true const manticDollarStripePrice = isProd @@ -42,7 +42,7 @@ const manticDollarStripePrice = isProd } export const createCheckoutSession = functions - .runWith({ minInstances: 1 }) + .runWith({ minInstances: 1, secrets: ['STRIPE_APIKEY'] }) .https.onRequest(async (req, res) => { const userId = req.query.userId?.toString() @@ -64,6 +64,7 @@ export const createCheckoutSession = functions const referrer = req.query.referer || req.headers.referer || 'https://manifold.markets' + const stripe = initStripe() const session = await stripe.checkout.sessions.create({ metadata: { userId, @@ -87,15 +88,19 @@ export const createCheckoutSession = functions }) export const stripeWebhook = functions - .runWith({ minInstances: 1 }) + .runWith({ + minInstances: 1, + secrets: ['STRIPE_APIKEY', 'STRIPE_WEBHOOKSECRET'], + }) .https.onRequest(async (req, res) => { + const stripe = initStripe() let event try { event = stripe.webhooks.constructEvent( req.rawBody, req.headers['stripe-signature'] as string, - functions.config().stripe.webhooksecret + process.env.STRIPE_WEBHOOKSECRET as string ) } catch (e: any) { console.log(`Webhook Error: ${e.message}`) diff --git a/yarn.lock b/yarn.lock index 20829cec..17e11464 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5868,16 +5868,17 @@ firebase-functions-test@0.3.3: "@types/lodash" "^4.14.104" lodash "^4.17.5" -firebase-functions@3.16.0: - version "3.16.0" - resolved "https://registry.yarnpkg.com/firebase-functions/-/firebase-functions-3.16.0.tgz#603e47c2a563a5d0d1bc28f7362d0349c2f0d33f" - integrity sha512-6ISOn0JckMtpA3aJ/+wCCGhThUhBUrpZD+tSkUeolx0Vr+NoYFXA0+2YzJZa/A2MDU8gotPzUtnauLSEQvfClQ== +firebase-functions@3.21.2: + version "3.21.2" + resolved "https://registry.yarnpkg.com/firebase-functions/-/firebase-functions-3.21.2.tgz#53dde54cdcb2a645b4cdc3e3cbb7480dd46c9293" + integrity sha512-XZOSv7mLnd8uIzNA+rc+n+oM/g2Nn4rtUkOKeTMccYiWOMdMMUwhzuqRnE28mB65bveU12aTHkaJY6p3Pk6MUw== dependencies: "@types/cors" "^2.8.5" "@types/express" "4.17.3" cors "^2.8.5" express "^4.17.1" lodash "^4.17.14" + node-fetch "^2.6.7" firebase@9.6.0: version "9.6.0"