Switch to Google Secret Manager for function secrets (#418)

* Upgrade firebase-functions 3.16.0 -> 3.21.2

* Use Secret Manager instead of config

* Small refactoring on new stripe/mailgun initialization

* Teach README about new secrets workflow

functions/README.md

@@ -22,25 +22,20 @@ Adapted from https://firebase.google.com/docs/functions/get-started
 
 ### For local development
 
-0. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev
+0. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI
-1. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI
+1. If you don't have java (or see the error `Error: Process java -version has exited with code 1. Please make sure Java is installed and on your system PATH.`): 0. `$ brew install java`
-2. If you don't have java (or see the error `Error: Process java -version has exited with code 1. Please make sure Java is installed and on your system PATH.`):
+   1. `$ sudo ln -sfn /opt/homebrew/opt/openjdk/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk.jdk`
-   1. `$ brew install java`
+2. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud
-   2. `$ sudo ln -sfn /opt/homebrew/opt/openjdk/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk.jdk` 
+3. `$ gcloud config set project <project-id>` to choose the project (`$ gcloud projects list` to see options)
-3. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud
+4. `$ mkdir firestore_export` to create a folder to store the exported database
-4. `$ gcloud config set project <project-id>` to choose the project (`$ gcloud projects list` to see options)
+5. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local 0. TODO: this won't work when open source, we'll have to point to the public db
-5. `$ mkdir firestore_export` to create a folder to store the exported database
-6. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local
-   1. TODO: this won't work when open source, we'll have to point to the public db
 
 ## Developing locally
 
 0. `$ firebase use dev` if you haven't already
-1. `$ yarn serve` to spin up the emulators
+1. `$ yarn serve` to spin up the emulators 0. The Emulator UI is at http://localhost:4000; the functions are hosted on :5001.
-   1. The Emulator UI is at http://localhost:4000; the functions are hosted on :5001.
+   Note: You have to kill and restart emulators when you change code; no hot reload =(
-      Note: You have to kill and restart emulators when you change code; no hot reload =(
+2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend 0. Note: emulated database is cleared after every shutdown
-2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend
-   1. Note: emulated database is cleared after every shutdown
 
 ## Firestore Commands
 
@@ -62,8 +57,7 @@ Adapted from https://firebase.google.com/docs/functions/get-started
 
 ## Secrets management
 
-Secrets are strings that shouldn't be checked into Git (eg API keys, passwords). We store these using [environment config on Firebase Functions](https://firebase.google.com/docs/functions/config-env). Some useful workflows:
+Secrets are strings that shouldn't be checked into Git (eg API keys, passwords). We store these using [Google Secret Manager](https://console.cloud.google.com/security/secret-manager), which provides them as environment variables to functions that require them. Some useful workflows:
 
-- Set a secret: `$ firebase functions:config:set stripe.test_secret="THE-API-KEY"`
+- Set a secret: `$ firebase functions:secrets:set stripe.test_secret="THE-API-KEY"`
-- Preview all secrets: `$ firebase functions:config:get`
+- Read a secret: `$ firebase functions:secrets:access STRIPE_APIKEY`
-- Cache for local dev:`$ firebase functions:config:get > .runtimeconfig.json`

functions/package.json

@@ -24,7 +24,7 @@
     "cors": "2.8.5",
     "fetch": "1.1.0",
     "firebase-admin": "10.0.0",
-    "firebase-functions": "3.16.0",
+    "firebase-functions": "3.21.2",
     "lodash": "4.17.21",
     "mailgun-js": "0.22.0",
     "module-alias": "2.2.2",

functions/src/create-answer.ts

@@ -8,113 +8,121 @@ import { Answer, MAX_ANSWER_LENGTH } from '../../common/answer'
 import { getContract, getValues } from './utils'
 import { sendNewAnswerEmail } from './emails'
 
-export const createAnswer = functions.runWith({ minInstances: 1 }).https.onCall(
+export const createAnswer = functions
-  async (
+  .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] })
-    data: {
+  .https.onCall(
-      contractId: string
+    async (
-      amount: number
+      data: {
-      text: string
+        contractId: string
-    },
+        amount: number
-    context
+        text: string
-  ) => {
+      },
-    const userId = context?.auth?.uid
+      context
-    if (!userId) return { status: 'error', message: 'Not authorized' }
+    ) => {
+      const userId = context?.auth?.uid
+      if (!userId) return { status: 'error', message: 'Not authorized' }
 
-    const { contractId, amount, text } = data
+      const { contractId, amount, text } = data
 
-    if (amount <= 0 || isNaN(amount) || !isFinite(amount))
+      if (amount <= 0 || isNaN(amount) || !isFinite(amount))
-      return { status: 'error', message: 'Invalid amount' }
+        return { status: 'error', message: 'Invalid amount' }
 
-    if (!text || typeof text !== 'string' || text.length > MAX_ANSWER_LENGTH)
+      if (!text || typeof text !== 'string' || text.length > MAX_ANSWER_LENGTH)
-      return { status: 'error', message: 'Invalid text' }
+        return { status: 'error', message: 'Invalid text' }
 
-    // Run as transaction to prevent race conditions.
+      // Run as transaction to prevent race conditions.
-    const result = await firestore.runTransaction(async (transaction) => {
+      const result = await firestore.runTransaction(async (transaction) => {
-      const userDoc = firestore.doc(`users/${userId}`)
+        const userDoc = firestore.doc(`users/${userId}`)
-      const userSnap = await transaction.get(userDoc)
+        const userSnap = await transaction.get(userDoc)
-      if (!userSnap.exists)
+        if (!userSnap.exists)
-        return { status: 'error', message: 'User not found' }
+          return { status: 'error', message: 'User not found' }
-      const user = userSnap.data() as User
+        const user = userSnap.data() as User
 
-      if (user.balance < amount)
+        if (user.balance < amount)
-        return { status: 'error', message: 'Insufficient balance' }
+          return { status: 'error', message: 'Insufficient balance' }
 
-      const contractDoc = firestore.doc(`contracts/${contractId}`)
+        const contractDoc = firestore.doc(`contracts/${contractId}`)
-      const contractSnap = await transaction.get(contractDoc)
+        const contractSnap = await transaction.get(contractDoc)
-      if (!contractSnap.exists)
+        if (!contractSnap.exists)
-        return { status: 'error', message: 'Invalid contract' }
+          return { status: 'error', message: 'Invalid contract' }
-      const contract = contractSnap.data() as Contract
+        const contract = contractSnap.data() as Contract
 
-      if (contract.outcomeType !== 'FREE_RESPONSE')
+        if (contract.outcomeType !== 'FREE_RESPONSE')
-        return {
+          return {
-          status: 'error',
+            status: 'error',
-          message: 'Requires a free response contract',
+            message: 'Requires a free response contract',
-        }
+          }
 
-      const { closeTime, volume } = contract
+        const { closeTime, volume } = contract
-      if (closeTime && Date.now() > closeTime)
+        if (closeTime && Date.now() > closeTime)
-        return { status: 'error', message: 'Trading is closed' }
+          return { status: 'error', message: 'Trading is closed' }
 
-      const [lastAnswer] = await getValues<Answer>(
+        const [lastAnswer] = await getValues<Answer>(
-        firestore
+          firestore
+            .collection(`contracts/${contractId}/answers`)
+            .orderBy('number', 'desc')
+            .limit(1)
+        )
+
+        if (!lastAnswer)
+          return { status: 'error', message: 'Could not fetch last answer' }
+
+        const number = lastAnswer.number + 1
+        const id = `${number}`
+
+        const newAnswerDoc = firestore
           .collection(`contracts/${contractId}/answers`)
-          .orderBy('number', 'desc')
+          .doc(id)
-          .limit(1)
-      )
 
-      if (!lastAnswer)
+        const answerId = newAnswerDoc.id
-        return { status: 'error', message: 'Could not fetch last answer' }
+        const { username, name, avatarUrl } = user
 
-      const number = lastAnswer.number + 1
+        const answer: Answer = {
-      const id = `${number}`
+          id,
+          number,
+          contractId,
+          createdTime: Date.now(),
+          userId: user.id,
+          username,
+          name,
+          avatarUrl,
+          text,
+        }
+        transaction.create(newAnswerDoc, answer)
 
-      const newAnswerDoc = firestore
+        const loanAmount = 0
-        .collection(`contracts/${contractId}/answers`)
-        .doc(id)
 
-      const answerId = newAnswerDoc.id
+        const { newBet, newPool, newTotalShares, newTotalBets } =
-      const { username, name, avatarUrl } = user
+          getNewMultiBetInfo(answerId, amount, contract, loanAmount)
 
-      const answer: Answer = {
+        const newBalance = user.balance - amount
-        id,
+        const betDoc = firestore
-        number,
+          .collection(`contracts/${contractId}/bets`)
-        contractId,
+          .doc()
-        createdTime: Date.now(),
+        transaction.create(betDoc, {
-        userId: user.id,
+          id: betDoc.id,
-        username,
+          userId: user.id,
-        name,
+          ...newBet,
-        avatarUrl,
+        })
-        text,
+        transaction.update(userDoc, { balance: newBalance })
-      }
+        transaction.update(contractDoc, {
-      transaction.create(newAnswerDoc, answer)
+          pool: newPool,
+          totalShares: newTotalShares,
+          totalBets: newTotalBets,
+          answers: [...(contract.answers ?? []), answer],
+          volume: volume + amount,
+        })
 
-      const loanAmount = 0
+        return { status: 'success', answerId, betId: betDoc.id, answer }
-
-      const { newBet, newPool, newTotalShares, newTotalBets } =
-        getNewMultiBetInfo(answerId, amount, contract, loanAmount)
-
-      const newBalance = user.balance - amount
-      const betDoc = firestore.collection(`contracts/${contractId}/bets`).doc()
-      transaction.create(betDoc, { id: betDoc.id, userId: user.id, ...newBet })
-      transaction.update(userDoc, { balance: newBalance })
-      transaction.update(contractDoc, {
-        pool: newPool,
-        totalShares: newTotalShares,
-        totalBets: newTotalBets,
-        answers: [...(contract.answers ?? []), answer],
-        volume: volume + amount,
       })
 
-      return { status: 'success', answerId, betId: betDoc.id, answer }
+      const { answer } = result
-    })
+      const contract = await getContract(contractId)
 
-    const { answer } = result
+      if (answer && contract) await sendNewAnswerEmail(answer, contract)
-    const contract = await getContract(contractId)
 
-    if (answer && contract) await sendNewAnswerEmail(answer, contract)
+      return result
-
+    }
-    return result
+  )
-  }
-)
 
 const firestore = admin.firestore()

functions/src/create-user.ts

@@ -17,7 +17,7 @@ import { sendWelcomeEmail } from './emails'
 import { isWhitelisted } from '../../common/envs/constants'
 
 export const createUser = functions
-  .runWith({ minInstances: 1 })
+  .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] })
   .https.onCall(async (data: { deviceToken?: string }, context) => {
     const userId = context?.auth?.uid
     if (!userId) return { status: 'error', message: 'Not authorized' }

functions/src/on-create-comment.ts

@@ -11,8 +11,9 @@ import { createNotification } from './create-notification'
 
 const firestore = admin.firestore()
 
-export const onCreateComment = functions.firestore
+export const onCreateComment = functions
-  .document('contracts/{contractId}/comments/{commentId}')
+  .runWith({ secrets: ['MAILGUN_KEY'] })
+  .firestore.document('contracts/{contractId}/comments/{commentId}')
   .onCreate(async (change, context) => {
     const { contractId } = context.params as {
       contractId: string

functions/src/resolve-market.ts

@@ -17,7 +17,7 @@ import { removeUndefinedProps } from '../../common/util/object'
 import { LiquidityProvision } from '../../common/liquidity-provision'
 
 export const resolveMarket = functions
-  .runWith({ minInstances: 1 })
+  .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] })
   .https.onCall(
     async (
       data: {

functions/src/send-email.ts

@@ -1,8 +1,9 @@
 import * as mailgun from 'mailgun-js'
-import * as functions from 'firebase-functions'
 
-const DOMAIN = 'mg.manifold.markets'
+const initMailgun = () => {
-const mg = mailgun({ apiKey: functions.config().mailgun.key, domain: DOMAIN })
+  const apiKey = process.env.MAILGUN_KEY as string
+  return mailgun({ apiKey, domain: 'mg.manifold.markets' })
+}
 
 export const sendTextEmail = (to: string, subject: string, text: string) => {
   const data: mailgun.messages.SendData = {
@@ -13,7 +14,7 @@ export const sendTextEmail = (to: string, subject: string, text: string) => {
     // Don't rewrite urls in plaintext emails
     'o:tracking-clicks': 'htmlonly',
   }
-
+  const mg = initMailgun()
   return mg.messages().send(data, (error) => {
     if (error) console.log('Error sending email', error)
     else console.log('Sent text email', to, subject)
@@ -34,6 +35,7 @@ export const sendTemplateEmail = (
     template: templateId,
     'h:X-Mailgun-Variables': JSON.stringify(templateData),
   }
+  const mg = initMailgun()
   return mg.messages().send(data, (error) => {
     if (error) console.log('Error sending email', error)
     else console.log('Sent template email', templateId, to, subject)

functions/src/stripe.ts

@@ -21,10 +21,10 @@ export type StripeTransaction = {
   timestamp: number
 }
 
-const stripe = new Stripe(functions.config().stripe.apikey, {
+const initStripe = () => {
-  apiVersion: '2020-08-27',
+  const apiKey = process.env.STRIPE_APIKEY as string
-  typescript: true,
+  return new Stripe(apiKey, { apiVersion: '2020-08-27', typescript: true })
-})
+}
 
 // manage at https://dashboard.stripe.com/test/products?active=true
 const manticDollarStripePrice = isProd
@@ -42,7 +42,7 @@ const manticDollarStripePrice = isProd
     }
 
 export const createCheckoutSession = functions
-  .runWith({ minInstances: 1 })
+  .runWith({ minInstances: 1, secrets: ['STRIPE_APIKEY'] })
   .https.onRequest(async (req, res) => {
     const userId = req.query.userId?.toString()
 
@@ -64,6 +64,7 @@ export const createCheckoutSession = functions
     const referrer =
       req.query.referer || req.headers.referer || 'https://manifold.markets'
 
+    const stripe = initStripe()
     const session = await stripe.checkout.sessions.create({
       metadata: {
         userId,
@@ -87,15 +88,19 @@ export const createCheckoutSession = functions
   })
 
 export const stripeWebhook = functions
-  .runWith({ minInstances: 1 })
+  .runWith({
+    minInstances: 1,
+    secrets: ['STRIPE_APIKEY', 'STRIPE_WEBHOOKSECRET'],
+  })
   .https.onRequest(async (req, res) => {
+    const stripe = initStripe()
     let event
 
     try {
       event = stripe.webhooks.constructEvent(
         req.rawBody,
         req.headers['stripe-signature'] as string,
-        functions.config().stripe.webhooksecret
+        process.env.STRIPE_WEBHOOKSECRET as string
       )
     } catch (e: any) {
       console.log(`Webhook Error: ${e.message}`)

yarn.lock

@@ -5868,16 +5868,17 @@ firebase-functions-test@0.3.3:
     "@types/lodash" "^4.14.104"
     lodash "^4.17.5"
 
-firebase-functions@3.16.0:
+firebase-functions@3.21.2:
-  version "3.16.0"
+  version "3.21.2"
-  resolved "https://registry.yarnpkg.com/firebase-functions/-/firebase-functions-3.16.0.tgz#603e47c2a563a5d0d1bc28f7362d0349c2f0d33f"
+  resolved "https://registry.yarnpkg.com/firebase-functions/-/firebase-functions-3.21.2.tgz#53dde54cdcb2a645b4cdc3e3cbb7480dd46c9293"
-  integrity sha512-6ISOn0JckMtpA3aJ/+wCCGhThUhBUrpZD+tSkUeolx0Vr+NoYFXA0+2YzJZa/A2MDU8gotPzUtnauLSEQvfClQ==
+  integrity sha512-XZOSv7mLnd8uIzNA+rc+n+oM/g2Nn4rtUkOKeTMccYiWOMdMMUwhzuqRnE28mB65bveU12aTHkaJY6p3Pk6MUw==
   dependencies:
     "@types/cors" "^2.8.5"
     "@types/express" "4.17.3"
     cors "^2.8.5"
     express "^4.17.1"
     lodash "^4.17.14"
+    node-fetch "^2.6.7"
 
 firebase@9.6.0:
   version "9.6.0"