Switch to Google Secret Manager for function secrets (#418)

* Upgrade firebase-functions 3.16.0 -> 3.21.2

* Use Secret Manager instead of config

* Small refactoring on new stripe/mailgun initialization

* Teach README about new secrets workflow

functions/README.md

@@ -22,25 +22,20 @@ Adapted from https://firebase.google.com/docs/functions/get-started
 
 ### For local development
 
-0. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev
+0. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI
-1. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI
+1. If you don't have java (or see the error `Error: Process java -version has exited with code 1. Please make sure Java is installed and on your system PATH.`): 0. `$ brew install java`
-2. If you don't have java (or see the error `Error: Process java -version has exited with code 1. Please make sure Java is installed and on your system PATH.`):
+   1. `$ sudo ln -sfn /opt/homebrew/opt/openjdk/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk.jdk`
-   1. `$ brew install java`
+2. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud
-   2. `$ sudo ln -sfn /opt/homebrew/opt/openjdk/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk.jdk` 
+3. `$ gcloud config set project <project-id>` to choose the project (`$ gcloud projects list` to see options)
-3. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud
+4. `$ mkdir firestore_export` to create a folder to store the exported database
-4. `$ gcloud config set project <project-id>` to choose the project (`$ gcloud projects list` to see options)
+5. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local 0. TODO: this won't work when open source, we'll have to point to the public db
-5. `$ mkdir firestore_export` to create a folder to store the exported database
-6. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local
-   1. TODO: this won't work when open source, we'll have to point to the public db
 
 ## Developing locally
 
 0. `$ firebase use dev` if you haven't already
-1. `$ yarn serve` to spin up the emulators
+1. `$ yarn serve` to spin up the emulators 0. The Emulator UI is at http://localhost:4000; the functions are hosted on :5001.
-   1. The Emulator UI is at http://localhost:4000; the functions are hosted on :5001.
    Note: You have to kill and restart emulators when you change code; no hot reload =(
-2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend
+2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend 0. Note: emulated database is cleared after every shutdown
-   1. Note: emulated database is cleared after every shutdown
 
 ## Firestore Commands
 
@@ -62,8 +57,7 @@ Adapted from https://firebase.google.com/docs/functions/get-started
 
 ## Secrets management
 
-Secrets are strings that shouldn't be checked into Git (eg API keys, passwords). We store these using [environment config on Firebase Functions](https://firebase.google.com/docs/functions/config-env). Some useful workflows:
+Secrets are strings that shouldn't be checked into Git (eg API keys, passwords). We store these using [Google Secret Manager](https://console.cloud.google.com/security/secret-manager), which provides them as environment variables to functions that require them. Some useful workflows:
 
-- Set a secret: `$ firebase functions:config:set stripe.test_secret="THE-API-KEY"`
+- Set a secret: `$ firebase functions:secrets:set stripe.test_secret="THE-API-KEY"`
-- Preview all secrets: `$ firebase functions:config:get`
+- Read a secret: `$ firebase functions:secrets:access STRIPE_APIKEY`
-- Cache for local dev:`$ firebase functions:config:get > .runtimeconfig.json`

functions/package.json

@@ -24,7 +24,7 @@
     "cors": "2.8.5",
     "fetch": "1.1.0",
     "firebase-admin": "10.0.0",
-    "firebase-functions": "3.16.0",
+    "firebase-functions": "3.21.2",
     "lodash": "4.17.21",
     "mailgun-js": "0.22.0",
     "module-alias": "2.2.2",

functions/src/create-answer.ts

@@ -8,7 +8,9 @@ import { Answer, MAX_ANSWER_LENGTH } from '../../common/answer'
 import { getContract, getValues } from './utils'
 import { sendNewAnswerEmail } from './emails'
 
-export const createAnswer = functions.runWith({ minInstances: 1 }).https.onCall(
+export const createAnswer = functions
+  .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] })
+  .https.onCall(
     async (
       data: {
         contractId: string
@@ -94,8 +96,14 @@ export const createAnswer = functions.runWith({ minInstances: 1 }).https.onCall(
           getNewMultiBetInfo(answerId, amount, contract, loanAmount)
 
         const newBalance = user.balance - amount
-      const betDoc = firestore.collection(`contracts/${contractId}/bets`).doc()
+        const betDoc = firestore
-      transaction.create(betDoc, { id: betDoc.id, userId: user.id, ...newBet })
+          .collection(`contracts/${contractId}/bets`)
+          .doc()
+        transaction.create(betDoc, {
+          id: betDoc.id,
+          userId: user.id,
+          ...newBet,
+        })
         transaction.update(userDoc, { balance: newBalance })
         transaction.update(contractDoc, {
           pool: newPool,
@@ -115,6 +123,6 @@ export const createAnswer = functions.runWith({ minInstances: 1 }).https.onCall(
 
       return result
     }
-)
+  )
 
 const firestore = admin.firestore()

functions/src/create-user.ts

@@ -17,7 +17,7 @@ import { sendWelcomeEmail } from './emails'
 import { isWhitelisted } from '../../common/envs/constants'
 
 export const createUser = functions
-  .runWith({ minInstances: 1 })
+  .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] })
   .https.onCall(async (data: { deviceToken?: string }, context) => {
     const userId = context?.auth?.uid
     if (!userId) return { status: 'error', message: 'Not authorized' }

functions/src/on-create-comment.ts

@@ -11,8 +11,9 @@ import { createNotification } from './create-notification'
 
 const firestore = admin.firestore()
 
-export const onCreateComment = functions.firestore
+export const onCreateComment = functions
-  .document('contracts/{contractId}/comments/{commentId}')
+  .runWith({ secrets: ['MAILGUN_KEY'] })
+  .firestore.document('contracts/{contractId}/comments/{commentId}')
   .onCreate(async (change, context) => {
     const { contractId } = context.params as {
       contractId: string

functions/src/resolve-market.ts

@@ -17,7 +17,7 @@ import { removeUndefinedProps } from '../../common/util/object'
 import { LiquidityProvision } from '../../common/liquidity-provision'
 
 export const resolveMarket = functions
-  .runWith({ minInstances: 1 })
+  .runWith({ minInstances: 1, secrets: ['MAILGUN_KEY'] })
   .https.onCall(
     async (
       data: {

functions/src/send-email.ts

@@ -1,8 +1,9 @@
 import * as mailgun from 'mailgun-js'
-import * as functions from 'firebase-functions'
 
-const DOMAIN = 'mg.manifold.markets'
+const initMailgun = () => {
-const mg = mailgun({ apiKey: functions.config().mailgun.key, domain: DOMAIN })
+  const apiKey = process.env.MAILGUN_KEY as string
+  return mailgun({ apiKey, domain: 'mg.manifold.markets' })
+}
 
 export const sendTextEmail = (to: string, subject: string, text: string) => {
   const data: mailgun.messages.SendData = {
@@ -13,7 +14,7 @@ export const sendTextEmail = (to: string, subject: string, text: string) => {
     // Don't rewrite urls in plaintext emails
     'o:tracking-clicks': 'htmlonly',
   }
-
+  const mg = initMailgun()
   return mg.messages().send(data, (error) => {
     if (error) console.log('Error sending email', error)
     else console.log('Sent text email', to, subject)
@@ -34,6 +35,7 @@ export const sendTemplateEmail = (
     template: templateId,
     'h:X-Mailgun-Variables': JSON.stringify(templateData),
   }
+  const mg = initMailgun()
   return mg.messages().send(data, (error) => {
     if (error) console.log('Error sending email', error)
     else console.log('Sent template email', templateId, to, subject)

functions/src/stripe.ts

@@ -21,10 +21,10 @@ export type StripeTransaction = {
   timestamp: number
 }
 
-const stripe = new Stripe(functions.config().stripe.apikey, {
+const initStripe = () => {
-  apiVersion: '2020-08-27',
+  const apiKey = process.env.STRIPE_APIKEY as string
-  typescript: true,
+  return new Stripe(apiKey, { apiVersion: '2020-08-27', typescript: true })
-})
+}
 
 // manage at https://dashboard.stripe.com/test/products?active=true
 const manticDollarStripePrice = isProd
@@ -42,7 +42,7 @@ const manticDollarStripePrice = isProd
     }
 
 export const createCheckoutSession = functions
-  .runWith({ minInstances: 1 })
+  .runWith({ minInstances: 1, secrets: ['STRIPE_APIKEY'] })
   .https.onRequest(async (req, res) => {
     const userId = req.query.userId?.toString()
 
@@ -64,6 +64,7 @@ export const createCheckoutSession = functions
     const referrer =
       req.query.referer || req.headers.referer || 'https://manifold.markets'
 
+    const stripe = initStripe()
     const session = await stripe.checkout.sessions.create({
       metadata: {
         userId,
@@ -87,15 +88,19 @@ export const createCheckoutSession = functions
   })
 
 export const stripeWebhook = functions
-  .runWith({ minInstances: 1 })
+  .runWith({
+    minInstances: 1,
+    secrets: ['STRIPE_APIKEY', 'STRIPE_WEBHOOKSECRET'],
+  })
   .https.onRequest(async (req, res) => {
+    const stripe = initStripe()
     let event
 
     try {
       event = stripe.webhooks.constructEvent(
         req.rawBody,
         req.headers['stripe-signature'] as string,
-        functions.config().stripe.webhooksecret
+        process.env.STRIPE_WEBHOOKSECRET as string
       )
     } catch (e: any) {
       console.log(`Webhook Error: ${e.message}`)

yarn.lock

@@ -5868,16 +5868,17 @@ firebase-functions-test@0.3.3:
     "@types/lodash" "^4.14.104"
     lodash "^4.17.5"
 
-firebase-functions@3.16.0:
+firebase-functions@3.21.2:
-  version "3.16.0"
+  version "3.21.2"
-  resolved "https://registry.yarnpkg.com/firebase-functions/-/firebase-functions-3.16.0.tgz#603e47c2a563a5d0d1bc28f7362d0349c2f0d33f"
+  resolved "https://registry.yarnpkg.com/firebase-functions/-/firebase-functions-3.21.2.tgz#53dde54cdcb2a645b4cdc3e3cbb7480dd46c9293"
-  integrity sha512-6ISOn0JckMtpA3aJ/+wCCGhThUhBUrpZD+tSkUeolx0Vr+NoYFXA0+2YzJZa/A2MDU8gotPzUtnauLSEQvfClQ==
+  integrity sha512-XZOSv7mLnd8uIzNA+rc+n+oM/g2Nn4rtUkOKeTMccYiWOMdMMUwhzuqRnE28mB65bveU12aTHkaJY6p3Pk6MUw==
   dependencies:
     "@types/cors" "^2.8.5"
     "@types/express" "4.17.3"
     cors "^2.8.5"
     express "^4.17.1"
     lodash "^4.17.14"
+    node-fetch "^2.6.7"
 
 firebase@9.6.0:
   version "9.6.0"