Fix backup DB job to actually backup most things, refactor (#605)

* Make backup manually invokable and thereby testable

* Add a shitload of missing stuff to our backups

* Also backup follows as per James

functions/src/backup-db.ts

@@ -18,46 +18,63 @@
 
 import * as functions from 'firebase-functions'
 import * as firestore from '@google-cloud/firestore'
-const client = new firestore.v1.FirestoreAdminClient()
+import { FirestoreAdminClient } from '@google-cloud/firestore/types/v1/firestore_admin_client'
 
-const bucket = 'gs://manifold-firestore-backup'
+export const backupDbCore = async (
+  client: FirestoreAdminClient,
+  project: string,
+  bucket: string
+) => {
+  const name = client.databasePath(project, '(default)')
+  const outputUriPrefix = `gs://${bucket}`
+  // Leave collectionIds empty to export all collections
+  // or set to a list of collection IDs to export,
+  // collectionIds: ['users', 'posts']
+  // NOTE: Subcollections are not backed up by default
+  const collectionIds = [
+    'contracts',
+    'groups',
+    'private-users',
+    'stripe-transactions',
+    'transactions',
+    'users',
+    'bets',
+    'comments',
+    'follows',
+    'followers',
+    'answers',
+    'txns',
+    'manalinks',
+    'liquidity',
+    'stats',
+    'cache',
+    'latency',
+    'views',
+    'notifications',
+    'portfolioHistory',
+    'folds',
+  ]
+  return await client.exportDocuments({ name, outputUriPrefix, collectionIds })
+}
 
 export const backupDb = functions.pubsub
   .schedule('every 24 hours')
-  .onRun((_context) => {
-    const projectId = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT
-    if (projectId == null) {
-      throw new Error('No project ID environment variable set.')
+  .onRun(async (_context) => {
+    try {
+      const client = new firestore.v1.FirestoreAdminClient()
+      const project = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT
+      if (project == null) {
+        throw new Error('No project ID environment variable set.')
+      }
+      const responses = await backupDbCore(
+        client,
+        project,
+        'manifold-firestore-backup'
+      )
+      const response = responses[0]
+      console.log(`Operation Name: ${response['name']}`)
+    } catch (err) {
+      console.error(err)
+      throw new Error('Export operation failed')
     }
-    const databaseName = client.databasePath(projectId, '(default)')
-
-    return client
-      .exportDocuments({
-        name: databaseName,
-        outputUriPrefix: bucket,
-        // Leave collectionIds empty to export all collections
-        // or set to a list of collection IDs to export,
-        // collectionIds: ['users', 'posts']
-        // NOTE: Subcollections are not backed up by default
-        collectionIds: [
-          'contracts',
-          'groups',
-          'private-users',
-          'stripe-transactions',
-          'users',
-          'bets',
-          'comments',
-          'followers',
-          'answers',
-          'txns',
-        ],
-      })
-      .then((responses) => {
-        const response = responses[0]
-        console.log(`Operation Name: ${response['name']}`)
-      })
-      .catch((err) => {
-        console.error(err)
-        throw new Error('Export operation failed')
-      })
   })

functions/src/scripts/backup-db.ts

@@ -0,0 +1,16 @@
+import * as firestore from '@google-cloud/firestore'
+import { getServiceAccountCredentials } from './script-init'
+import { backupDbCore } from '../backup-db'
+
+async function backupDb() {
+  const credentials = getServiceAccountCredentials()
+  const projectId = credentials.project_id
+  const client = new firestore.v1.FirestoreAdminClient({ credentials })
+  const bucket = 'manifold-firestore-backup'
+  const resp = await backupDbCore(client, projectId, bucket)
+  console.log(`Operation: ${resp[0]['name']}`)
+}
+
+if (require.main === module) {
+  backupDb().then(() => process.exit())
+}

functions/src/scripts/script-init.ts

@@ -47,26 +47,29 @@ const getFirebaseActiveProject = (cwd: string) => {
   }
 }
 
-export const initAdmin = (env?: string) => {
+export const getServiceAccountCredentials = (env?: string) => {
   env = env || getFirebaseActiveProject(process.cwd())
   if (env == null) {
-    console.error(
+    throw new Error(
       "Couldn't find active Firebase project; did you do `firebase use <alias>?`"
     )
-    return
   }
   const envVar = `GOOGLE_APPLICATION_CREDENTIALS_${env.toUpperCase()}`
   const keyPath = process.env[envVar]
   if (keyPath == null) {
-    console.error(
+    throw new Error(
       `Please set the ${envVar} environment variable to contain the path to your ${env} environment key file.`
     )
-    return
   }
-  console.log(`Initializing connection to ${env} Firebase...`)
   /* eslint-disable-next-line @typescript-eslint/no-var-requires */
-  const serviceAccount = require(keyPath)
-  admin.initializeApp({
+  return require(keyPath)
+}
+
+export const initAdmin = (env?: string) => {
+  const serviceAccount = getServiceAccountCredentials(env)
+  console.log(`Initializing connection to ${serviceAccount.project_id}...`)
+  return admin.initializeApp({
+    projectId: serviceAccount.project_id,
     credential: admin.credential.cert(serviceAccount),
   })
 }