Fix backup DB job to actually backup most things, refactor (#605)

* Make backup manually invokable and thereby testable

* Add a shitload of missing stuff to our backups

* Also backup follows as per James

functions/src/backup-db.ts

@@ -18,46 +18,63 @@
 
 import * as functions from 'firebase-functions'
 import * as firestore from '@google-cloud/firestore'
-const client = new firestore.v1.FirestoreAdminClient()
+import { FirestoreAdminClient } from '@google-cloud/firestore/types/v1/firestore_admin_client'
 
-const bucket = 'gs://manifold-firestore-backup'
+export const backupDbCore = async (
-
+  client: FirestoreAdminClient,
-export const backupDb = functions.pubsub
+  project: string,
-  .schedule('every 24 hours')
+  bucket: string
-  .onRun((_context) => {
+) => {
-    const projectId = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT
+  const name = client.databasePath(project, '(default)')
-    if (projectId == null) {
+  const outputUriPrefix = `gs://${bucket}`
-      throw new Error('No project ID environment variable set.')
-    }
-    const databaseName = client.databasePath(projectId, '(default)')
-
-    return client
-      .exportDocuments({
-        name: databaseName,
-        outputUriPrefix: bucket,
   // Leave collectionIds empty to export all collections
   // or set to a list of collection IDs to export,
   // collectionIds: ['users', 'posts']
   // NOTE: Subcollections are not backed up by default
-        collectionIds: [
+  const collectionIds = [
     'contracts',
     'groups',
     'private-users',
     'stripe-transactions',
+    'transactions',
     'users',
     'bets',
     'comments',
+    'follows',
     'followers',
     'answers',
     'txns',
-        ],
+    'manalinks',
-      })
+    'liquidity',
-      .then((responses) => {
+    'stats',
+    'cache',
+    'latency',
+    'views',
+    'notifications',
+    'portfolioHistory',
+    'folds',
+  ]
+  return await client.exportDocuments({ name, outputUriPrefix, collectionIds })
+}
+
+export const backupDb = functions.pubsub
+  .schedule('every 24 hours')
+  .onRun(async (_context) => {
+    try {
+      const client = new firestore.v1.FirestoreAdminClient()
+      const project = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT
+      if (project == null) {
+        throw new Error('No project ID environment variable set.')
+      }
+      const responses = await backupDbCore(
+        client,
+        project,
+        'manifold-firestore-backup'
+      )
       const response = responses[0]
       console.log(`Operation Name: ${response['name']}`)
-      })
+    } catch (err) {
-      .catch((err) => {
       console.error(err)
       throw new Error('Export operation failed')
-      })
+    }
   })

functions/src/scripts/backup-db.ts

@@ -0,0 +1,16 @@
+import * as firestore from '@google-cloud/firestore'
+import { getServiceAccountCredentials } from './script-init'
+import { backupDbCore } from '../backup-db'
+
+async function backupDb() {
+  const credentials = getServiceAccountCredentials()
+  const projectId = credentials.project_id
+  const client = new firestore.v1.FirestoreAdminClient({ credentials })
+  const bucket = 'manifold-firestore-backup'
+  const resp = await backupDbCore(client, projectId, bucket)
+  console.log(`Operation: ${resp[0]['name']}`)
+}
+
+if (require.main === module) {
+  backupDb().then(() => process.exit())
+}

functions/src/scripts/script-init.ts

@@ -47,26 +47,29 @@ const getFirebaseActiveProject = (cwd: string) => {
   }
 }
 
-export const initAdmin = (env?: string) => {
+export const getServiceAccountCredentials = (env?: string) => {
   env = env || getFirebaseActiveProject(process.cwd())
   if (env == null) {
-    console.error(
+    throw new Error(
       "Couldn't find active Firebase project; did you do `firebase use <alias>?`"
     )
-    return
   }
   const envVar = `GOOGLE_APPLICATION_CREDENTIALS_${env.toUpperCase()}`
   const keyPath = process.env[envVar]
   if (keyPath == null) {
-    console.error(
+    throw new Error(
       `Please set the ${envVar} environment variable to contain the path to your ${env} environment key file.`
     )
-    return
   }
-  console.log(`Initializing connection to ${env} Firebase...`)
   /* eslint-disable-next-line @typescript-eslint/no-var-requires */
-  const serviceAccount = require(keyPath)
+  return require(keyPath)
-  admin.initializeApp({
+}
+
+export const initAdmin = (env?: string) => {
+  const serviceAccount = getServiceAccountCredentials(env)
+  console.log(`Initializing connection to ${serviceAccount.project_id}...`)
+  return admin.initializeApp({
+    projectId: serviceAccount.project_id,
     credential: admin.credential.cert(serviceAccount),
   })
 }