Support local firestore emulation (#91)

* Support local firestore emulation

* Update readme

* Update readme

functions/.gitignore

@@ -15,3 +15,5 @@ node_modules/
 package-lock.json
 ui-debug.log
 firebase-debug.log
+firestore-debug.log
+firestore_export/

functions/README.md

@@ -19,14 +19,33 @@ Adapted from https://firebase.google.com/docs/functions/get-started
 2. `$ yarn` to install JS dependencies
 3. `$ firebase login` to authenticate the CLI tools to Firebase
 4. `$ firebase use dev` to choose the dev project
-5. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev (TODO: maybe not for Manifold)
+
+### For local development
+
+0. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev
+1. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI
+2. `$ brew install java` to install java if you don't already have it
+   1. `$ echo 'export PATH="/usr/local/opt/openjdk/bin:$PATH"' >> ~/.zshrc` to add java to your path
+3. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud
+4. `$ gcloud config set project <project-id>` to choose the project (`$ gcloud projects list` to see options)
+5. `$ mkdir firestore_export` to create a folder to store the exported database
+6. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local
+   1. TODO: this won't work when open source, we'll have to point to the public db
 
 ## Developing locally
 
-0. `$ yarn dev` to spin up the emulators
+1. `$ yarn serve` to spin up the emulators
    The Emulator UI is at http://localhost:4000; the functions are hosted on :5001.
    Note: You have to kill and restart emulators when you change code; no hot reload =(
-1. Connect to emulators by enabling `functions.useEmulator('localhost', 5001)`
+2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend
+   1. Note: emulated database is cleared after every shutdown
+
+## Firestore Commands
+
+- `db:update-local-from-remote` - Pull the remote db from Firestore to local, also calls:
+  - `db:backup-remote` - Exports the remote dev db to the backup folder on Google Cloud Storage (called on every `db:update-local-from-remote`)
+  - `db:rename-remote-backup-folder` - Renames the remote backup folder (called on every `db:backup-remote` to preserve the previous db backup)
+- `db:backup-local` - Save the local db changes to the disk (overwrites existing)
 
 ## Debugging
 

functions/package.json

@@ -1,14 +1,21 @@
 {
   "name": "functions",
   "version": "1.0.0",
+  "config": {
+    "firestore": "dev-mantic-markets.appspot.com"
+  },
   "scripts": {
     "build": "tsc",
     "watch": "tsc -w",
-    "serve": "yarn build && firebase emulators:start --only functions",
     "shell": "yarn build && firebase functions:shell",
     "start": "yarn shell",
     "deploy": "firebase deploy --only functions",
-    "logs": "firebase functions:log"
+    "logs": "firebase functions:log",
+    "serve": "yarn build && firebase emulators:start --only functions,firestore --import=./firestore_export",
+    "db:update-local-from-remote": "yarn db:backup-remote && gsutil rsync -r gs://$npm_package_config_firestore/firestore_export ./firestore_export",
+    "db:backup-local": "firebase emulators:export --force ./firestore_export",
+    "db:rename-remote-backup-folder": "gsutil mv gs://$npm_package_config_firestore/firestore_export gs://$npm_package_config_firestore/firestore_export_$(date +%d-%m-%Y-%H-%M)",
+    "db:backup-remote": "yarn db:rename-remote-backup-folder && gcloud firestore export gs://$npm_package_config_firestore/firestore_export/"
   },
   "main": "lib/functions/src/index.js",
   "dependencies": {

web/README.md

@@ -8,6 +8,11 @@
 
 (`yarn dev` will point you to prod database)
 
+### Running with local emulated database and functions
+
+1. `yarn serve` first in `/functions` and wait for it to start
+2. `yarn dev:emulate` will point you to the emulated database
+
 ## Formatting
 
 Before committing, run `yarn format` to format your code.

web/lib/firebase/api-call.ts

@@ -1,10 +1,9 @@
-import { getFunctions, httpsCallable } from 'firebase/functions'
+import { httpsCallable } from 'firebase/functions'
 import { Fold } from '../../../common/fold'
 import { User } from '../../../common/user'
 import { randomString } from '../../../common/util/random'
 import './init'
-
+import { functions } from './init'
-const functions = getFunctions()
 
 export const cloudFunction = <RequestData, ResponseData>(name: string) =>
   httpsCallable<RequestData, ResponseData>(functions, name)

web/lib/firebase/init.ts

@@ -1,8 +1,26 @@
 import { getFirestore } from '@firebase/firestore'
 import { initializeApp, getApps, getApp } from 'firebase/app'
 import { FIREBASE_CONFIG } from '../../../common/envs/constants'
+import { connectFirestoreEmulator } from 'firebase/firestore'
+import { connectFunctionsEmulator, getFunctions } from 'firebase/functions'
 
 // Initialize Firebase
 export const app = getApps().length ? getApp() : initializeApp(FIREBASE_CONFIG)
+export const db = getFirestore()
+export const functions = getFunctions()
 
-export const db = getFirestore(app)
+const EMULATORS_STARTED = 'EMULATORS_STARTED'
+function startEmulators() {
+  // I don't like this but this is the only way to reconnect to the emulators without error, see: https://stackoverflow.com/questions/65066963/firebase-firestore-emulator-error-host-has-been-set-in-both-settings-and-usee
+  // @ts-ignore
+  if (!global[EMULATORS_STARTED]) {
+    // @ts-ignore
+    global[EMULATORS_STARTED] = true
+    connectFirestoreEmulator(db, 'localhost', 8080)
+    connectFunctionsEmulator(functions, 'localhost', 5001)
+  }
+}
+
+if (process.env.NEXT_PUBLIC_FIREBASE_EMULATE) {
+  startEmulators()
+}

web/package.json

@@ -7,6 +7,7 @@
     "devdev": "NEXT_PUBLIC_FIREBASE_ENV=DEV concurrently -n NEXT,TS -c magenta,cyan \"FIREBASE_ENV=DEV next dev -p 3000\" \"FIREBASE_ENV=DEV yarn ts --watch\" # see https://github.com/vercel/next.js/discussions/33634",
     "dev:dev": "yarn devdev",
     "dev:the": "NEXT_PUBLIC_FIREBASE_ENV=THEOREMONE concurrently -n NEXT,TS -c magenta,cyan \"FIREBASE_ENV=THEOREMONE next dev -p 3000\" \"FIREBASE_ENV=THEOREMONE yarn ts --watch\"",
+    "dev:emulate": "NEXT_PUBLIC_FIREBASE_EMULATE=TRUE yarn devdev",
     "ts": "tsc --noEmit --incremental --preserveWatchOutput --pretty",
     "build": "next build",
     "start": "next start",