diff --git a/README.md b/README.md index 9e1180c..1021081 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ $ npm install You'll need a PostgreSQL instance, either local (see https://www.postgresql.org/download/) or in the cloud (for example, you can spin one up on https://www.digitalocean.com/products/managed-databases-postgresql or https://supabase.com/). -Environment can be set up with an `.env` file. You'll need to configure at least `DIGITALOCEAN_POSTGRES` for the fetching to work, and `NEXT_PUBLIC_SITE_URL` for the frontend. +Environment can be set up with an `.env` file. You'll need to configure at least `DIGITALOCEAN_POSTGRES`. See [./docs/configuration.md](./docs/configuration.md) for details. diff --git a/docs/configuration.md b/docs/configuration.md index fc8fb7b..4be5f52 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -5,14 +5,16 @@ All configuration is done through environment variables. Not all of these are necessary to run the code. The most important ones are: - `DIGITALOCEAN_POSTGRES` pointing to the working Postgres database -- `NEXT_PUBLIC_SITE_URL` for the frontend to work properly + +Environment for production deployments is configured through Terraform, see [infra.md](./infra.md) for details. + +For local development you can write `.env` file by hand or import it from Heroku with `heroku config -s -a metaforecast-backend` and then modify accordingly. There's also a template configuration file in `../env.example`. ## Database endpoints - `DIGITALOCEAN_POSTGRES`, of the form `postgres://username:password@domain.com:port/configvars`. (Disregard `DIGITALOCEAN_` prefix, you can use any endpoint you like). -- `DIGITALOCEAN_POSTGRES_PUBLIC` - `ALGOLIA_MASTER_API_KEY`, a string of 32 hexidecimal characters, like `19b6c2234e50c98d30668659a39e3127` (not an actual key). - `NEXT_PUBLIC_ALGOLIA_APP_ID`, - `NEXT_PUBLIC_ALGOLIA_SEARCH_KEY` @@ -25,7 +27,6 @@ Note that not all of these cookies are needed to use all parts of the source cod - `GOODJUDGMENTOPENCOOKIE` - `INFER_COOKIE` -- `CSETFORETELL_COOKIE`, deprecated, superseded by `INFER_COOKIE`. - `HYPERMINDCOOKIE` - `GOOGLE_API_KEY`, necessary to fetch Peter Wildeford's predictions. - `SECRET_BETFAIR_ENDPOINT` @@ -37,7 +38,5 @@ Note that not all of these cookies are needed to use all parts of the source cod ## Others -- `NEXT_PUBLIC_SITE_URL`, e.g., `http://localhost:3000` if you're running a local instance -- `REBUIDNETLIFYHOOKURL` - `BACKUP_PROXY_IP` - `BACKUP_PROXY_PORT` diff --git a/docs/infra.md b/docs/infra.md new file mode 100644 index 0000000..5685f5c --- /dev/null +++ b/docs/infra.md @@ -0,0 +1,45 @@ +# Terraform + +Infra is managed by [Terraform](https://www.terraform.io/) (WIP, not everything is migrated yet). + +Managed with Terraform: + +- Vercel +- Digital Ocean (database) +- Heroku + +TODO: + +- Algolia +- Twitter bot +- DNS? + +## Recipes + +### Set up a new dev repo for managing prod + +1. Install [Terraform CLI](https://www.terraform.io/downloads) +2. `cd tf` +3. `terraform init` +4. Get a current version of prod tfvars configuration + - Source is in `metaforecast-notes-and-secrets` secret repo, `tf/prod.auto.tfvars` for now (will move to Terraform Cloud later) + - Store it in `tf/prod.auto.tfvars` (or somewhere else, there are [other ways](https://www.terraform.io/language/values/variables#assigning-values-to-root-module-variables)) +5. Get a current version of terraform state + - Source is in `metaforecast-notes-and-secrets` secret repo for now (will move to Terraform Cloud or [pg backend](https://www.terraform.io/language/settings/backends/pg) later) + - Store it in `tf/terraform.tfstate` + +Now everything is set up. + +Check with `terraform plan`; it should output `"No changes. Your infrastructure matches the configuration."`. + +### Edit environment variables in prod + +1. Update terraform state and vars from `metaforecast-notes-and-secrets` +2. Modify `tf/prod.auto.tfvars` as needed +3. Run `terraform apply` + - Check if proposed actions list is appropriate + - Enter `yes` + - Terraform will push the new configuration to Heroku and Vercel. +4. Push terraform state and vars back to `metaforecast-notes-and-secrets` + +(After we move to Terraform Cloud (1) and (4) won't be needed.) diff --git a/env.example b/env.example index 8145c9d..481bf11 100644 --- a/env.example +++ b/env.example @@ -8,8 +8,6 @@ # DIGITALOCEAN_POSTGRES=postgresql://...@localhost:5432/...?schema=public # POSTGRES_NO_SSL=1 -# NEXT_PUBLIC_SITE_URL=http://localhost:3000 - # DEBUG_MODE=off # INFER_COOKIE=... diff --git a/src/pages/dashboards/view/[id].tsx b/src/pages/dashboards/view/[id].tsx index 8c60b72..84184e0 100644 --- a/src/pages/dashboards/view/[id].tsx +++ b/src/pages/dashboards/view/[id].tsx @@ -8,6 +8,7 @@ import { InfoBox } from "../../../web/display/InfoBox"; import { Layout } from "../../../web/display/Layout"; import { LineHeader } from "../../../web/display/LineHeader"; import { FrontendForecast } from "../../../web/platforms"; +import { reqToBasePath } from "../../../web/utils"; import { getDashboardForecastsByDashboardId } from "../../../web/worker/getDashboardForecasts"; interface Props { @@ -23,6 +24,7 @@ export const getServerSideProps: GetServerSideProps = async ( const { dashboardForecasts, dashboardItem } = await getDashboardForecastsByDashboardId({ dashboardId, + basePath: reqToBasePath(context.req), // required on server side to find the API endpoint }); if (!dashboardItem) { diff --git a/src/web/utils.ts b/src/web/utils.ts new file mode 100644 index 0000000..ddee0c1 --- /dev/null +++ b/src/web/utils.ts @@ -0,0 +1,11 @@ +import { IncomingMessage } from "http"; + +export const reqToBasePath = (req: IncomingMessage) => { + if (process.env.NEXT_PUBLIC_VERCEL_URL) { + console.log(process.env.NEXT_PUBLIC_VERCEL_URL); + return `https://${process.env.NEXT_PUBLIC_VERCEL_URL}`; + } + + // we could just hardcode http://localhost:3000 here, but then `next dev -p ` would break + return "http://" + req.headers.host; +}; diff --git a/src/web/worker/getDashboardForecasts.ts b/src/web/worker/getDashboardForecasts.ts index 2a85232..95dba39 100644 --- a/src/web/worker/getDashboardForecasts.ts +++ b/src/web/worker/getDashboardForecasts.ts @@ -6,16 +6,24 @@ import { addLabelsToForecasts, FrontendForecast } from "../platforms"; export async function getDashboardForecastsByDashboardId({ dashboardId, + basePath, +}: { + dashboardId: string; + basePath?: string; }): Promise<{ dashboardForecasts: FrontendForecast[]; dashboardItem: DashboardItem; }> { console.log("getDashboardForecastsByDashboardId: "); + if (typeof window === undefined && !basePath) { + throw new Error("`basePath` option is required on server side"); + } + let dashboardForecasts: Forecast[] = []; let dashboardItem: DashboardItem | null = null; try { - const { data } = await axios({ - url: `${process.env.NEXT_PUBLIC_SITE_URL}/api/dashboard-by-id`, + let { data } = await axios({ + url: `${basePath || ""}/api/dashboard-by-id`, method: "post", data: { id: dashboardId, diff --git a/tf/.gitignore b/tf/.gitignore new file mode 100644 index 0000000..458cf26 --- /dev/null +++ b/tf/.gitignore @@ -0,0 +1,3 @@ +/*.tfstate* +/.terraform* +/prod.*tfvars diff --git a/tf/main.tf b/tf/main.tf new file mode 100644 index 0000000..f76dbf3 --- /dev/null +++ b/tf/main.tf @@ -0,0 +1,95 @@ +terraform { + required_providers { + # official vercel/vercel provider seems less stable + vercel = { + source = "registry.terraform.io/chronark/vercel" + version = ">=0.10.3" + } + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + heroku = { + source = "heroku/heroku" + version = "~> 5.0.2" + } + local = { + source = "hashicorp/local" + version = "~> 2" + } + } +} + +provider "vercel" { + token = var.vercel_api_token +} + +provider "digitalocean" { + token = var.digital_ocean_token +} + +provider "heroku" { + email = var.heroku_email + api_key = var.heroku_api_key +} + +resource "digitalocean_database_cluster" "metaforecast_db" { + name = "postgres-green" + engine = "pg" + size = "db-s-1vcpu-1gb" + region = "nyc1" + node_count = 1 + version = 14 +} + +locals { + generated_env = merge(var.metaforecast_env, { + # should we bring proper DO certificates to prod instead? + DIGITALOCEAN_POSTGRES = replace(digitalocean_database_cluster.metaforecast_db.uri, "/\\?sslmode=require$/", "") + }) +} + +resource "heroku_app" "metaforecast_backend" { + name = "metaforecast-backend" + region = "us" + + config_vars = local.generated_env +} + +resource "vercel_project" "metaforecast" { + name = "metaforecast" + team_id = var.vercel_team + framework = "nextjs" + + git_repository { + type = "github" + repo = "quantified-uncertainty/metaforecast" + } + + domain { + name = "metaforecast.org" + } + + domain { + name = "www.metaforecast.org" + redirect = "metaforecast.org" + redirect_status_code = 308 + } +} + +resource "vercel_env" "metaforecast" { + project_id = vercel_project.metaforecast.id + team_id = var.vercel_team + type = "plain" + for_each = local.generated_env + key = each.key + value = each.value + target = ["preview", "production"] +} + +# should probably be replaced with local bash script +# resource "local_file" "foo" { +# content = join("", concat(["# generated by terraform\n"], [for k, v in var.metaforecast_env : "${k} = \"${v}\"\n"])) +# filename = "${path.module}/../.env.prod" +# file_permission = "0644" +# } diff --git a/tf/variables.tf b/tf/variables.tf new file mode 100644 index 0000000..450d3fb --- /dev/null +++ b/tf/variables.tf @@ -0,0 +1,24 @@ +variable "vercel_api_token" { + type = string +} + +variable "digital_ocean_token" { + type = string +} + +variable "heroku_api_key" { + type = string +} + +variable "heroku_email" { + type = string +} + +variable "vercel_team" { + type = string + default = "quantified-uncertainty" +} + +variable "metaforecast_env" { + type = map(string) +}