Merge pull request #55 from quantified-uncertainty/vercel

merge: Vercel and Terraform
This commit is contained in:
Nuño Sempere 2022-04-12 17:20:21 -04:00 committed by GitHub
commit 132deea7b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 195 additions and 10 deletions

View File

@ -20,7 +20,7 @@ $ npm install
You'll need a PostgreSQL instance, either local (see https://www.postgresql.org/download/) or in the cloud (for example, you can spin one up on https://www.digitalocean.com/products/managed-databases-postgresql or https://supabase.com/).
Environment can be set up with an `.env` file. You'll need to configure at least `DIGITALOCEAN_POSTGRES` for the fetching to work, and `NEXT_PUBLIC_SITE_URL` for the frontend.
Environment can be set up with an `.env` file. You'll need to configure at least `DIGITALOCEAN_POSTGRES`.
See [./docs/configuration.md](./docs/configuration.md) for details.

View File

@ -5,14 +5,16 @@ All configuration is done through environment variables.
Not all of these are necessary to run the code. The most important ones are:
- `DIGITALOCEAN_POSTGRES` pointing to the working Postgres database
- `NEXT_PUBLIC_SITE_URL` for the frontend to work properly
Environment for production deployments is configured through Terraform, see [infra.md](./infra.md) for details.
For local development you can write `.env` file by hand or import it from Heroku with `heroku config -s -a metaforecast-backend` and then modify accordingly.
There's also a template configuration file in `../env.example`.
## Database endpoints
- `DIGITALOCEAN_POSTGRES`, of the form `postgres://username:password@domain.com:port/configvars`. (Disregard `DIGITALOCEAN_` prefix, you can use any endpoint you like).
- `DIGITALOCEAN_POSTGRES_PUBLIC`
- `ALGOLIA_MASTER_API_KEY`, a string of 32 hexidecimal characters, like `19b6c2234e50c98d30668659a39e3127` (not an actual key).
- `NEXT_PUBLIC_ALGOLIA_APP_ID`,
- `NEXT_PUBLIC_ALGOLIA_SEARCH_KEY`
@ -25,7 +27,6 @@ Note that not all of these cookies are needed to use all parts of the source cod
- `GOODJUDGMENTOPENCOOKIE`
- `INFER_COOKIE`
- `CSETFORETELL_COOKIE`, deprecated, superseded by `INFER_COOKIE`.
- `HYPERMINDCOOKIE`
- `GOOGLE_API_KEY`, necessary to fetch Peter Wildeford's predictions.
- `SECRET_BETFAIR_ENDPOINT`
@ -37,7 +38,5 @@ Note that not all of these cookies are needed to use all parts of the source cod
## Others
- `NEXT_PUBLIC_SITE_URL`, e.g., `http://localhost:3000` if you're running a local instance
- `REBUIDNETLIFYHOOKURL`
- `BACKUP_PROXY_IP`
- `BACKUP_PROXY_PORT`

45
docs/infra.md Normal file
View File

@ -0,0 +1,45 @@
# Terraform
Infra is managed by [Terraform](https://www.terraform.io/) (WIP, not everything is migrated yet).
Managed with Terraform:
- Vercel
- Digital Ocean (database)
- Heroku
TODO:
- Algolia
- Twitter bot
- DNS?
## Recipes
### Set up a new dev repo for managing prod
1. Install [Terraform CLI](https://www.terraform.io/downloads)
2. `cd tf`
3. `terraform init`
4. Get a current version of prod tfvars configuration
- Source is in `metaforecast-notes-and-secrets` secret repo, `tf/prod.auto.tfvars` for now (will move to Terraform Cloud later)
- Store it in `tf/prod.auto.tfvars` (or somewhere else, there are [other ways](https://www.terraform.io/language/values/variables#assigning-values-to-root-module-variables))
5. Get a current version of terraform state
- Source is in `metaforecast-notes-and-secrets` secret repo for now (will move to Terraform Cloud or [pg backend](https://www.terraform.io/language/settings/backends/pg) later)
- Store it in `tf/terraform.tfstate`
Now everything is set up.
Check with `terraform plan`; it should output `"No changes. Your infrastructure matches the configuration."`.
### Edit environment variables in prod
1. Update terraform state and vars from `metaforecast-notes-and-secrets`
2. Modify `tf/prod.auto.tfvars` as needed
3. Run `terraform apply`
- Check if proposed actions list is appropriate
- Enter `yes`
- Terraform will push the new configuration to Heroku and Vercel.
4. Push terraform state and vars back to `metaforecast-notes-and-secrets`
(After we move to Terraform Cloud (1) and (4) won't be needed.)

View File

@ -8,8 +8,6 @@
# DIGITALOCEAN_POSTGRES=postgresql://...@localhost:5432/...?schema=public
# POSTGRES_NO_SSL=1
# NEXT_PUBLIC_SITE_URL=http://localhost:3000
# DEBUG_MODE=off
# INFER_COOKIE=...

View File

@ -8,6 +8,7 @@ import { InfoBox } from "../../../web/display/InfoBox";
import { Layout } from "../../../web/display/Layout";
import { LineHeader } from "../../../web/display/LineHeader";
import { FrontendForecast } from "../../../web/platforms";
import { reqToBasePath } from "../../../web/utils";
import { getDashboardForecastsByDashboardId } from "../../../web/worker/getDashboardForecasts";
interface Props {
@ -23,6 +24,7 @@ export const getServerSideProps: GetServerSideProps<Props> = async (
const { dashboardForecasts, dashboardItem } =
await getDashboardForecastsByDashboardId({
dashboardId,
basePath: reqToBasePath(context.req), // required on server side to find the API endpoint
});
if (!dashboardItem) {

11
src/web/utils.ts Normal file
View File

@ -0,0 +1,11 @@
import { IncomingMessage } from "http";
export const reqToBasePath = (req: IncomingMessage) => {
if (process.env.NEXT_PUBLIC_VERCEL_URL) {
console.log(process.env.NEXT_PUBLIC_VERCEL_URL);
return `https://${process.env.NEXT_PUBLIC_VERCEL_URL}`;
}
// we could just hardcode http://localhost:3000 here, but then `next dev -p <CUSTOM_PORT>` would break
return "http://" + req.headers.host;
};

View File

@ -6,16 +6,24 @@ import { addLabelsToForecasts, FrontendForecast } from "../platforms";
export async function getDashboardForecastsByDashboardId({
dashboardId,
basePath,
}: {
dashboardId: string;
basePath?: string;
}): Promise<{
dashboardForecasts: FrontendForecast[];
dashboardItem: DashboardItem;
}> {
console.log("getDashboardForecastsByDashboardId: ");
if (typeof window === undefined && !basePath) {
throw new Error("`basePath` option is required on server side");
}
let dashboardForecasts: Forecast[] = [];
let dashboardItem: DashboardItem | null = null;
try {
const { data } = await axios({
url: `${process.env.NEXT_PUBLIC_SITE_URL}/api/dashboard-by-id`,
let { data } = await axios({
url: `${basePath || ""}/api/dashboard-by-id`,
method: "post",
data: {
id: dashboardId,

3
tf/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
/*.tfstate*
/.terraform*
/prod.*tfvars

95
tf/main.tf Normal file
View File

@ -0,0 +1,95 @@
terraform {
required_providers {
# official vercel/vercel provider seems less stable
vercel = {
source = "registry.terraform.io/chronark/vercel"
version = ">=0.10.3"
}
digitalocean = {
source = "digitalocean/digitalocean"
version = "~> 2.0"
}
heroku = {
source = "heroku/heroku"
version = "~> 5.0.2"
}
local = {
source = "hashicorp/local"
version = "~> 2"
}
}
}
provider "vercel" {
token = var.vercel_api_token
}
provider "digitalocean" {
token = var.digital_ocean_token
}
provider "heroku" {
email = var.heroku_email
api_key = var.heroku_api_key
}
resource "digitalocean_database_cluster" "metaforecast_db" {
name = "postgres-green"
engine = "pg"
size = "db-s-1vcpu-1gb"
region = "nyc1"
node_count = 1
version = 14
}
locals {
generated_env = merge(var.metaforecast_env, {
# should we bring proper DO certificates to prod instead?
DIGITALOCEAN_POSTGRES = replace(digitalocean_database_cluster.metaforecast_db.uri, "/\\?sslmode=require$/", "")
})
}
resource "heroku_app" "metaforecast_backend" {
name = "metaforecast-backend"
region = "us"
config_vars = local.generated_env
}
resource "vercel_project" "metaforecast" {
name = "metaforecast"
team_id = var.vercel_team
framework = "nextjs"
git_repository {
type = "github"
repo = "quantified-uncertainty/metaforecast"
}
domain {
name = "metaforecast.org"
}
domain {
name = "www.metaforecast.org"
redirect = "metaforecast.org"
redirect_status_code = 308
}
}
resource "vercel_env" "metaforecast" {
project_id = vercel_project.metaforecast.id
team_id = var.vercel_team
type = "plain"
for_each = local.generated_env
key = each.key
value = each.value
target = ["preview", "production"]
}
# should probably be replaced with local bash script
# resource "local_file" "foo" {
# content = join("", concat(["# generated by terraform\n"], [for k, v in var.metaforecast_env : "${k} = \"${v}\"\n"]))
# filename = "${path.module}/../.env.prod"
# file_permission = "0644"
# }

24
tf/variables.tf Normal file
View File

@ -0,0 +1,24 @@
variable "vercel_api_token" {
type = string
}
variable "digital_ocean_token" {
type = string
}
variable "heroku_api_key" {
type = string
}
variable "heroku_email" {
type = string
}
variable "vercel_team" {
type = string
default = "quantified-uncertainty"
}
variable "metaforecast_env" {
type = map(string)
}