gjo-calibration/strmlt.py

101 lines
3.0 KiB
Python
Raw Normal View History

2021-05-31 18:59:24 +00:00
import numpy as np
import pandas as pd
import streamlit as st
import uncurl
from calibration import overconfidence
from firebase_requests import get_forecasts, get_resolutions
from gjo_requests import get_resolved_questions
from plotting import plotly_calibration, plotly_calibration_odds
if __name__ == "__main__":
st.title("Learn how calibrated are you?")
# ---
# if st.checkbox('I am new! Show me instructions.'):
# st.write("""
# Hey!
# """)
# ---
platform = st.selectbox(
"Which platform are you using?",
["Good Judgement Open", "CSET Foretell"],
)
platform_url = {
"Good Judgement Open": "https://www.gjopen.com",
"CSET Foretell": "https://www.cset-foretell.com",
}[platform]
uid = st.number_input("What is your user ID?", min_value=1, value=28899)
uid = str(uid)
curl_value = ""
curl_command = st.text_area(
"Ugh... Gimme your cURL info...", value=curl_value.strip()
)
curl_content = uncurl.parse_context(curl_command)
headers, cookies = curl_content.headers, curl_content.cookies
# ---
questions = get_resolved_questions(uid, platform_url, headers, cookies)
st.write(f"{len(questions)} questions you forecasted on have resolved.")
# ---
# TODO: Make a progress bar..?
forecasts = get_forecasts(uid, questions, platform_url, headers, cookies)
resolutions = get_resolutions(questions, platform_url, headers, cookies)
# ---
num_forecasts = sum(len(f) for f in forecasts.values())
st.write(
f"On these {len(questions)} questions you've made {num_forecasts} forecasts."
)
flatten = lambda t: [item for sublist in t for item in sublist]
y_true = flatten(resolutions[q]["y_true"] for q in questions for _ in forecasts[q])
y_pred = flatten(f["y_pred"] for q in questions for f in forecasts[q])
# Note that I am "double counting" each prediction.
if st.checkbox("Drop last"):
y_true = flatten(
resolutions[q]["y_true"][:-1] for q in questions for _ in forecasts[q]
)
y_pred = flatten(f["y_pred"][:-1] for q in questions for f in forecasts[q])
y_true, y_pred = np.array(y_true), np.array(y_pred)
st.write(f"Which gives us {len(y_pred)} datapoints to work with.")
# ---
strategy = st.selectbox(
"Which binning stranegy do you prefer?",
["uniform", "quantile"],
)
recommended_n_bins = int(np.sqrt(len(y_pred))) if strategy == "quantile" else 20 + 1
n_bins = st.number_input(
"How many bins do you want me to display?",
min_value=1,
value=recommended_n_bins,
)
fig = plotly_calibration(y_true, y_pred, n_bins=n_bins, strategy=strategy)
st.plotly_chart(fig, use_container_width=True)
overconf = overconfidence(y_true, y_pred)
st.write(f"Your over/under- confidence score is {overconf:.2f}.")
# ---
fig = plotly_calibration_odds(y_true, y_pred, n_bins=n_bins, strategy=strategy)
st.plotly_chart(fig, use_container_width=True)