101 lines
3.0 KiB
Python
101 lines
3.0 KiB
Python
|
import numpy as np
|
||
|
|
import pandas as pd
|
||
|
|
import streamlit as st
|
||
|
|
import uncurl
|
||
|
|
from calibration import overconfidence
|
||
|
|
from firebase_requests import get_forecasts, get_resolutions
|
||
|
|
from gjo_requests import get_resolved_questions
|
||
|
|
from plotting import plotly_calibration, plotly_calibration_odds
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
|
||
|
|
st.title("Learn how calibrated are you?")
|
||
|
|
|
||
|
|
# ---
|
||
|
|
|
||
|
|
# if st.checkbox('I am new! Show me instructions.'):
|
||
|
|
# st.write("""
|
||
|
|
# Hey!
|
||
|
|
# """)
|
||
|
|
|
||
|
|
# ---
|
||
|
|
|
||
|
|
platform = st.selectbox(
|
||
|
|
"Which platform are you using?",
|
||
|
|
["Good Judgement Open", "CSET Foretell"],
|
||
|
|
)
|
||
|
|
platform_url = {
|
||
|
|
"Good Judgement Open": "https://www.gjopen.com",
|
||
|
|
"CSET Foretell": "https://www.cset-foretell.com",
|
||
|
|
}[platform]
|
||
|
|
|
||
|
|
uid = st.number_input("What is your user ID?", min_value=1, value=28899)
|
||
|
|
uid = str(uid)
|
||
|
|
|
||
|
|
curl_value = ""
|
||
|
|
|
||
|
|
curl_command = st.text_area(
|
||
|
|
"Ugh... Gimme your cURL info...", value=curl_value.strip()
|
||
|
|
)
|
||
|
|
curl_content = uncurl.parse_context(curl_command)
|
||
|
|
headers, cookies = curl_content.headers, curl_content.cookies
|
||
|
|
|
||
|
|
# ---
|
||
|
|
|
||
|
|
questions = get_resolved_questions(uid, platform_url, headers, cookies)
|
||
|
|
|
||
|
|
st.write(f"{len(questions)} questions you forecasted on have resolved.")
|
||
|
|
|
||
|
|
# ---
|
||
|
|
# TODO: Make a progress bar..?
|
||
|
|
|
||
|
|
forecasts = get_forecasts(uid, questions, platform_url, headers, cookies)
|
||
|
|
resolutions = get_resolutions(questions, platform_url, headers, cookies)
|
||
|
|
|
||
|
|
# ---
|
||
|
|
|
||
|
|
num_forecasts = sum(len(f) for f in forecasts.values())
|
||
|
|
st.write(
|
||
|
|
f"On these {len(questions)} questions you've made {num_forecasts} forecasts."
|
||
|
|
)
|
||
|
|
|
||
|
|
flatten = lambda t: [item for sublist in t for item in sublist]
|
||
|
|
y_true = flatten(resolutions[q]["y_true"] for q in questions for _ in forecasts[q])
|
||
|
|
y_pred = flatten(f["y_pred"] for q in questions for f in forecasts[q])
|
||
|
|
|
||
|
|
# Note that I am "double counting" each prediction.
|
||
|
|
if st.checkbox("Drop last"):
|
||
|
|
y_true = flatten(
|
||
|
|
resolutions[q]["y_true"][:-1] for q in questions for _ in forecasts[q]
|
||
|
|
)
|
||
|
|
y_pred = flatten(f["y_pred"][:-1] for q in questions for f in forecasts[q])
|
||
|
|
|
||
|
|
y_true, y_pred = np.array(y_true), np.array(y_pred)
|
||
|
|
|
||
|
|
st.write(f"Which gives us {len(y_pred)} datapoints to work with.")
|
||
|
|
|
||
|
|
# ---
|
||
|
|
|
||
|
|
strategy = st.selectbox(
|
||
|
|
"Which binning stranegy do you prefer?",
|
||
|
|
["uniform", "quantile"],
|
||
|
|
)
|
||
|
|
|
||
|
|
recommended_n_bins = int(np.sqrt(len(y_pred))) if strategy == "quantile" else 20 + 1
|
||
|
|
n_bins = st.number_input(
|
||
|
|
"How many bins do you want me to display?",
|
||
|
|
min_value=1,
|
||
|
|
value=recommended_n_bins,
|
||
|
|
)
|
||
|
|
|
||
|
|
fig = plotly_calibration(y_true, y_pred, n_bins=n_bins, strategy=strategy)
|
||
|
|
st.plotly_chart(fig, use_container_width=True)
|
||
|
|
|
||
|
|
overconf = overconfidence(y_true, y_pred)
|
||
|
|
st.write(f"Your over/under- confidence score is {overconf:.2f}.")
|
||
|
|
|
||
|
|
# ---
|
||
|
|
|
||
|
|
fig = plotly_calibration_odds(y_true, y_pred, n_bins=n_bins, strategy=strategy)
|
||
|
|
st.plotly_chart(fig, use_container_width=True)
|