gjo-calibration/strmlt.py

import numpy as np
import pandas as pd
import streamlit as st
import uncurl
from calibration import overconfidence
from firebase_requests import get_forecasts, get_resolutions
from gjo_requests import get_resolved_questions
from plotting import plotly_calibration, plotly_calibration_odds

if __name__ == "__main__":

    st.title("Learn how calibrated are you?")

    # ---

    # if st.checkbox('I am new! Show me instructions.'):
    #     st.write("""
    #         Hey!
    #     """)

    # ---

    platform = st.selectbox(
        "Which platform are you using?",
        ["Good Judgement Open", "CSET Foretell"],
    )
    platform_url = {
        "Good Judgement Open": "https://www.gjopen.com",
        "CSET Foretell": "https://www.cset-foretell.com",
    }[platform]

    uid = st.number_input("What is your user ID?", min_value=1, value=28899)
    uid = str(uid)

    curl_value = """curl 'https://www.gjopen.com/' \\
  -H 'authority: www.gjopen.com' \\
  -H '...: ...'"""
    curl_command = st.text_area(
        "Ugh... Gimme your cURL info...", value=curl_value
    )
    curl_command = "".join(curl_command.split("\\\n"))
    
    if curl_command != curl_value:

        curl_content = uncurl.parse_context(curl_command)
        headers, cookies = curl_content.headers, curl_content.cookies

        # ---

        questions = get_resolved_questions(uid, platform_url, headers, cookies)

        st.write(f"{len(questions)} questions you forecasted on have resolved.")

        # ---
        # TODO: Make a progress bar..?

        forecasts = get_forecasts(uid, questions, platform_url, headers, cookies)
        resolutions = get_resolutions(questions, platform_url, headers, cookies)

        # ---

        num_forecasts = sum(len(f) for f in forecasts.values())
        st.write(
            f"On these {len(questions)} questions you've made {num_forecasts} forecasts."
        )

        flatten = lambda t: [item for sublist in t for item in sublist]
        y_true = flatten(resolutions[q]["y_true"] for q in questions for _ in forecasts[q])
        y_pred = flatten(f["y_pred"] for q in questions for f in forecasts[q])

        # Note that I am "double counting" each prediction.
        if st.checkbox("Drop last"):
            y_true = flatten(
                resolutions[q]["y_true"][:-1] for q in questions for _ in forecasts[q]
            )
            y_pred = flatten(f["y_pred"][:-1] for q in questions for f in forecasts[q])

        y_true, y_pred = np.array(y_true), np.array(y_pred)

        st.write(f"Which gives us {len(y_pred)} datapoints to work with.")

        # ---

        strategy = st.selectbox(
            "Which binning stranegy do you prefer?",
            ["uniform", "quantile"],
        )

        recommended_n_bins = int(np.sqrt(len(y_pred))) if strategy == "quantile" else 20 + 1
        n_bins = st.number_input(
            "How many bins do you want me to display?",
            min_value=1,
            value=recommended_n_bins,
        )

        fig = plotly_calibration(y_true, y_pred, n_bins=n_bins, strategy=strategy)
        st.plotly_chart(fig, use_container_width=True)

        overconf = overconfidence(y_true, y_pred)
        st.write(f"Your over/under- confidence score is {overconf:.2f}.")

        # ---

        fig = plotly_calibration_odds(y_true, y_pred, n_bins=n_bins, strategy=strategy)
        st.plotly_chart(fig, use_container_width=True)