Merge pull request #913 from quantified-uncertainty/develop

Develop -> Master, V0.3.0

packages/components/src/components/FunctionChart1Number.tsx

@@ -39,13 +39,10 @@ interface FunctionChart1NumberProps {
 type point = { x: number; value: result<number, string> };
 
 let getFunctionImage = ({ chartSettings, fn, environment }) => {
-  //We adjust the count, because the count is made for distributions, which are much more expensive to estimate
-  let adjustedCount = chartSettings.count * 20;
-
   let chartPointsToRender = _rangeByCount(
     chartSettings.start,
     chartSettings.stop,
-    adjustedCount
+    chartSettings.count
   );
 
   let chartPointsData: point[] = chartPointsToRender.map((x) => {

packages/components/src/components/SquiggleChart.tsx

@@ -73,7 +73,7 @@ export const SquiggleChart: React.FC<SquiggleChartProps> = React.memo(
     expY = false,
     diagramStart = 0,
     diagramStop = 10,
-    diagramCount = 100,
+    diagramCount = 20,
     tickFormat,
     minX,
     maxX,

packages/squiggle-lang/src/rescript/Distributions/PointSetDist/Continuous.res

@@ -250,7 +250,7 @@ module T = Dist({
 
   let downsample = (length, t): t =>
     t |> shapeMap(XYShape.XsConversion.proportionByProbabilityMass(length, integral(t).xyShape))
-  let integralEndY = (t: t) => t.integralSumCache |> E.O.default(t |> integral |> lastY)
+  let integralEndY = (t: t) => t.integralSumCache |> E.O.defaultFn(() => t |> integral |> lastY)
   let integralXtoY = (f, t: t) => t |> integral |> shapeFn(XYShape.XtoY.linear(f))
   let integralYtoX = (f, t: t) => t |> integral |> shapeFn(XYShape.YtoX.linear(f))
   let toContinuous = t => Some(t)

packages/squiggle-lang/src/rescript/Distributions/PointSetDist/Discrete.res

@@ -158,7 +158,8 @@ module T = Dist({
       Continuous.make(~interpolation=#Stepwise, integralShape)
     }
 
-  let integralEndY = (t: t) => t.integralSumCache |> E.O.default(t |> integral |> Continuous.lastY)
+  let integralEndY = (t: t) =>
+    t.integralSumCache |> E.O.defaultFn(() => t |> integral |> Continuous.lastY)
   let minX = shapeFn(XYShape.T.minX)
   let maxX = shapeFn(XYShape.T.maxX)
   let toDiscreteProbabilityMassFraction = _ => 1.0

packages/squiggle-lang/src/rescript/Distributions/PointSetDist/MixedShapeBuilder.res

@@ -13,9 +13,11 @@ let buildSimple = (
   ~discrete: option<PointSetTypes.discreteShape>,
 ): option<PointSetTypes.pointSetDist> => {
   let continuous =
-    continuous |> E.O.default(Continuous.make(~integralSumCache=Some(0.0), {xs: [], ys: []}))
+    continuous |> E.O.defaultFn(() =>
+      Continuous.make(~integralSumCache=Some(0.0), {xs: [], ys: []})
+    )
   let discrete =
-    discrete |> E.O.default(Discrete.make(~integralSumCache=Some(0.0), {xs: [], ys: []}))
+    discrete |> E.O.defaultFn(() => Discrete.make(~integralSumCache=Some(0.0), {xs: [], ys: []}))
   let cLength = continuous |> Continuous.getShape |> XYShape.T.xs |> E.A.length
   let dLength = discrete |> Discrete.getShape |> XYShape.T.xs |> E.A.length
   switch (cLength, dLength) {

packages/squiggle-lang/src/rescript/ReducerInterface/ReducerInterface_ExternalLibrary.res

@@ -16,7 +16,7 @@ let dispatch = (
     () => ReducerInterface_Duration.dispatch(call, environment),
     () => ReducerInterface_Number.dispatch(call, environment),
     () => FunctionRegistry_Library.dispatch(call, environment, reducer),
-  ])->E.O2.default(chain(call, environment, reducer))
+  ])->E.O2.defaultFn(() => chain(call, environment, reducer))
 }
 
 /*

packages/squiggle-lang/src/rescript/SquiggleLibrary/SquiggleLibrary_Versions.res

@@ -2,7 +2,7 @@ module Bindings = Reducer_Bindings
 
 let bindings: Bindings.t =
   [
-    ("System.version", ReducerInterface_InternalExpressionValue.IEvString("0.2.12")),
+    ("System.version", ReducerInterface_InternalExpressionValue.IEvString("0.3.0")),
   ]->Bindings.fromArray
 
 let makeBindings = (previousBindings: Bindings.t): Bindings.t =>

packages/squiggle-lang/src/rescript/Utility/E.res

@@ -82,6 +82,11 @@ module O = {
     | None => d
     | Some(a) => a
     }
+  let defaultFn = (d, o) =>
+    switch o {
+    | None => d()
+    | Some(a) => a
+    }
   let isSome = o =>
     switch o {
     | Some(_) => true
@@ -158,6 +163,7 @@ module O = {
 
 module O2 = {
   let default = (a, b) => O.default(b, a)
+  let defaultFn = (a, b) => O.defaultFn(b, a)
   let toExn = (a, b) => O.toExn(b, a)
   let fmap = (a, b) => O.fmap(b, a)
   let toResult = (a, b) => O.toResult(b, a)

packages/website/docs/Api/List.md

@@ -73,6 +73,14 @@ map: (list<'a>, a => b) => list<'b>
 
 See [Rescript implementation](https://rescript-lang.org/docs/manual/latest/api/belt/array#map).
 
+### filter
+
+```
+filter: (list<'a>, 'a => bool) => list<'a>
+```
+
+See [Rescript implementation of keep](https://rescript-lang.org/docs/manual/latest/api/belt/array#keep), which is functionally equivalent.
+
 ### reduce
 
 ```

packages/website/docs/Discussions/Bugs.mdx

@@ -7,7 +7,12 @@ import { SquiggleEditor } from "../../src/components/SquiggleEditor";
 
 Much of the Squiggle math is imprecise. This can cause significant errors, so watch out.
 
-Below are some specific examples to watch for. We'll work on improving these over time and adding much better warnings and error management.
+Below are a few specific examples to watch for. We'll work on improving these over time and adding much better warnings and error management.
+
+## Operations on very small or large numbers, silently round to 0 and 1
+
+Squiggle is poor at dealing with very small or large numbers, given fundamental limitations of floating point precision.
+See [this Github Issue](https://github.com/quantified-uncertainty/squiggle/issues/834).
 
 ## Mixtures of distributions with very different means
 

packages/website/docs/Discussions/Future-Features.md

@@ -3,26 +3,21 @@ title: Future Features
 sidebar_position: 3
 ---
 
-Squiggle is still very early. The main first goal is to become stable. This means having a clean codebase, having decent test coverage, and having a syntax we are reasonably confident in. Later on, there are many other features that will be interesting to explore.
+Squiggle is still very early. The main first goal is to become stable (to reach version 1.0). Right now we think it is useable to use for small projects, but do note that there are very likely some math bugs and performance problems.
 
 ## Programming Language Features
 
-- Equality (a == b)
-- If/else statements
-- Arrays
 - Tables / Matrices
-- Simple objects
 - A simple type system
-- Simple module system (`Dist.Normal` instead of `normal`)
-- A simple time library & notation
 - Optional and default paramaters for functions
-- Anonymous Functions (This is particularly convenient in cases where tiny functions are submitted in forecasting competitions)
 - A notation to limit the domain of functions. For example, maybe a function only applies for t=[2 to 20]
-- Custom parser (Right now we're using Math.js's parser, which doesn't give us much flexibility)
+- Some story for tests
-- "Partial-domain" distributions. For example, maybe someone has a distribution for when AGI will happen, but doesn't want to make any estimates past 2200.
+- Much better code editor integration
 
 ## Distribution Features
 
+There are many important distribution types that Squiggle doesn't yet support. Some key functions we'd like include:
+
 [Metalog Distribution](https://en.wikipedia.org/wiki/Metalog_distribution)  
 Add the Metalog distribution, and some convenient methods for generating these distributions. This might be a bit tricky because we might need or build a library to fit data. There's no Metalog javascript library yet, this would be pretty useful. There's already a Metalog library in Python, so that one could be used for inspiration.
 
@@ -34,15 +29,6 @@ Takes a distribution and smoothens it. For example, [Elicit Forecast](https://fo
 **Probabilities**  
 Right now Squiggle mostly works with probability distributions only, but it should also work smoothly with probabilities.
 
-**Scoring**  
-Have functions to score probabilities, probability distributions, and functions that return probability distributions.
-
-**Full javascript library**  
-A full Javascript library that accesses most of the probabilistic functionality of Squiggle, but can be used directly in javascript functions.
-
-**Importance & quality scores**  
-Workflows/functionality to declare the importance and coveredness of each part of the paramater space. For example, some subsets of the paramater space of a function might be much more important to get right than others. Similarly, the analyst might be much more certain about some parts than others. Ideally. they could decline sections.
-
 **An interface to interpret & score Squiggle files**  
 Squiggle functions need to be aggregated and scored. This should be done outside one Squiggle file. Maybe this should also be done in Squiggle, or maybe it should be done using Javascript.
 
@@ -52,55 +38,17 @@ Of course, we'd also need good math for how the scoring should work, exactly.
 
 This interface should also be able to handle changing Squiggle values. This is because people would be likely to want to update their functions over time, and that should be taken into account for scoring.
 
-**Easily call other functions**  
+**Importance & quality scores**  
-It would be great to be able to call other people's Squiggle functions, from other Squiggle functions. This could raise a whole bunch of challenging issues. Additionally, it would be neat to call other data, both from knowledge graphs, and from regular APIs. Note that this could obviously complicate scoring a lot; I imagine that either easy scoring, or simple data fetching, would have to accept sacrifices.
+Workflows/functionality to declare the importance and coveredness of each part of the paramater space. For example, some subsets of the paramater space of a function might be much more important to get right than others. Similarly, the analyst might be much more certain about some parts than others. Ideally. they could decline sections.
 
-**Correlated uncertainties**  
+**Static / sensitivity analysis**  
-Right now there's no functionality to declare that two different distributions are correlated.
-
-**Static / Sensitivity Analysis**  
 Guesstimate has Sensitivity analysis that's pretty useful. This could be quite feasible to add, though it will likely require some thinking.
 
 **Annotation**  
 It might be useful to allow people to annotate functions and variables with longer descriptions, maybe Markdown. This could very much help interpretation/analysis of these items.
 
-**Randomness Seeds**  
+**Randomness seeds**  
 Right now, Monte Carlo simulations are totally random. It would be nicer to be able to enter a seed somehow in order to control the randomness. Or, with the same seed, the function should always return the same values. This would make debugging and similar easier.
 
-## Major Standard Language Features
+**Caching/memoization**  
-
+There are many performance improvements that Squiggle could have. We'll get to some of them eventually.
-- Some testing story.
-- A custom code highlighting format.
-- Possibly a decent web GUI (a much more advanced playground).
-- A VS Code extention and similar.
-
-## Bugs
-
-- Discrete distributions are particularly buggy. Try `mm(1,2,3,4,5,6,7,8,9,10) .* (5 to 8)`
-
-## New Functions
-
-### Distributions
-
-```js
-cauchy();
-pareto();
-metalog();
-```
-
-Possibly change mm to mix, or mx(). Also, change input format, maybe to mx([a,b,c], [a,b,c]).
-
-### Functions
-
-```js
-samples(distribution, n);
-toPdf(distribution);
-toCdf(distribution);
-toHash(distribution);
-trunctate(distribution, leftValue, rightValue);
-leftTrunctate(distribution, leftValue);
-rightTrunctate(distribution, rightValue);
-distributionFromSamples(array, params);
-distributionFromPoints();
-distributionFromHash();
-```

packages/website/docs/Discussions/Gallery.md

@@ -3,6 +3,8 @@ sidebar_position: 2
 title: Gallery
 ---
 
-- [Adjusting probabilities for the passage of time](https://www.lesswrong.com/s/rDe8QE5NvXcZYzgZ3/p/j8o6sgRerE3tqNWdj) by Nuño Sempere
 - [GiveWell's GiveDirectly cost effectiveness analysis](https://observablehq.com/@hazelfire/givewells-givedirectly-cost-effectiveness-analysis) by Sam Nolan
-- [Astronomical Waste](https://observablehq.com/@quinn-dougherty/waste)
+- [A Critical Review of Open Philanthropy’s Bet On Criminal Justice Reform](https://forum.effectivealtruism.org/posts/h2N9qEbvQ6RHABcae/a-critical-review-of-open-philanthropy-s-bet-on-criminal) by Nuño Sempere
+- [Samotsvety Nuclear Risk Forecasts — March 2022](https://forum.effectivealtruism.org/posts/KRFXjCqqfGQAYirm5/samotsvety-nuclear-risk-forecasts-march-2022) by Nuño Sempere, Misha Yagudin, Eli Lifland
+- [Adjusting probabilities for the passage of time](https://www.lesswrong.com/s/rDe8QE5NvXcZYzgZ3/p/j8o6sgRerE3tqNWdj) by Nuño Sempere
+- [List of QURI Squiggle Models](https://github.com/quantified-uncertainty/squiggle-models) by Nuño Sempere, Sam Nolan, and Ozzie Gooen

packages/website/docs/Discussions/Three-Formats-Of-Distributions.md

@@ -5,17 +5,17 @@ author: Ozzie Gooen
 date: 02-19-2022
 ---
 
-Probability distributions have several subtle possible formats. Three important ones that we deal with in Squiggle are symbolic, sample set, and graph formats.
+Probability distributions have several subtle possible formats. Three important ones that we deal with in Squiggle are symbolic, sample set, and point set formats.
 
 _Symbolic_ formats are just the math equations. `normal(5,3)` is the symbolic representation of a normal distribution.
 
 When you sample distributions (usually starting with symbolic formats), you get lists of samples. Monte Carlo techniques return lists of samples. Let’s call this the “_Sample Set_” format.
 
-Lastly is what I’ll refer to as the _Graph_ format. It describes the coordinates, or the shape, of the distribution. You can save these formats in JSON, for instance, like, `{xs: [1, 2, 3, 4, …], ys: [.0001, .0003, .002, …]}`.
+Lastly is what I’ll refer to as the _Point Set_ format. It describes the coordinates, or the shape, of the distribution. You can save these formats in JSON, for instance, like, `{xs: [1, 2, 3, 4, …], ys: [.0001, .0003, .002, …]}`.
 
-Symbolic, Sample Set, and Graph formats all have very different advantages and disadvantages.
+Symbolic, Sample Set, and Point Set formats all have very different advantages and disadvantages.
 
-Note that the name "Symbolic" is fairly standard, but I haven't found common names for what I'm referring to as "Sample Set" and "Graph" formats. The formats aren't often specifically referred to for these purposes, from what I can tell.
+Note that the name "Symbolic" is fairly standard, but I haven't found common names for what I'm referring to as "Sample Set" and "Point Set" formats. The formats aren't often specifically referred to for these purposes, from what I can tell.
 
 ## Symbolic Formats
 
@@ -40,7 +40,7 @@ To perform calculations of symbolic systems, you need to find analytical solutio
 - It’s often either impossible or computationally infeasible to find analytical solutions to most symbolic equations.
 - Solving symbolic equations requires very specialized tooling that’s very rare. There are a few small symbolic solver libraries out there, but not many. Wolfram Research is the main group that seems very strong here, and their work is mostly closed source + expensive.
 
-**Converting to Graph Formats**
+**Converting to Point Set Formats**
 
 - Very easy. Choose X points such that you capture most of the distribution (you can set a threshold, like 99.9%). For each X point, calculate the pdf, and save as the Y points.
 
@@ -49,23 +49,23 @@ To perform calculations of symbolic systems, you need to find analytical solutio
 - Very easy. Just sample a bunch of times. The regular way is to randomly sample (This is trivial to do for all distributions with inverse-cdf functions.) If you want to get more fancy, you could provide extra samples from the tails, that would be weighted lower. Or, you could take samples in equal distances (of probability mass) along the entire distribution, then optionally shuffle it. (In the latter case, these would not be random samples, but sometimes that’s fine.)
 
 **How to Visualize**  
-Convert to graph, then display that. (Optionally, you can also convert to samples, then display those using a histogram, but this is often worse you have both options.)
+Convert to point set, then display that. (Optionally, you can also convert to samples, then display those using a histogram, but this is often worse you have both options.)
 
 **Bonus: The Metalog Distribution**
 
 The Metalog distribution seems like it can represent almost any reasonable distribution. It’s symbolic. This is great for storage, but it’s not clear if it helps with calculation. My impression is that we don’t have symbolic ways of doing most functions (addition, multiplication, etc) on metalog distributions. Also, note that it can take a fair bit of computation to fit a shape to the Metalog distribution.
 
-## Graph Formats
+## Point Set Formats
 
 **TL;DR**  
-Lists of the x-y coordinates of the shape of a distribution. (Usually the pdf, which is more compressed than the cdf). Some key functions (like pdf, cdf) and manipulations can work on almost any graphically-described distribution.
+Lists of the x-y coordinates of the shape of a distribution. (Usually the pdf, which is more compressed than the cdf). Some key functions (like pdf, cdf) and manipulations can work on almost any point set distribution.
 
 **Alternative Names:**  
 Grid, Mesh, Graph, Vector, Pdf, PdfCoords/PdfPoints, Discretised, Bezier, Curve  
 See [this facebook thread](https://www.facebook.com/ozzie.gooen/posts/10165936265785363?notif_id=1644937423623638&notif_t=feedback_reaction_generic&ref=notif).
 
 **How to Do Computation**  
-Use graph techniques. These can be fairly computationally-intensive (particularly finding integrals, which take a whole lot of adding). In the case that you want to multiply independent distributions, you can try convolution, but it’s pretty expensive.
+Use point set techniques. These can be fairly computationally-intensive (particularly finding integrals, which take a whole lot of adding). In the case that you want to multiply independent distributions, you can try convolution, but it’s pretty expensive.
 
 **Examples**  
 `{xs: [1, 2, 3, 4…], ys: [.0001, .0003, .002, .04, ...]} `  
@@ -74,18 +74,18 @@ Use graph techniques. These can be fairly computationally-intensive (particularl
 **Advantages**
 
 - Much more compressed than Sample List formats, but much less compressed than Symbolic formats.
-- Many functions (pdf, cdf, percentiles, mean, integration, etc) and manipulations (truncation, scaling horizontally or vertically), are possible on essentially all graph distributions.
+- Many functions (pdf, cdf, percentiles, mean, integration, etc) and manipulations (truncation, scaling horizontally or vertically), are possible on essentially all point set distributions.
 
 **Disadvantages**
 
-- Most calculations are infeasible/impossible to perform graphically. In these cases, you need to use sampling.
+- Most calculations are infeasible/impossible to perform using point sets formats. In these cases, you need to use sampling.
 - Not as accurate or fast as symbolic methods, where the symbolic methods are applicable.
-- The tails get cut off, which is subideal. It’s assumed that the value of the pdf outside of the bounded range is exactly 0, which is not correct. (Note: If you have ideas on how to store graph formats that don’t cut off tails, let me know)
+- The tails get cut off, which is subideal. It’s assumed that the value of the pdf outside of the bounded range is exactly 0, which is not correct. (Note: If you have ideas on how to store point set formats that don’t cut off tails, let me know)
 
 **Converting to Symbolic Formats**
 
 - Okay, if you are okay with a Metalog approximation or similar. Metaculus uses an additive combination of up to [Logistic distributions](https://www.metaculus.com/help/faq/); you could also fit this. Fitting takes a little time (it requires several attempts and some optimization), can be arbitrarily accurate.
-- If you want to be very fancy, you could try to fit graph distributions into normal / lognormal / etc. but this seems like a lot of work for little gain.
+- If you want to be very fancy, you could try to fit point set distributions into normal / lognormal / etc. but this seems like a lot of work for little gain.
 
 **Converting to Sample List Formats**
 

packages/website/docs/Guides/DistributionCreation.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Distribution Creation"
+title: "Distributions: Creation"
 sidebar_position: 2
 ---
 
@@ -35,8 +35,9 @@ If both values are above zero, a `lognormal` distribution is used. If not, a `no
     <SquiggleEditor defaultCode="-5 to -3" />
   </TabItem>
   <TabItem value="ex4" label="1 to 10000">
-    It's very easy to generate distributions with very long tails. If this
+    It's very easy to generate distributions with very long tails. These can be
-    happens, you can click the "log x scale" box to view this using a log scale.
+    impossible to see without changing view settings. (These settings are
+    available in the Playground, but not this smaller editor component)
     <SquiggleEditor defaultCode="1 to 10000" />
   </TabItem>
 </Tabs>
@@ -110,6 +111,11 @@ The `mixture` mixes combines multiple distributions to create a mixture. You can
   <p>
     In this case, I have a 20% chance of spending 0 time with it. I might estimate my hours with,
   </p>
+  <Admonition type="caution" title="Caution">
+    <p>
+      There's a temporary bug where the below render is compressed. If you toggle the code it will fix render correctly.
+    </p>
+  </Admonition>
   <SquiggleEditor
     defaultCode={`hours_the_project_will_take = 5 to 20
 chance_of_doing_anything = 0.8
@@ -143,8 +149,8 @@ Creates a [normal distribution](https://en.wikipedia.org/wiki/Normal_distributio
   <TabItem value="ex1" label="normal(5,1)" default>
     <SquiggleEditor defaultCode="normal(5, 1)" />
   </TabItem>
-  <TabItem value="ex2" label="normal(100000000000, 100000000000)">
+  <TabItem value="ex2" label="normal(1G, 1G)">
-    <SquiggleEditor defaultCode="normal(100000000000, 100000000000)" />
+    <SquiggleEditor defaultCode="normal(1G, 1G)" />
   </TabItem>
 </Tabs>
 
@@ -279,11 +285,8 @@ Creates a [beta distribution](https://en.wikipedia.org/wiki/Beta_distribution) w
   <TabItem value="ex5" label="beta(0.8, 0.8)">
     <SquiggleEditor defaultCode="beta(0.8, 0.8)" />
   </TabItem>
-  <TabItem
+  <TabItem value="ex6" label="beta({mean: 0.39, stdev: 0.1})">
-    value="from mean and standard deviation"
+    <SquiggleEditor defaultCode="beta({mean: 0.39, stdev: 0.1})" />
-    label="beta({mean: 0.39, stdev: 0.1})"
-  >
-    <SquiggleEditor initialSquiggleString="beta({mean: 0.39, stdev: 0.1})" />
   </TabItem>
 </Tabs>
 
@@ -343,7 +346,7 @@ Creates a [triangular distribution](https://en.wikipedia.org/wiki/Triangular_dis
 
 <SquiggleEditor defaultCode="triangular(1, 2, 4)" />
 
-## FromList
+## SampleSet.fromList
 
 `SampleSet.fromList(samples:number[])`
 
@@ -367,3 +370,41 @@ Creates a sample set distribution using an array of samples.
     specificity.
   </p>
 </Admonition>
+
+## PointSet.makeContinuous
+
+`PointSet.makeContinuous(points:{x: number, y: number})`
+
+Creates a continuous point set distribution using a list of points.
+
+<SquiggleEditor
+  defaultCode={`PointSet.makeContinuous([
+  { x: 0, y: 0.1 },
+  { x: 1, y: 0.2 },
+  { x: 2, y: 0.15 },
+  { x: 3, y: 0.1 }
+])`}
+/>
+
+### Arguments
+
+- `points`: An array of at least 3 coordinates.
+
+## PointSet.makeDiscrete
+
+`PointSet.makeDiscrete(points:{x: number, y: number})`
+
+Creates a discrete point set distribution using a list of points.
+
+<SquiggleEditor
+  defaultCode={`PointSet.makeDiscrete([
+  { x: 0, y: 0.1 },
+  { x: 1, y: 0.2 },
+  { x: 2, y: 0.15 },
+  { x: 3, y: 0.1 }
+])`}
+/>
+
+### Arguments
+
+- `points`: An array of at least 1 coordinate.

packages/website/docs/Guides/Functions.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Distribution Functions"
+title: "Distributions: Key Functions"
 sidebar_position: 3
 ---
 
@@ -181,7 +181,7 @@ The `sample(distribution)` samples a given distribution.
 
 Recall the [three formats of distributions](https://develop--squiggle-documentation.netlify.app/docs/Discussions/Three-Types-Of-Distributions). We can force any distribution into `SampleSet` format
 
-<SquiggleEditor defaultCode="PointSet.fromDist(normal(5, 10))" />
+<SquiggleEditor defaultCode="SampleSet.fromDist(normal(5, 10))" />
 
 Or `PointSet` format
 

packages/website/docs/Guides/Gotchas.mdx

@@ -0,0 +1,41 @@
+---
+title: Gotchas
+sidebar_position: 8
+---
+
+import { SquiggleEditor } from "../../src/components/SquiggleEditor";
+import Admonition from "@theme/Admonition";
+
+## Point Set Distributions Conversions
+
+Point Set conversions are done with [kernel density estimation](https://en.wikipedia.org/wiki/Kernel_density_estimation), which is lossy. This might be particularly noticeable in cases where distributions should be entirely above zero.
+
+In this example, we see that the median of this (highly skewed) distribution is positive when it's in a Sample Set format, but negative when it's converted to a Point Set format.
+
+<SquiggleEditor
+  defaultCode={`dist = SampleSet.fromDist(5 to 100000000)
+{
+    sampleSetMedian: quantile(dist, .5),
+    pointSetMedian: quantile(PointSet.fromDist(dist), .5),
+    dist: dist
+}`}
+/>
+
+---
+
+This can be particularly confusing for visualizations. Visualizations automatically convert distributions into Point Set formats. Therefore, they might often show negative values, even if the underlying distribution is fully positive.
+
+We plan to later support more configuration of kernel density estimation, and for visualiations of Sample Set distributions to instead use histograms.
+
+## Sample Set Correlations
+
+Correlations with Sample Set distributions are a bit complicated. Monte Carlo generations with Squiggle are ordered. The first sample in one Sample Set distribution will correspond to the first sample in a distribution that comes from a resulting Monte Carlo generation. Therefore, Sample Set distributions in a chain of Monte Carlo generations are likely to all be correlated with each other. This connection breaks if any node changes to the Point Set or Symbolic format.
+
+In this example, we subtract all three types of distributions by themselves. Notice that the Sample Set distribution returns 1. The other two return the result of subtracting one normal distribution from a separate uncorrelated distribution. These results are clearly very different to each other.
+
+<SquiggleEditor
+  defaultCode={`sampleSetDist = normal(5,2) |> SampleSet.fromDist
+sampleSetDistToPointSet = sampleSetDist |> PointSet.fromDist
+symbolicDist = normal(5,2)
+[sampleSetDist-sampleSetDist, sampleSetDistToPointSet-sampleSetDistToPointSet, symbolicDist-symbolicDist]`}
+/>

packages/website/docs/Guides/Language.mdx

@@ -5,50 +5,103 @@ title: Language Basics
 
 import { SquiggleEditor } from "../../src/components/SquiggleEditor";
 
-## Expressions
+Squiggle supports some simple types and language features.
 
-### Numbers
+## Numbers
 
 <SquiggleEditor defaultCode="4.32" />
 
-### Distributions
+## Distributions
+
+There are several ways of easily entering distributions. See the [documentation](/docs/Api/Dist/) on distributions for a complete API.
 
 <SquiggleEditor
   defaultCode={`a = normal(4,2)
 b = 30 to 50
 c = lognormal({mean:90, stdev: 7})
-d = mixture(a,b,c, [0.3, 0.3, .4])
+d = mixture(a,b,c, [.3, .3, .4])
-{a:a, b:b, c:c, d:d}`}
+d`}
 />
 
-### Lists
+## Lists
+
+Squiggle lists can accept items of any type, similar to those in Python. [API](/docs/Api/List).
 
 <SquiggleEditor
   defaultCode={`[beta(1,10), 4, isNormalized(SampleSet.fromDist(1 to 2))]`}
 />
 
-<SquiggleEditor defaultCode={`List.make(5,1)`} />
+## Dictionaries
 
-### Dictionaries
+Squiggle dictionaries work similarly to Python dictionaries. [API](/docs/Api/Dictionary).
 
 <SquiggleEditor
   defaultCode={`d = {dist: triangular(0, 1, 2), weight: 0.25}
 d.dist`}
 />
 
-### Functions
+## Functions
 
 <SquiggleEditor
   defaultCode={`f(t) = normal(t^2, t^1.2+.01)
 f`}
 />
 
-### Anonymous Functions
+## Anonymous Functions
 
 <SquiggleEditor defaultCode={`{|t| normal(t^2, t^1.2+.01)}`} />
 
-## See more
+## Comments
 
-- [Distribution creation](./DistributionCreation)
+<SquiggleEditor
-- [Functions reference](./Functions)
+  defaultCode={`// This is a single-line comment\n
-- [Gallery](../Discussions/Gallery)
+/*
+This is a multiple
+-line comment.
+*/
+""
+`}
+/>
+
+## Pipes
+
+Squiggle features [data-first](https://www.javierchavarri.com/data-first-and-data-last-a-comparison/) pipes. Functions in the standard library are organized to make this convenient.
+
+<SquiggleEditor
+  defaultCode={`normal(5,2) |> truncateLeft(3) |> SampleSet.fromDist |> SampleSet.map({|r| r + 10})`}
+/>
+
+## Standard Library
+
+Squiggle features a simple [standard libary](/docs/Api/Dist).
+
+Most functions are namespaced under their respective types to keep functionality distinct. Certain popular functions are usable without their namespaces.
+
+For example,
+
+<SquiggleEditor
+  defaultCode={`a = List.upTo(0, 5000) |> SampleSet.fromList // namespaces required
+b = normal(5,2) // namespace not required
+c = 5 to 10 // namespace not required
+""`}
+/>
+
+## Number Prefixes
+
+Numbers support a few scientific notation prefixes.
+
+| prefix | multiplier |
+| ------ | ---------- |
+| n      | 10^-9      |
+| m      | 10^-3      |
+| k      | 10^3       |
+| M      | 10^6       |
+| B,G    | 10^9       |
+| T      | 10^12      |
+| P      | 10^15      |
+
+<SquiggleEditor
+  defaultCode={`simpleNumber = 4.32k
+distribution = 40M to 50M
+distribution`}
+/>

packages/website/docs/Integrations.md

@@ -0,0 +1,34 @@
+---
+sidebar_position: 4
+title: "Integrations"
+---
+
+## Node Packages
+
+There are two JavaScript packages currently available for Squiggle:
+
+- [`@quri/squiggle-lang`](https://www.npmjs.com/package/@quri/squiggle-lang)
+- [`@quri/squiggle-components`](https://www.npmjs.com/package/@quri/squiggle-components)
+
+Types are available for both packages.
+
+## [Squiggle Language](https://www.npmjs.com/package/@quri/squiggle-lang) ![npm version](https://badge.fury.io/js/@quri%2Fsquiggle-lang.svg)
+
+[_See `README.md` in Github_](https://github.com/quantified-uncertainty/squiggle/tree/develop/packages/squiggle-lang#use-the-npm-package)
+
+## [Squiggle Components](https://www.npmjs.com/package/@quri/squiggle-components) ![npm version](https://badge.fury.io/js/@quri%2Fsquiggle-components.svg)
+
+[_See `README.md` in Github_](https://github.com/quantified-uncertainty/squiggle/tree/develop/packages/components#usage-in-a-react-project)
+
+This documentation uses `@quri/squiggle-components` frequently.
+
+We host [a storybook](https://squiggle-components.netlify.app/) with details
+and usage of each of the components made available.
+
+## [Visual Studio Code Extension](https://marketplace.visualstudio.com/items?itemName=QURI.vscode-squiggle) ![npm version](https://vsmarketplacebadge.apphb.com/version/QURI.vscode-squiggle.svg)
+
+This extention allows you to run and visualize Squiggle code.
+
+## [Observable Library](https://observablehq.com/@hazelfire/squiggle)
+
+An exportable [Observable Notebook](https://observablehq.com/@hazelfire/squiggle) of the key components that you can directly import and use in Observable notebooks.

packages/website/docs/Introduction.md

@@ -1,30 +0,0 @@
----
-sidebar_position: 1
-title: Introduction
----
-
-Squiggle is a simple programming language for intuitive probabilistic estimation. It's meant for quantitative forecasting and evaluations.
-
-The basics of Squiggle can be pretty simple and intuitive. The more advanced functionality can take some time to learn.
-
-## What Squiggle Is
-
-- A simple programming language for doing math with probability distributions
-- An embeddable language that can be used in Javascript applications
-- A tool to embed functions as forecasts that can be embedded in other applications
-
-## What Squiggle Is Not
-
-- A complete replacement for enterprise Risk Analysis tools (See Crystal Ball, @Risk, Lumina Analytica)
-- A Probabilistic Programming Language with backwards inference and sophisticated sampling algorithms. (See [PPLs](https://en.wikipedia.org/wiki/Probabilistic_programming))
-- A visual tool aimed at casual users (see Guesstimate, Causal)
-
-## Get started
-
-- [Gallery](./Discussions/Gallery)
-- [Squiggle playground](/playground)
-- [Language basics](./Guides/Language)
-- [Squiggle functions source of truth](./Guides/Functions)
-- [Known bugs](./Discussions/Bugs)
-- [Original lesswrong sequence](https://www.lesswrong.com/s/rDe8QE5NvXcZYzgZ3)
-- [Author your squiggle models as Observable notebooks](https://observablehq.com/@hazelfire/squiggle)

packages/website/docs/Node-Packages.md

@@ -1,24 +0,0 @@
----
-sidebar_position: 4
-title: Node Packages
----
-
-There are two JavaScript packages currently available for Squiggle:
-
-- [`@quri/squiggle-lang`](https://www.npmjs.com/package/@quri/squiggle-lang) ![npm version](https://badge.fury.io/js/@quri%2Fsquiggle-lang.svg)
-- [`@quri/squiggle-components`](https://www.npmjs.com/package/@quri/squiggle-components) ![npm version](https://badge.fury.io/js/@quri%2Fsquiggle-components.svg)
-
-Types are available for both packages.
-
-## Squiggle Language
-
-[_See `README.md` in Github_](https://github.com/quantified-uncertainty/squiggle/tree/develop/packages/squiggle-lang#use-the-npm-package)
-
-## Squiggle Components
-
-[_See `README.md` in Github_](https://github.com/quantified-uncertainty/squiggle/tree/develop/packages/components#usage-in-a-react-project)
-
-This documentation uses `@quri/squiggle-components` frequently.
-
-We host [a storybook](https://squiggle-components.netlify.app/) with details
-and usage of each of the components made available.

packages/website/docs/Overview.mdx

@@ -0,0 +1,119 @@
+---
+sidebar_position: 1
+title: Overview
+---
+
+import { SquiggleEditor } from "../src/components/SquiggleEditor";
+
+Squiggle is a minimalist programming language for probabilistic estimation. It's meant for intuitively-driven quantitative estimation instead of data analysis or data-driven statistical techniques.
+
+The basics of Squiggle are fairly straightforward. This can be enough for many models. The more advanced functionality can take some time to learn.
+
+## Simple example
+
+Say you're trying to estimate the number of piano tuners in New York City. You can build a simple model of this, like so.  
+(Tip: This is interactive! Feel free to modify the code directly.)
+
+<SquiggleEditor
+  defaultCode={`
+// Piano tuners in NYC over the next 5 years
+populationOfNewYork2022 = 8.1M to 8.4M // This means that you're 90% confident the value is between 8.1 and 8.4 Million.\n
+proportionOfPopulationWithPianos = {
+    percentage = (.2 to 1)
+    percentage * 0.01
+} // We assume there are almost no people with multiple pianos\n
+pianoTunersPerPiano = {
+    pianosPerPianoTuner = 2k to 50k // This is artificially narrow, to help graphics later
+    1 / pianosPerPianoTuner
+} \n
+totalTunersIn2022 = populationOfNewYork2022 * proportionOfPopulationWithPianos * pianoTunersPerPiano
+totalTunersIn2022
+`}
+/>
+
+---
+
+Now let's take this a bit further. Let's imagine that you think that NYC will grow over time, and you'd like to estimate the number of piano tuners for every point in time for the next few years.
+
+<SquiggleEditor
+  defaultCode={`// Piano tuners in NYC over the next 5 years
+populationOfNewYork2022 = 8.1M to 8.4M\n
+proportionOfPopulationWithPianos = {
+    percentage = (.2 to 1)
+    percentage * 0.01
+} // We assume there are almost no people with multiple pianos\n
+pianoTunersPerPiano = {
+    pianosPerPianoTuner = 2k to 50k // This is artificially narrow, to help graphics later
+    1 / pianosPerPianoTuner
+} \n
+//Time in years after 2022
+populationAtTime(t) = {
+    averageYearlyPercentageChange = -0.01 to 0.05 // We're expecting NYC to continuously grow with an mean of roughly between -1% and +4% per year
+    populationOfNewYork2022 * ((averageYearlyPercentageChange + 1) ^ t)
+}\n
+median(v) = quantile(v, .5)
+totalTunersAtTime(t) = populationAtTime(t) * proportionOfPopulationWithPianos * pianoTunersPerPiano\n
+{
+    populationAtTime: populationAtTime,
+    totalTunersAtTimeMedian: {|t| median(totalTunersAtTime(t))}
+}`}
+/>
+
+If you haven't noticed yet, you can hover over the `populationAtTime` graph to see the distribution of population at different points in time.
+
+## Using Squiggle
+
+You can currently interact with Squiggle in a few ways:
+
+**[Playground](/playground)**  
+The [Squiggle Playground](/playground) is a nice tool for working with small models and making prototypes. You can make simple shareable links, but you can't save models that change over time.
+
+**[Visual Studio Code Extension](https://marketplace.visualstudio.com/items?itemName=QURI.vscode-squiggle)**  
+There's a simple [VS Code extension](https://marketplace.visualstudio.com/items?itemName=QURI.vscode-squiggle) for running and visualizing Squiggle code. We find that VS Code is a useful editor for managing larger Squiggle setups.
+
+**[Typescript Library](https://www.npmjs.com/package/@quri/squiggle-lang)**  
+Squiggle is built using [Rescript](https://rescript-lang.org/), and is accessible via a simple Typescript library. You can use this library to either run Squiggle code in full, or to call select specific functions within Squiggle (though this latter functionality is very minimal).
+
+**[React Components Library](https://www.npmjs.com/package/@quri/squiggle-components)**  
+All of the components used in the playground and documentation are available in a separate component NPM repo. You can see the full Storybook of components [here](https://squiggle-components.netlify.app).
+
+**[Observable](https://observablehq.com/@hazelfire/squiggle)**  
+You can use Squiggle Components in Observable notebooks. Sam Nolan put together an exportable [Observable Notebook](https://observablehq.com/@hazelfire/squiggle) of the key components that you can directly import and use in your Observable notebooks.
+
+## Squiggle Vs. Other Tools
+
+### What Squiggle Is
+
+- A simple programming language for doing math with probability distributions.
+- An embeddable language that can be used in Javascript applications.
+- A tool to encode functions as forecasts that can be embedded in other applications.
+
+### What Squiggle Is Not
+
+- A complete replacement for enterprise Risk Analysis tools. (See [Crystal Ball](https://www.oracle.com/applications/crystalball/), [@Risk](https://www.palisade.com/risk/), [Lumina Analytica](https://lumina.com/))
+- A [probabilistic programming language](https://en.wikipedia.org/wiki/Probabilistic_programming). Squiggle does not support Bayesian inference.
+- A tool for substantial data analysis. (See programming languages like [Python](https://www.python.org/) or [Julia](https://julialang.org/))
+- A programming language for anything other than estimation.
+- A visually-driven tool. (See [Guesstimate](https://www.getguesstimate.com/) and [Causal](https://causal.app/))
+
+### Strengths
+
+- Simple and readable syntax, especially for dealing with probabilistic math.
+- Fast for relatively small models. Strong for rapid prototyping.
+- Optimized for using some numeric and symbolic approaches, not just Monte Carlo.
+- Embeddable in Javascript.
+- Free and open-source.
+
+### Weaknesses
+
+- Limited scientific capabilities.
+- Much slower than serious probabilistic programming languages on sizeable models.
+- Can't do Bayesian backwards inference.
+- Essentially no support for libraries or modules (yet).
+- Still very new, so a tiny ecosystem.
+- Still very new, so there are likely math bugs.
+- Generally not as easy to use as Guesstimate or Causal, especially for non programmers.
+
+## Organization
+
+Squiggle is one of the main projects of [The Quantified Uncertainty Research Institute](https://quantifieduncertainty.org/). QURI is a nonprofit funded primarily by [Effective Altruist](https://www.effectivealtruism.org/) donors.

packages/website/docusaurus.config.js

@@ -57,18 +57,18 @@ const config = {
         hideOnScroll: true,
         logo: {
           alt: "Squiggle Logo",
-          src: "img/quri-logo.png",
+          src: "img/squiggle-logo.png",
         },
         items: [
           {
             type: "doc",
-            docId: "Introduction",
+            docId: "Overview",
             position: "left",
             label: "Documentation",
           },
           {
             type: "doc",
-            docId: "Api/DistGeneric",
+            docId: "Api/Dist",
             position: "left",
             label: "API",
           },
@@ -84,6 +84,11 @@ const config = {
             label: "GitHub",
             position: "right",
           },
+          {
+            href: "https://quantifieduncertainty.org/",
+            label: "QURI",
+            position: "right",
+          },
         ],
       },
       footer: {

packages/website/sidebars.js

@@ -23,13 +23,13 @@ const sidebars = {
   tutorialSidebar: [
     {
       type: "doc",
-      id: "Introduction",
+      id: "Overview",
-      label: "Introduction",
+      label: "Overview",
     },
     {
       type: "doc",
-      id: "Node-Packages",
+      id: "Integrations",
-      label: "Node Packages",
+      label: "Integrations",
     },
     {
       type: "category",

packages/website/src/css/custom.css

@@ -60,7 +60,7 @@ html[data-theme="dark"] .docusaurus-highlight-code-line {
 }
 
 .hero__subtitle2 {
-  color: #ba3e3e;
+  color: #777;
   font-size: 1.5em;
   font-family: "Lora";
   font-weight: 500;
@@ -88,6 +88,10 @@ h2 {
   font-weight: 700;
 }
 
+.navbar__logo {
+  height: 1.5rem;
+}
+
 :root {
   /* --ifm-font-family-base: 'Lora'; */
 }

packages/website/src/pages/index.js

@@ -1,7 +1,6 @@
 import React from "react";
 import clsx from "clsx";
 import Layout from "@theme/Layout";
-import Link from "@docusaurus/Link";
 import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
 import styles from "./index.module.css";
 import HomepageFeatures from "../components/HomepageFeatures";
@@ -11,6 +10,12 @@ function HomepageHeader() {
   return (
     <header className={clsx("hero hero--primary", styles.heroBanner)}>
       <div className="container">
+        <img
+          alt={"Docusaurus with Keytar"}
+          className={styles.heroLogo}
+          src={"/img/squiggle-logo.png"}
+          width="70"
+        />
         <h1 className="hero__title">{siteConfig.title}</h1>
         <p className="hero__subtitle">Early Access</p>
         <p className="hero__subtitle2">{siteConfig.tagline}</p>