Merge pull request #66 from foretold-app/various-enhancements-July-2020

Various enhancements July 2020

showcase/entries/Continuous2.re

@@ -19,8 +19,9 @@ let timeDist ={
 let setup = dist =>
   RenderTypes.DistPlusRenderer.make(~distPlusIngredients=dist,())
   |> DistPlusRenderer.run
-  |> RenderTypes.DistPlusRenderer.Outputs.distplus
+  |> E.R.fmap(distPlus => <DistPlusPlot distPlus />)
-  |> R.O.fmapOrNull(distPlus => <DistPlusPlot distPlus />);
+  |> E.R.toOption
+  |> E.O.toExn("")
 
 let simpleExample = (name, guesstimatorString) =>
   <>

showcase/entries/ExpressionTreeExamples.re

@@ -1,8 +1,9 @@
 let setup = dist =>
   RenderTypes.DistPlusRenderer.make(~distPlusIngredients=dist, ())
   |> DistPlusRenderer.run
-  |> RenderTypes.DistPlusRenderer.Outputs.distplus
+  |> E.R.fmap(distPlus => <DistPlusPlot distPlus />)
-  |> R.O.fmapOrNull(distPlus => <DistPlusPlot distPlus />);
+  |> E.R.toOption
+  |> E.O.toExn("")
 
 let simpleExample = (guesstimatorString, ~problem="", ()) =>
   <>

src/components/DistBuilder.re

@@ -142,18 +142,15 @@ module DemoDist = {
                },
                ~distPlusIngredients,
                ~shouldDownsample=options.downsampleTo |> E.O.isSome,
-               ~recommendedLength=options.downsampleTo |> E.O.default(100),
+               ~recommendedLength=options.downsampleTo |> E.O.default(1000),
                (),
              );
            let response = DistPlusRenderer.run(inputs);
-           switch (RenderTypes.DistPlusRenderer.Outputs.distplus(response)) {
+           switch (response) {
-           | Some(distPlus) => {
+           | Ok(distPlus) =>
              let normalizedDistPlus = DistPlus.T.normalize(distPlus);
-             <DistPlusPlot distPlus={normalizedDistPlus} />;
+             <DistPlusPlot distPlus=normalizedDistPlus />;
-           }
+           | Error(r) => r |> R.ste
-           | _ =>
-             "Correct Guesstimator string input to show a distribution."
-             |> R.ste
            };
          | _ =>
            "Nothing to show. Try to change the distribution description."
@@ -183,10 +180,10 @@ let make = () => {
         unitType: "UnspecifiedDistribution",
         zero: MomentRe.momentNow(),
         unit: "days",
-        sampleCount: "3000",
+        sampleCount: "30000",
-        outputXYPoints: "100",
+        outputXYPoints: "1000",
-        downsampleTo: "100",
+        downsampleTo: "",
-        kernelWidth: "5",
+        kernelWidth: "",
       },
       (),
     );
@@ -484,7 +481,10 @@ let make = () => {
               />
             </Col>
             <Col span=4>
-              <FieldFloat field=FormConfig.DownsampleTo label="Downsample To" />
+              <FieldFloat
+                field=FormConfig.DownsampleTo
+                label="Downsample To"
+              />
             </Col>
             <Col span=4>
               <FieldFloat field=FormConfig.KernelWidth label="Kernel Width" />

src/components/Drawer.re

@@ -395,7 +395,7 @@ module Draw = {
         numSamples,
         {sampleCount: 10000, outputXYPoints: 10000, kernelWidth: None},
         `SymbolicDist(normal),
-      );
+      ) |> E.R.toExn;
     let xyShape: Types.xyShape =
       switch (normalShape) {
       | Mixed(_) => {xs: [||], ys: [||]}

src/distPlus/distribution/Continuous.re

@@ -78,7 +78,7 @@ let combinePointwise =
   make(
     ~integralSumCache=combinedIntegralSum,
     XYShape.PointwiseCombination.combine(
-      (+.),
+      fn,
       interpolator,
       t1.xyShape,
       t2.xyShape,

src/distPlus/expressionTree/ExpressionTree.re

@@ -10,14 +10,9 @@ let toShape = (intendedShapeLength: int, samplingInputs, node: node) => {
        });
 
   switch (renderResult) {
-  | Ok(`RenderedDist(rs)) =>
+  | Ok(`RenderedDist(shape)) => Ok(shape)
-    // todo: Why is this here? It converts a mixed shape to a mixed shape.
+  | Ok(_) => Error("Rendering failed.")
-    let continuous = Shape.T.toContinuous(rs);
+  | Error(e) => Error(e)
-    let discrete = Shape.T.toDiscrete(rs);
-    let shape = MixedShapeBuilder.buildSimple(~continuous, ~discrete);
-    shape |> E.O.toExt("Could not build final shape.");
-  | Ok(_) => E.O.toExn("Rendering failed.", None)
-  | Error(message) => E.O.toExn("No shape found, error: " ++ message, None)
   };
 };
 

src/distPlus/expressionTree/ExpressionTreeEvaluator.re

@@ -32,6 +32,47 @@ module AlgebraicCombination = {
        );
   };
 
+  let nodeScore: node => int =
+    fun
+    | `SymbolicDist(`Float(_)) => 1
+    | `SymbolicDist(_) => 1000
+    | `RenderedDist(Discrete(m)) => m.xyShape |> XYShape.T.length
+    | `RenderedDist(Mixed(_)) => 1000
+    | `RenderedDist(Continuous(_)) => 1000
+    | _ => 1000;
+
+  let choose = (t1: node, t2: node) => {
+    nodeScore(t1) * nodeScore(t2) > 10000 ? `Sampling : `Analytical;
+  };
+
+  let combine =
+      (evaluationParams, algebraicOp, t1: node, t2: node)
+      : result(node, string) => {
+    E.R.merge(
+      SamplingDistribution.renderIfIsNotSamplingDistribution(
+        evaluationParams,
+        t1,
+      ),
+      SamplingDistribution.renderIfIsNotSamplingDistribution(
+        evaluationParams,
+        t2,
+      ),
+    )
+    |> E.R.bind(_, ((a, b)) =>
+         switch (choose(a, b)) {
+         | `Sampling =>
+           SamplingDistribution.combineShapesUsingSampling(
+             evaluationParams,
+             algebraicOp,
+             a,
+             b,
+           )
+         | `Analytical =>
+           combinationByRendering(evaluationParams, algebraicOp, a, b)
+         }
+       );
+  };
+
   let operationToLeaf =
       (
         evaluationParams: evaluationParams,
@@ -45,8 +86,8 @@ module AlgebraicCombination = {
     |> E.R.bind(
          _,
          fun
-         | `SymbolicDist(d) as t => Ok(t)
+         | `SymbolicDist(_) as t => Ok(t)
-         | _ => combinationByRendering(evaluationParams, algebraicOp, t1, t2),
+         | _ => combine(evaluationParams, algebraicOp, t1, t2),
        );
 };
 
@@ -79,7 +120,10 @@ module VerticalScaling = {
 
 module PointwiseCombination = {
   let pointwiseAdd = (evaluationParams: evaluationParams, t1: t, t2: t) => {
-    switch (Render.render(evaluationParams, t1), Render.render(evaluationParams, t2)) {
+    switch (
+      Render.render(evaluationParams, t1),
+      Render.render(evaluationParams, t2),
+    ) {
     | (Ok(`RenderedDist(rs1)), Ok(`RenderedDist(rs2))) =>
       Ok(
         `RenderedDist(
@@ -110,9 +154,17 @@ module PointwiseCombination = {
   let pointwiseMultiply = (evaluationParams: evaluationParams, t1: t, t2: t) => {
     // TODO: construct a function that we can easily sample from, to construct
     // a RenderedDist. Use the xMin and xMax of the rendered shapes to tell the sampling function where to look.
-    Error(
+    // TODO: This should work for symbolic distributions too!
-      "Pointwise multiplication not yet supported.",
+    switch (
-    );
+      Render.render(evaluationParams, t1),
+      Render.render(evaluationParams, t2),
+    ) {
+    | (Ok(`RenderedDist(rs1)), Ok(`RenderedDist(rs2))) =>
+      Ok(`RenderedDist(Shape.combinePointwise(( *. ), rs1, rs2)))
+    | (Error(e1), _) => Error(e1)
+    | (_, Error(e2)) => Error(e2)
+    | _ => Error("Pointwise combination: rendering failed.")
+    };
   };
 
   let operationToLeaf =
@@ -133,8 +185,10 @@ module Truncate = {
   let trySimplification = (leftCutoff, rightCutoff, t): simplificationResult => {
     switch (leftCutoff, rightCutoff, t) {
     | (None, None, t) => `Solution(t)
-    | (Some(lc), Some(rc), t) when lc > rc =>
+    | (Some(lc), Some(rc), _) when lc > rc =>
-      `Error("Left truncation bound must be smaller than right truncation bound.")
+      `Error(
+        "Left truncation bound must be smaller than right truncation bound.",
+      )
     | (lc, rc, `SymbolicDist(`Uniform(u))) =>
       `Solution(
         `SymbolicDist(`Uniform(SymbolicDist.Uniform.truncate(lc, rc, u))),

src/distPlus/expressionTree/MathJsParser.re

@@ -15,12 +15,8 @@ module MathJsonToMathJsAdt = {
       switch (field("mathjs", string, j)) {
       | "FunctionNode" =>
         let args = j |> field("args", array(run));
-        Some(
+        let name = j |> optional(field("fn", field("name", string)));
-          Fn({
+        name |> E.O.fmap(name => Fn({name, args: args |> E.A.O.concatSomes}));
-            name: j |> field("fn", field("name", string)),
-            args: args |> E.A.O.concatSomes,
-          }),
-        );
       | "OperatorNode" =>
         let args = j |> field("args", array(run));
         Some(
@@ -240,6 +236,7 @@ module MathAdtToDistDst = {
         args: array(result(ExpressionTypes.ExpressionTree.node, string)),
       ) => {
     let toOkAlgebraic = r => Ok(`AlgebraicCombination(r));
+    let toOkPointwise = r => Ok(`PointwiseCombination(r));
     let toOkTruncate = r => Ok(`Truncate(r));
     let toOkFloatFromDist = r => Ok(`FloatFromDist(r));
     switch (name, args) {
@@ -249,6 +246,11 @@ module MathAdtToDistDst = {
     | ("subtract", _) => Error("Subtraction needs two operands")
     | ("multiply", [|Ok(l), Ok(r)|]) => toOkAlgebraic((`Multiply, l, r))
     | ("multiply", _) => Error("Multiplication needs two operands")
+    | ("dotMultiply", [|Ok(l), Ok(r)|]) => toOkPointwise((`Multiply, l, r))
+    | ("dotMultiply", _) =>
+      Error("Dotwise multiplication needs two operands")
+    | ("rightLogShift", [|Ok(l), Ok(r)|]) => toOkPointwise((`Add, l, r))
+    | ("rightLogShift", _) => Error("Dotwise addition needs two operands")
     | ("divide", [|Ok(l), Ok(r)|]) => toOkAlgebraic((`Divide, l, r))
     | ("divide", _) => Error("Division needs two operands")
     | ("pow", _) => Error("Exponentiation is not yet supported.")
@@ -324,6 +326,8 @@ module MathAdtToDistDst = {
     | "add"
     | "subtract"
     | "multiply"
+    | "dotMultiply"
+    | "rightLogShift"
     | "divide"
     | "pow"
     | "leftTruncate"
@@ -358,6 +362,13 @@ module MathAdtToDistDst = {
     r |> MathAdtCleaner.run |> topLevel;
 };
 
+/* The MathJs parser doesn't support '.+' syntax, but we want it because it
+   would make sense with '.*'. Our workaround is to change this to >>>, which is
+   logShift in mathJS. We don't expect to use logShift anytime soon, so this tradeoff
+   seems fine.
+   */
+let pointwiseToRightLogShift = Js.String.replaceByRe([%re "/\.\+/g"], ">>>");
+
 let fromString = str => {
   /* We feed the user-typed string into Mathjs.parseMath,
        which returns a JSON with (hopefully) a single-element array.
@@ -367,7 +378,7 @@ let fromString = str => {
        The function MathJsonToMathJsAdt then recursively unpacks this JSON into a typed data structure we can use.
        Inside of this function, MathAdtToDistDst is called whenever a distribution function is encountered.
      */
-  let mathJsToJson = Mathjs.parseMath(str);
+  let mathJsToJson = str |> pointwiseToRightLogShift |> Mathjs.parseMath;
   let mathJsParse =
     E.R.bind(mathJsToJson, r => {
       switch (MathJsonToMathJsAdt.run(r)) {

src/distPlus/expressionTree/SamplingDistribution.re

@@ -6,7 +6,7 @@ let isSamplingDistribution: node => bool =
   | `RenderedDist(_) => true
   | _ => false;
 
-let renderIfIsNotSamplingDistribution = (params, t) =>
+let renderIfIsNotSamplingDistribution = (params, t): result(node, string) =>
   !isSamplingDistribution(t)
     ? switch (Render.render(params, t)) {
       | Ok(r) => Ok(r)
@@ -70,9 +70,7 @@ let combineShapesUsingSampling =
                   },
                 ),
               )
-           |> E.O.bind(_, (r: RenderTypes.ShapeRenderer.Sampling.outputs) =>
+           |> E.O.bind(_, (r) => r.shape)
-                r.shape
-              )
            |> E.O.toResult("No response");
          shape |> E.R.fmap(r => `Normalize(`RenderedDist(r)));
        },

src/distPlus/renderers/DistPlusRenderer.re

@@ -1,18 +1,4 @@
-let downsampleIfShould =
+let run = (inputs: RenderTypes.DistPlusRenderer.inputs) => {
-    (
-      {recommendedLength, shouldDownsample}: RenderTypes.DistPlusRenderer.inputs,
-      outputs: RenderTypes.ShapeRenderer.Combined.outputs,
-      dist,
-    ) => {
-      let willDownsample =
-  shouldDownsample
-  && RenderTypes.ShapeRenderer.Combined.methodUsed(outputs) == `Sampling;
-    willDownsample ? dist |> DistPlus.T.downsample(recommendedLength) : dist;
-};
-
-let run =
-    (inputs: RenderTypes.DistPlusRenderer.inputs)
-    : RenderTypes.DistPlusRenderer.outputs => {
   let toDist = shape =>
     DistPlus.make(
       ~shape,
@@ -22,7 +8,7 @@ let run =
       (),
     )
     |> DistPlus.T.normalize;
-  let outputs =
+  let output =
     ShapeRenderer.run({
       samplingInputs: inputs.samplingInputs,
       guesstimatorString: inputs.distPlusIngredients.guesstimatorString,
@@ -30,8 +16,8 @@ let run =
         length: inputs.recommendedLength,
       },
     });
-  let shape = outputs |> RenderTypes.ShapeRenderer.Combined.getShape;
+  output
-  let dist =
+  |> E.R.fmap((o: RenderTypes.ShapeRenderer.Symbolic.outputs) =>
-    shape |> E.O.fmap(toDist) |> E.O.fmap(downsampleIfShould(inputs, outputs));
+       toDist(o.shape)
-  RenderTypes.DistPlusRenderer.Outputs.make(outputs, dist);
+     );
 };

src/distPlus/renderers/ShapeRenderer.re

@@ -18,9 +18,7 @@ let runSymbolic = (inputs: RenderTypes.ShapeRenderer.Combined.inputs) => {
   let str = formatString(inputs.guesstimatorString);
   let graph = MathJsParser.fromString(str);
   graph
-  |> E.R.fmap(g =>
+  |> E.R.bind(_, g =>
-       RenderTypes.ShapeRenderer.Symbolic.make(
-         g,
        ExpressionTree.toShape(
          inputs.symbolicInputs.length,
          {
@@ -31,14 +29,11 @@ let runSymbolic = (inputs: RenderTypes.ShapeRenderer.Combined.inputs) => {
            kernelWidth: inputs.samplingInputs.kernelWidth,
          },
          g,
-         ),
        )
+       |> E.R.fmap(RenderTypes.ShapeRenderer.Symbolic.make(g))
      );
 };
 
-let run =
+let run = (inputs: RenderTypes.ShapeRenderer.Combined.inputs) => {
-    (inputs: RenderTypes.ShapeRenderer.Combined.inputs)
+  runSymbolic(inputs);
-    : RenderTypes.ShapeRenderer.Combined.outputs => {
-  let symbolic = runSymbolic(inputs);
-  {symbolic: Some(symbolic), sampling: None};
 };

src/distPlus/renderers/samplesRenderer/Samples.re

@@ -1,3 +1,42 @@
+module Types = {
+  let defaultSampleCount = 5000;
+  let defaultOutputXYPoints = 10000;
+
+  type inputs = {
+    sampleCount: option(int),
+    outputXYPoints: option(int),
+    kernelWidth: option(float),
+  };
+
+  type samplingStats = {
+    sampleCount: int,
+    outputXYPoints: int,
+    bandwidthXSuggested: float,
+    bandwidthUnitSuggested: float,
+    bandwidthXImplemented: float,
+    bandwidthUnitImplemented: float,
+  };
+
+  type outputs = {
+    continuousParseParams: option(samplingStats),
+    shape: option(DistTypes.shape),
+  };
+
+  let empty = {sampleCount: None, outputXYPoints: None, kernelWidth: None};
+
+  type fInputs = {
+    sampleCount: int,
+    outputXYPoints: int,
+    kernelWidth: option(float),
+  };
+
+  let toF = (i: inputs): fInputs => {
+    sampleCount: i.sampleCount |> E.O.default(defaultSampleCount),
+    outputXYPoints: i.outputXYPoints |> E.O.default(defaultOutputXYPoints),
+    kernelWidth: i.kernelWidth,
+  };
+};
+
 module JS = {
   [@bs.deriving abstract]
   type distJs = {
@@ -21,39 +60,6 @@ module KDE = {
     |> JS.samplesToContinuousPdf(_, outputXYPoints, kernelWidth)
     |> JS.jsToDist;
   };
-
-  // Note: This was an experiment, but it didn't actually work that well.
-  let inGroups = (samples, outputXYPoints, kernelWidth, ~cuttoff=0.9, ()) => {
-    let partitionAt =
-      samples
-      |> E.A.length
-      |> float_of_int
-      |> (e => e *. cuttoff)
-      |> int_of_float;
-    let part1XYPoints =
-      outputXYPoints |> float_of_int |> (e => e *. cuttoff) |> int_of_float;
-    let part2XYPoints = outputXYPoints - part1XYPoints |> Js.Math.max_int(30);
-    let part1Data =
-      samples |> Belt.Array.slice(_, ~offset=0, ~len=partitionAt);
-    let part2DataLength = (samples |> E.A.length) - partitionAt;
-    let part2Data =
-      samples
-      |> Belt.Array.slice(
-           _,
-           ~offset=(-1) * part2DataLength,
-           ~len=part2DataLength,
-         );
-    let part1 =
-      part1Data
-      |> JS.samplesToContinuousPdf(_, part1XYPoints, kernelWidth)
-      |> JS.jsToDist;
-    let part2 =
-      part2Data
-      |> JS.samplesToContinuousPdf(_, part2XYPoints, 3)
-      |> JS.jsToDist;
-    let opp = 1.0 -. cuttoff;
-    part1;
-  };
 };
 
 module T = {
@@ -109,7 +115,7 @@ module T = {
   let toShape =
       (
         ~samples: t,
-        ~samplingInputs: RenderTypes.ShapeRenderer.Sampling.Inputs.fInputs,
+        ~samplingInputs: Types.fInputs,
         (),
       ) => {
     Array.fast_sort(compare, samples);
@@ -126,6 +132,7 @@ module T = {
       continuousPart |> E.A.length > 5
         ? {
           let _suggestedXWidth = Bandwidth.nrd0(continuousPart);
+          // todo: This does some recalculating from the last step.
           let _suggestedUnitWidth =
             suggestedUnitWidth(continuousPart, samplingInputs.outputXYPoints);
           let usedWidth =
@@ -136,7 +143,7 @@ module T = {
               samplingInputs.outputXYPoints,
               usedWidth,
             );
-          let foo: RenderTypes.ShapeRenderer.Sampling.samplingStats = {
+          let foo: Types.samplingStats = {
             sampleCount: samplingInputs.sampleCount,
             outputXYPoints: samplingInputs.outputXYPoints,
             bandwidthXSuggested: _suggestedXWidth,
@@ -159,7 +166,7 @@ module T = {
         ~continuous=pdf |> E.O.fmap(fst),
         ~discrete=Some(discrete),
       );
-    let samplesParse: RenderTypes.ShapeRenderer.Sampling.outputs = {
+    let samplesParse: Types.outputs = {
       continuousParseParams: pdf |> E.O.fmap(snd),
       shape,
     };
@@ -168,11 +175,11 @@ module T = {
 
   let fromSamples =
       (
-        ~samplingInputs=RenderTypes.ShapeRenderer.Sampling.Inputs.empty,
+        ~samplingInputs=Types.empty,
         samples,
       ) => {
     let samplingInputs =
-      RenderTypes.ShapeRenderer.Sampling.Inputs.toF(samplingInputs);
+      Types.toF(samplingInputs);
     toShape(~samples, ~samplingInputs, ());
   };
 };

src/distPlus/utility/E.re

@@ -145,6 +145,7 @@ module R = {
   let id = e => e |> result(U.id, U.id);
   let fmap = Rationale.Result.fmap;
   let bind = Rationale.Result.bind;
+  let toExn = Belt.Result.getExn;
   let merge = (a, b) =>
     switch (a, b) {
     | (Error(e), _) => Error(e)

src/interface/FormBuilder.re

@@ -25,11 +25,9 @@ let propValue = (t: Prop.Value.t) => {
         ~shouldDownsample=true,
         (),
       )
-      |> DistPlusRenderer.run
+      |> DistPlusRenderer.run;
-      |> RenderTypes.DistPlusRenderer.Outputs.distplus;
     switch (newDistribution) {
-    | Some(distribution) =>
+    | Ok(distribution) => <div> <DistPlusPlot distPlus=distribution /> </div>
-      <div> <DistPlusPlot distPlus=distribution /> </div>
     // <input
     //   readOnly=true
     //   className="shadow appearance-none border w-1/3 rounded py-2 px-3 text-gray-700 leading-tight focus:outline-none focus:shadow-outline"
@@ -45,7 +43,7 @@ let propValue = (t: Prop.Value.t) => {
     //   className="w-1/3 border w-1/2 rounded py-2 px-3 text-gray-700 leading-tight focus:outline-none focus:shadow-outline bg-white">
     //   {"30 to infinity, 80% mass" |> ReasonReact.string}
     // </div>
-    | None => "Something went wrong" |> ReasonReact.string
+    | Error(e) => e |> ReasonReact.string
     };
   | FloatCdf(_) => <div />
   | Probability(r) =>

src/models/EAFunds.re

@@ -112,8 +112,12 @@ module Model = {
     GlobalCatastrophe.makeI(MomentRe.momentNow())
     |> RenderTypes.DistPlusRenderer.make(~distPlusIngredients=_, ())
     |> DistPlusRenderer.run
-    |> RenderTypes.DistPlusRenderer.Outputs.distplus
+    |> E.R.bind(_, r =>
-    |> E.O.bind(_, DistPlusTime.Integral.xToY(Time(dateTime)));
+         r
+         |> DistPlusTime.Integral.xToY(Time(dateTime))
+         |> E.O.toResult("error")
+       )
+    |> E.R.toOption;
   };
 
   let make =