quantified.uncertainty/squiggle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Lerna with dependencies between packages

Browse Source
This commit is contained in:
Sam Nolan 2022-02-08 13:59:37 +11:00
parent 239c08b6cb
commit 1b3ad56977
64 changed files with 110972 additions and 64488 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
node_modules
.cache
.merlin

13
README.md
View File

@ -5,15 +5,10 @@ This is an experiment DSL/language for making probabilistic estimates.
## DistPlus
We have a custom library called DistPlus to handle distributions with additional metadata. This helps handle mixed distributions (continuous + discrete), a cache for a cdf, possible unit types (specific times are supported), and limited domains.
## Running
Currently it only has a few very simple models.
## Running packages in the monorepo
This application uses `lerna` to manage dependencies between packages. To install
dependencies of all packages, run:
```
yarn
yarn run start
yarn run parcel
lerna bootstrap
```
## Expected future setup
![setup](https://raw.githubusercontent.com/foretold-app/widedomain/master/Screen%20Shot%202020-06-30%20at%208.27.32%20AM.png)

6
lerna.json Normal file
View File

@ -0,0 +1,6 @@
{
"packages": [
"packages/*"
],
"version": "0.0.0"
}

5
package.json
View File

@ -1,8 +1,7 @@
{
"name": "root",
"private": true,
"devDependencies": {
"lerna": "^4.0.0"
},
"scripts": {
}
"name": "squiggle"
}

13
packages/playground/__tests__/Bandwidth__Test.re
View File

@ -1,13 +0,0 @@
open Jest;
open Expect;
describe("Bandwidth", () => {
test("nrd0()", () => {
let data = [|1., 4., 3., 2.|];
expect(Bandwidth.nrd0(data)) |> toEqual(0.7625801874014622);
});
test("nrd()", () => {
let data = [|1., 4., 3., 2.|];
expect(Bandwidth.nrd(data)) |> toEqual(0.8981499984950554);
});
});

104
packages/playground/__tests__/DistTypes__Test.re
View File

@ -1,104 +0,0 @@
open Jest;
open Expect;
let makeTest = (~only=false, str, item1, item2) =>
only
? Only.test(str, () =>
expect(item1) |> toEqual(item2)
)
: test(str, () =>
expect(item1) |> toEqual(item2)
);
describe("DistTypes", () => {
describe("Domain", () => {
let makeComplete = (yPoint, expectation) =>
makeTest(
"With input: " ++ Js.Float.toString(yPoint),
DistTypes.Domain.yPointToSubYPoint(Complete, yPoint),
expectation,
);
let makeSingle =
(
direction: [ | `left | `right],
excludingProbabilityMass,
yPoint,
expectation,
) =>
makeTest(
"Excluding: "
++ Js.Float.toString(excludingProbabilityMass)
++ " and yPoint: "
++ Js.Float.toString(yPoint),
DistTypes.Domain.yPointToSubYPoint(
direction == `left
? LeftLimited({xPoint: 3.0, excludingProbabilityMass})
: RightLimited({xPoint: 3.0, excludingProbabilityMass}),
yPoint,
),
expectation,
);
let makeDouble = (domain, yPoint, expectation) =>
makeTest(
"Excluding: limits",
DistTypes.Domain.yPointToSubYPoint(domain, yPoint),
expectation,
);
describe("With Complete Domain", () => {
makeComplete(0.0, Some(0.0));
makeComplete(0.6, Some(0.6));
makeComplete(1.0, Some(1.0));
});
describe("With Left Limit", () => {
makeSingle(`left, 0.5, 1.0, Some(1.0));
makeSingle(`left, 0.5, 0.75, Some(0.5));
makeSingle(`left, 0.8, 0.9, Some(0.5));
makeSingle(`left, 0.5, 0.4, None);
makeSingle(`left, 0.5, 0.5, Some(0.0));
});
describe("With Right Limit", () => {
makeSingle(`right, 0.5, 1.0, None);
makeSingle(`right, 0.5, 0.25, Some(0.5));
makeSingle(`right, 0.8, 0.5, None);
makeSingle(`right, 0.2, 0.2, Some(0.25));
makeSingle(`right, 0.5, 0.5, Some(1.0));
makeSingle(`right, 0.5, 0.0, Some(0.0));
makeSingle(`right, 0.5, 0.5, Some(1.0));
});
describe("With Left and Right Limit", () => {
makeDouble(
LeftAndRightLimited(
{excludingProbabilityMass: 0.25, xPoint: 3.0},
{excludingProbabilityMass: 0.25, xPoint: 10.0},
),
0.5,
Some(0.5),
);
makeDouble(
LeftAndRightLimited(
{excludingProbabilityMass: 0.1, xPoint: 3.0},
{excludingProbabilityMass: 0.1, xPoint: 10.0},
),
0.2,
Some(0.125),
);
makeDouble(
LeftAndRightLimited(
{excludingProbabilityMass: 0.1, xPoint: 3.0},
{excludingProbabilityMass: 0.1, xPoint: 10.0},
),
0.1,
Some(0.0),
);
makeDouble(
LeftAndRightLimited(
{excludingProbabilityMass: 0.1, xPoint: 3.0},
{excludingProbabilityMass: 0.1, xPoint: 10.0},
),
0.05,
None,
);
});
})
});

415
packages/playground/__tests__/Distributions__Test.re
View File

@ -1,415 +0,0 @@
open Jest;
open Expect;
let shape: DistTypes.xyShape = {xs: [|1., 4., 8.|], ys: [|8., 9., 2.|]};
// let makeTest = (~only=false, str, item1, item2) =>
// only
// ? Only.test(str, () =>
// expect(item1) |> toEqual(item2)
// )
// : test(str, () =>
// expect(item1) |> toEqual(item2)
// );
// let makeTestCloseEquality = (~only=false, str, item1, item2, ~digits) =>
// only
// ? Only.test(str, () =>
// expect(item1) |> toBeSoCloseTo(item2, ~digits)
// )
// : test(str, () =>
// expect(item1) |> toBeSoCloseTo(item2, ~digits)
// );
// describe("Shape", () => {
// describe("Continuous", () => {
// open Continuous;
// let continuous = make(`Linear, shape, None);
// makeTest("minX", T.minX(continuous), 1.0);
// makeTest("maxX", T.maxX(continuous), 8.0);
// makeTest(
// "mapY",
// T.mapY(r => r *. 2.0, continuous) |> getShape |> (r => r.ys),
// [|16., 18.0, 4.0|],
// );
// describe("xToY", () => {
// describe("when Linear", () => {
// makeTest(
// "at 4.0",
// T.xToY(4., continuous),
// {continuous: 9.0, discrete: 0.0},
// );
// // Note: This below is weird to me, I'm not sure if it's what we want really.
// makeTest(
// "at 0.0",
// T.xToY(0., continuous),
// {continuous: 8.0, discrete: 0.0},
// );
// makeTest(
// "at 5.0",
// T.xToY(5., continuous),
// {continuous: 7.25, discrete: 0.0},
// );
// makeTest(
// "at 10.0",
// T.xToY(10., continuous),
// {continuous: 2.0, discrete: 0.0},
// );
// });
// describe("when Stepwise", () => {
// let continuous = make(`Stepwise, shape, None);
// makeTest(
// "at 4.0",
// T.xToY(4., continuous),
// {continuous: 9.0, discrete: 0.0},
// );
// makeTest(
// "at 0.0",
// T.xToY(0., continuous),
// {continuous: 0.0, discrete: 0.0},
// );
// makeTest(
// "at 5.0",
// T.xToY(5., continuous),
// {continuous: 9.0, discrete: 0.0},
// );
// makeTest(
// "at 10.0",
// T.xToY(10., continuous),
// {continuous: 2.0, discrete: 0.0},
// );
// });
// });
// makeTest(
// "integral",
// T.Integral.get(~cache=None, continuous) |> getShape,
// {xs: [|1.0, 4.0, 8.0|], ys: [|0.0, 25.5, 47.5|]},
// );
// makeTest(
// "toLinear",
// {
// let continuous =
// make(`Stepwise, {xs: [|1., 4., 8.|], ys: [|0.1, 5., 1.0|]}, None);
// continuous |> toLinear |> E.O.fmap(getShape);
// },
// Some({
// xs: [|1.00007, 1.00007, 4.0, 4.00007, 8.0, 8.00007|],
// ys: [|0.0, 0.1, 0.1, 5.0, 5.0, 1.0|],
// }),
// );
// makeTest(
// "toLinear",
// {
// let continuous = make(`Stepwise, {xs: [|0.0|], ys: [|0.3|]}, None);
// continuous |> toLinear |> E.O.fmap(getShape);
// },
// Some({xs: [|0.0|], ys: [|0.3|]}),
// );
// makeTest(
// "integralXToY",
// T.Integral.xToY(~cache=None, 0.0, continuous),
// 0.0,
// );
// makeTest(
// "integralXToY",
// T.Integral.xToY(~cache=None, 2.0, continuous),
// 8.5,
// );
// makeTest(
// "integralXToY",
// T.Integral.xToY(~cache=None, 100.0, continuous),
// 47.5,
// );
// makeTest(
// "integralEndY",
// continuous
// |> T.normalize //scaleToIntegralSum(~intendedSum=1.0)
// |> T.Integral.sum(~cache=None),
// 1.0,
// );
// });
// describe("Discrete", () => {
// open Discrete;
// let shape: DistTypes.xyShape = {
// xs: [|1., 4., 8.|],
// ys: [|0.3, 0.5, 0.2|],
// };
// let discrete = make(shape, None);
// makeTest("minX", T.minX(discrete), 1.0);
// makeTest("maxX", T.maxX(discrete), 8.0);
// makeTest(
// "mapY",
// T.mapY(r => r *. 2.0, discrete) |> (r => getShape(r).ys),
// [|0.6, 1.0, 0.4|],
// );
// makeTest(
// "xToY at 4.0",
// T.xToY(4., discrete),
// {discrete: 0.5, continuous: 0.0},
// );
// makeTest(
// "xToY at 0.0",
// T.xToY(0., discrete),
// {discrete: 0.0, continuous: 0.0},
// );
// makeTest(
// "xToY at 5.0",
// T.xToY(5., discrete),
// {discrete: 0.0, continuous: 0.0},
// );
// makeTest(
// "scaleBy",
// scaleBy(~scale=4.0, discrete),
// make({xs: [|1., 4., 8.|], ys: [|1.2, 2.0, 0.8|]}, None),
// );
// makeTest(
// "normalize, then scale by 4.0",
// discrete
// |> T.normalize
// |> scaleBy(~scale=4.0),
// make({xs: [|1., 4., 8.|], ys: [|1.2, 2.0, 0.8|]}, None),
// );
// makeTest(
// "scaleToIntegralSum: back and forth",
// discrete
// |> T.normalize
// |> scaleBy(~scale=4.0)
// |> T.normalize,
// discrete,
// );
// makeTest(
// "integral",
// T.Integral.get(~cache=None, discrete),
// Continuous.make(
// `Stepwise,
// {xs: [|1., 4., 8.|], ys: [|0.3, 0.8, 1.0|]},
// None
// ),
// );
// makeTest(
// "integral with 1 element",
// T.Integral.get(~cache=None, Discrete.make({xs: [|0.0|], ys: [|1.0|]}, None)),
// Continuous.make(`Stepwise, {xs: [|0.0|], ys: [|1.0|]}, None),
// );
// makeTest(
// "integralXToY",
// T.Integral.xToY(~cache=None, 6.0, discrete),
// 0.9,
// );
// makeTest("integralEndY", T.Integral.sum(~cache=None, discrete), 1.0);
// makeTest("mean", T.mean(discrete), 3.9);
// makeTestCloseEquality(
// "variance",
// T.variance(discrete),
// 5.89,
// ~digits=7,
// );
// });
// describe("Mixed", () => {
// open Distributions.Mixed;
// let discreteShape: DistTypes.xyShape = {
// xs: [|1., 4., 8.|],
// ys: [|0.3, 0.5, 0.2|],
// };
// let discrete = Discrete.make(discreteShape, None);
// let continuous =
// Continuous.make(
// `Linear,
// {xs: [|3., 7., 14.|], ys: [|0.058, 0.082, 0.124|]},
// None
// )
// |> Continuous.T.normalize; //scaleToIntegralSum(~intendedSum=1.0);
// let mixed = Mixed.make(
// ~continuous,
// ~discrete,
// );
// makeTest("minX", T.minX(mixed), 1.0);
// makeTest("maxX", T.maxX(mixed), 14.0);
// makeTest(
// "mapY",
// T.mapY(r => r *. 2.0, mixed),
// Mixed.make(
// ~continuous=
// Continuous.make(
// `Linear,
// {
// xs: [|3., 7., 14.|],
// ys: [|
// 0.11588411588411589,
// 0.16383616383616384,
// 0.24775224775224775,
// |],
// },
// None
// ),
// ~discrete=Discrete.make({xs: [|1., 4., 8.|], ys: [|0.6, 1.0, 0.4|]}, None)
// ),
// );
// makeTest(
// "xToY at 4.0",
// T.xToY(4., mixed),
// {discrete: 0.25, continuous: 0.03196803196803197},
// );
// makeTest(
// "xToY at 0.0",
// T.xToY(0., mixed),
// {discrete: 0.0, continuous: 0.028971028971028972},
// );
// makeTest(
// "xToY at 5.0",
// T.xToY(7., mixed),
// {discrete: 0.0, continuous: 0.04095904095904096},
// );
// makeTest("integralEndY", T.Integral.sum(~cache=None, mixed), 1.0);
// makeTest(
// "scaleBy",
// Mixed.scaleBy(~scale=2.0, mixed),
// Mixed.make(
// ~continuous=
// Continuous.make(
// `Linear,
// {
// xs: [|3., 7., 14.|],
// ys: [|
// 0.11588411588411589,
// 0.16383616383616384,
// 0.24775224775224775,
// |],
// },
// None
// ),
// ~discrete=Discrete.make({xs: [|1., 4., 8.|], ys: [|0.6, 1.0, 0.4|]}, None),
// ),
// );
// makeTest(
// "integral",
// T.Integral.get(~cache=None, mixed),
// Continuous.make(
// `Linear,
// {
// xs: [|1.00007, 1.00007, 3., 4., 4.00007, 7., 8., 8.00007, 14.|],
// ys: [|
// 0.0,
// 0.0,
// 0.15,
// 0.18496503496503497,
// 0.4349674825174825,
// 0.5398601398601399,
// 0.5913086913086913,
// 0.6913122927072927,
// 1.0,
// |],
// },
// None,
// ),
// );
// });
// describe("Distplus", () => {
// open DistPlus;
// let discreteShape: DistTypes.xyShape = {
// xs: [|1., 4., 8.|],
// ys: [|0.3, 0.5, 0.2|],
// };
// let discrete = Discrete.make(discreteShape, None);
// let continuous =
// Continuous.make(
// `Linear,
// {xs: [|3., 7., 14.|], ys: [|0.058, 0.082, 0.124|]},
// None
// )
// |> Continuous.T.normalize; //scaleToIntegralSum(~intendedSum=1.0);
// let mixed =
// Mixed.make(
// ~continuous,
// ~discrete,
// );
// let distPlus =
// DistPlus.make(
// ~shape=Mixed(mixed),
// ~squiggleString=None,
// (),
// );
// makeTest("minX", T.minX(distPlus), 1.0);
// makeTest("maxX", T.maxX(distPlus), 14.0);
// makeTest(
// "xToY at 4.0",
// T.xToY(4., distPlus),
// {discrete: 0.25, continuous: 0.03196803196803197},
// );
// makeTest(
// "xToY at 0.0",
// T.xToY(0., distPlus),
// {discrete: 0.0, continuous: 0.028971028971028972},
// );
// makeTest(
// "xToY at 5.0",
// T.xToY(7., distPlus),
// {discrete: 0.0, continuous: 0.04095904095904096},
// );
// makeTest("integralEndY", T.Integral.sum(~cache=None, distPlus), 1.0);
// makeTest(
// "integral",
// T.Integral.get(~cache=None, distPlus) |> T.toContinuous,
// Some(
// Continuous.make(
// `Linear,
// {
// xs: [|1.00007, 1.00007, 3., 4., 4.00007, 7., 8., 8.00007, 14.|],
// ys: [|
// 0.0,
// 0.0,
// 0.15,
// 0.18496503496503497,
// 0.4349674825174825,
// 0.5398601398601399,
// 0.5913086913086913,
// 0.6913122927072927,
// 1.0,
// |],
// },
// None,
// ),
// ),
// );
// });
// describe("Shape", () => {
// let mean = 10.0;
// let stdev = 4.0;
// let variance = stdev ** 2.0;
// let numSamples = 10000;
// open Distributions.Shape;
// let normal: SymbolicTypes.symbolicDist = `Normal({mean, stdev});
// let normalShape = ExpressionTree.toShape(numSamples, `SymbolicDist(normal));
// let lognormal = SymbolicDist.Lognormal.fromMeanAndStdev(mean, stdev);
// let lognormalShape = ExpressionTree.toShape(numSamples, `SymbolicDist(lognormal));
// makeTestCloseEquality(
// "Mean of a normal",
// T.mean(normalShape),
// mean,
// ~digits=2,
// );
// makeTestCloseEquality(
// "Variance of a normal",
// T.variance(normalShape),
// variance,
// ~digits=1,
// );
// makeTestCloseEquality(
// "Mean of a lognormal",
// T.mean(lognormalShape),
// mean,
// ~digits=2,
// );
// makeTestCloseEquality(
// "Variance of a lognormal",
// T.variance(lognormalShape),
// variance,
// ~digits=0,
// );
// });
// });

57
packages/playground/__tests__/Hardcoded__Test.re
View File

@ -1,57 +0,0 @@
open Jest;
open Expect;
let makeTest = (~only=false, str, item1, item2) =>
only
? Only.test(str, () =>
expect(item1) |> toEqual(item2)
)
: test(str, () =>
expect(item1) |> toEqual(item2)
);
let evalParams: ExpressionTypes.ExpressionTree.evaluationParams = {
samplingInputs: {
sampleCount: 1000,
outputXYPoints: 10000,
kernelWidth: None,
shapeLength: 1000,
},
environment:
[|
("K", `SymbolicDist(`Float(1000.0))),
("M", `SymbolicDist(`Float(1000000.0))),
("B", `SymbolicDist(`Float(1000000000.0))),
("T", `SymbolicDist(`Float(1000000000000.0))),
|]
->Belt.Map.String.fromArray,
evaluateNode: ExpressionTreeEvaluator.toLeaf,
};
let shape1: DistTypes.xyShape = {xs: [|1., 4., 8.|], ys: [|0.2, 0.4, 0.8|]};
describe("XYShapes", () => {
describe("logScorePoint", () => {
makeTest(
"When identical",
{
let foo =
HardcodedFunctions.(
makeRenderedDistFloat("scaleMultiply", (dist, float) =>
verticalScaling(`Multiply, dist, float)
)
);
TypeSystem.Function.T.run(
evalParams,
[|
`SymbolicDist(`Float(100.0)),
`SymbolicDist(`Float(1.0)),
|],
foo,
);
},
Error("Sad"),
)
})
});

24
packages/playground/__tests__/Lodash__test.re
View File

@ -1,24 +0,0 @@
open Jest;
open Expect;
let makeTest = (~only=false, str, item1, item2) =>
only
? Only.test(str, () =>
expect(item1) |> toEqual(item2)
)
: test(str, () =>
expect(item1) |> toEqual(item2)
);
describe("Lodash", () => {
describe("Lodash", () => {
makeTest("min", Lodash.min([|1, 3, 4|]), 1);
makeTest("max", Lodash.max([|1, 3, 4|]), 4);
makeTest("uniq", Lodash.uniq([|1, 3, 4, 4|]), [|1, 3, 4|]);
makeTest(
"countBy",
Lodash.countBy([|1, 3, 4, 4|], r => r),
Js.Dict.fromArray([|("1", 1), ("3", 1), ("4", 2)|]),
);
})
});

51
packages/playground/__tests__/Samples__test.re
View File

@ -1,51 +0,0 @@
open Jest;
open Expect;
let makeTest = (~only=false, str, item1, item2) =>
only
? Only.test(str, () =>
expect(item1) |> toEqual(item2)
)
: test(str, () =>
expect(item1) |> toEqual(item2)
);
describe("Lodash", () => {
describe("Lodash", () => {
makeTest(
"split",
SamplesToShape.Internals.T.splitContinuousAndDiscrete([|1.432, 1.33455, 2.0|]),
([|1.432, 1.33455, 2.0|], E.FloatFloatMap.empty()),
);
makeTest(
"split",
SamplesToShape.Internals.T.splitContinuousAndDiscrete([|
1.432,
1.33455,
2.0,
2.0,
2.0,
2.0,
|])
|> (((c, disc)) => (c, disc |> E.FloatFloatMap.toArray)),
([|1.432, 1.33455|], [|(2.0, 4.0)|]),
);
let makeDuplicatedArray = count => {
let arr = Belt.Array.range(1, count) |> E.A.fmap(float_of_int);
let sorted = arr |> Belt.SortArray.stableSortBy(_, compare);
E.A.concatMany([|sorted, sorted, sorted, sorted|])
|> Belt.SortArray.stableSortBy(_, compare);
};
let (_, discrete) =
SamplesToShape.Internals.T.splitContinuousAndDiscrete(makeDuplicatedArray(10));
let toArr = discrete |> E.FloatFloatMap.toArray;
makeTest("splitMedium", toArr |> Belt.Array.length, 10);
let (c, discrete) =
SamplesToShape.Internals.T.splitContinuousAndDiscrete(makeDuplicatedArray(500));
let toArr = discrete |> E.FloatFloatMap.toArray;
makeTest("splitMedium", toArr |> Belt.Array.length, 500);
})
});

63
packages/playground/__tests__/XYShape__Test.re
View File

@ -1,63 +0,0 @@
open Jest;
open Expect;
let makeTest = (~only=false, str, item1, item2) =>
only
? Only.test(str, () =>
expect(item1) |> toEqual(item2)
)
: test(str, () =>
expect(item1) |> toEqual(item2)
);
let shape1: DistTypes.xyShape = {xs: [|1., 4., 8.|], ys: [|0.2, 0.4, 0.8|]};
let shape2: DistTypes.xyShape = {
xs: [|1., 5., 10.|],
ys: [|0.2, 0.5, 0.8|],
};
let shape3: DistTypes.xyShape = {
xs: [|1., 20., 50.|],
ys: [|0.2, 0.5, 0.8|],
};
describe("XYShapes", () => {
describe("logScorePoint", () => {
makeTest(
"When identical",
XYShape.logScorePoint(30, shape1, shape1),
Some(0.0),
);
makeTest(
"When similar",
XYShape.logScorePoint(30, shape1, shape2),
Some(1.658971191043856),
);
makeTest(
"When very different",
XYShape.logScorePoint(30, shape1, shape3),
Some(210.3721280423322),
);
});
// describe("transverse", () => {
// makeTest(
// "When very different",
// XYShape.Transversal._transverse(
// (aCurrent, aLast) => aCurrent +. aLast,
// [|1.0, 2.0, 3.0, 4.0|],
// ),
// [|1.0, 3.0, 6.0, 10.0|],
// )
// });
describe("integrateWithTriangles", () => {
makeTest(
"integrates correctly",
XYShape.Range.integrateWithTriangles(shape1),
Some({
xs: [|1., 4., 8.|],
ys: [|0.0, 0.9000000000000001, 3.3000000000000007|],
}),
)
});
});

2
packages/playground/bsconfig.json
View File

@ -39,7 +39,7 @@
"@rescript/react",
"bs-css",
"bs-css-dom",
"squiggle-experimental",
"@foretold-app/squiggle",
"rationale",
"bs-moment",
"reschema"

30832
packages/playground/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

18
packages/playground/package.json
View File

@ -1,9 +1,10 @@
{
"name": "estiband",
"name": "@foretold-app/squiggle-playground",
"version": "0.1.0",
"homepage": "https://foretold-app.github.io/estiband/",
"scripts": {
"build": "rescript build",
"build:deps": "rescript build -with-deps",
"build:style": "tailwind build src/styles/index.css -o src/styles/tailwind.css",
"start": "rescript build -w",
"clean": "rescript clean",
@ -38,8 +39,8 @@
"bs-css": "^15.1.0",
"bs-css-dom": "^3.1.0",
"bs-moment": "0.6.0",
"bs-reform": "^10.0.3",
"bsb-js": "1.1.7",
"css-loader": "^6.6.0",
"d3": "7.3.0",
"gh-pages": "2.2.0",
"jest": "^25.5.1",
@ -51,17 +52,16 @@
"moduleserve": "0.9.1",
"moment": "2.24.0",
"pdfast": "^0.2.0",
"postcss-cli": "7.1.0",
"postcss-cli": "^9.1.0",
"rationale": "0.2.0",
"react": "^16.10.0",
"react": "17.0.2",
"react-ace": "^9.2.0",
"react-dom": "^0.13.0 || ^0.14.0 || ^15.0.1 || ^16.0.0",
"react-dom": "^17.0.2",
"react-use": "^17.3.2",
"react-vega": "^7.4.4",
"reason-react": ">=0.7.0",
"reschema": "^2.2.0",
"rescript": "^9.1.4",
"squiggle-experimental": "^0.1.8",
"@foretold-app/squiggle": "^0.1.9",
"tailwindcss": "1.2.0",
"vega": "*",
"vega-embed": "6.6.0",
@ -69,9 +69,9 @@
},
"devDependencies": {
"@glennsl/bs-jest": "^0.5.1",
"bs-platform": "9.0.2",
"bs-platform": "8.4.2",
"docsify": "^4.12.2",
"parcel-bundler": "1.12.4",
"parcel-bundler": "^1.12.5",
"parcel-plugin-bundle-visualiser": "^1.2.0",
"parcel-plugin-less-js-enabled": "1.0.2"
},

17
packages/playground/src/distPlus/utility/E.res → packages/playground/src/E.res
View File

@ -85,11 +85,6 @@ module O = {
let min = compare(\"<")
let max = compare(\">")
module React = {
let defaultNull = default(React.null)
let fmapOrNull = fn => \"||>"(fmap(fn), default(React.null))
let flatten = default(React.null)
}
}
/* Functions */
@ -196,18 +191,6 @@ module J = {
}
}
module M = {
let format = MomentRe.Moment.format
let format_standard = "MMM DD, YYYY HH:mm"
let format_simple = "L"
/* TODO: Figure out better name */
let goFormat_simple = MomentRe.Moment.format(format_simple)
let goFormat_standard = MomentRe.Moment.format(format_standard)
let toUtc = MomentRe.momentUtc
let toJSON = MomentRe.Moment.toJSON
let momentDefaultFormat = MomentRe.momentDefaultFormat
}
module JsDate = {
let fromString = Js.Date.fromString
let now = Js.Date.now

12
packages/playground/src/components/DistBuilder.res
View File

@ -97,7 +97,7 @@ module DemoDist = {
<div>
{switch options {
| Some(options) =>
let inputs1 = ProgramEvaluator.Inputs.make(
let inputs1 = ForetoldAppSquiggle.ProgramEvaluator.Inputs.make(
~samplingInputs={
sampleCount: Some(options.sampleCount),
outputXYPoints: Some(options.outputXYPoints),
@ -114,15 +114,15 @@ module DemoDist = {
(),
)
let distributionList = ProgramEvaluator.evaluateProgram(inputs1)
let distributionList = ForetoldAppSquiggle.ProgramEvaluator.evaluateProgram(inputs1)
let renderExpression = response1 =>
switch response1 {
| #DistPlus(distPlus1) => <DistPlusPlot distPlus={DistPlus.T.normalize(distPlus1)} />
| #DistPlus(distPlus1) => <DistPlusPlot distPlus={ForetoldAppSquiggle.DistPlus.T.normalize(distPlus1)} />
| #Float(f) => <NumberShower number=f precision=3 />
| #Function((f, a), env) =>
// Problem: When it gets the function, it doesn't save state about previous commands
let foo: ProgramEvaluator.Inputs.inputs = {
let foo: ForetoldAppSquiggle.ProgramEvaluator.Inputs.inputs = {
squiggleString: squiggleString,
samplingInputs: inputs1.samplingInputs,
environment: env,
@ -130,13 +130,13 @@ module DemoDist = {
let results =
E.A.Floats.range(options.diagramStart, options.diagramStop, options.diagramCount)
|> E.A.fmap(r =>
ProgramEvaluator.evaluateFunction(
ForetoldAppSquiggle.ProgramEvaluator.evaluateFunction(
foo,
(f, a),
[#SymbolicDist(#Float(r))],
) |> E.R.bind(_, a =>
switch a {
| #DistPlus(d) => Ok((r, DistPlus.T.normalize(d)))
| #DistPlus(d) => Ok((r, ForetoldAppSquiggle.DistPlus.T.normalize(d)))
| n =>
Js.log2("Error here", n)
Error("wrong type")

74
packages/playground/src/components/charts/DistPlusPlot.res
View File

@ -1,7 +1,7 @@
open DistPlusPlotReducer
let plotBlue = #hex("1860ad")
let showAsForm = (distPlus: DistTypes.distPlus) =>
let showAsForm = (distPlus: ForetoldAppSquiggle.DistTypes.distPlus) =>
<div> <Antd.Input value={distPlus.squiggleString |> E.O.default("")} /> </div>
let showFloat = (~precision=3, number) => <NumberShower number precision />
@ -23,27 +23,27 @@ let table = (distPlus, x) =>
<td className="px-4 py-2 border"> {x |> E.Float.toString |> React.string} </td>
<td className="px-4 py-2 border ">
{distPlus
|> DistPlus.T.xToY(x)
|> DistTypes.MixedPoint.toDiscreteValue
|> ForetoldAppSquiggle.DistPlus.T.xToY(x)
|> ForetoldAppSquiggle.DistTypes.MixedPoint.toDiscreteValue
|> Js.Float.toPrecisionWithPrecision(_, ~digits=7)
|> React.string}
</td>
<td className="px-4 py-2 border ">
{distPlus
|> DistPlus.T.xToY(x)
|> DistTypes.MixedPoint.toContinuousValue
|> ForetoldAppSquiggle.DistPlus.T.xToY(x)
|> ForetoldAppSquiggle.DistTypes.MixedPoint.toContinuousValue
|> Js.Float.toPrecisionWithPrecision(_, ~digits=7)
|> React.string}
</td>
<td className="px-4 py-2 border ">
{distPlus
|> DistPlus.T.Integral.xToY(x)
|> ForetoldAppSquiggle.DistPlus.T.Integral.xToY(x)
|> E.Float.with2DigitsPrecision
|> React.string}
</td>
<td className="px-4 py-2 border ">
{distPlus
|> DistPlus.T.Integral.sum
|> ForetoldAppSquiggle.DistPlus.T.Integral.sum
|> E.Float.with2DigitsPrecision
|> React.string}
</td>
@ -61,16 +61,16 @@ let table = (distPlus, x) =>
<tr>
<td className="px-4 py-2 border">
{distPlus
|> DistPlus.T.toContinuous
|> E.O.fmap(Continuous.T.Integral.sum)
|> ForetoldAppSquiggle.DistPlus.T.toContinuous
|> E.O.fmap(ForetoldAppSquiggle.Continuous.T.Integral.sum)
|> E.O.fmap(E.Float.with2DigitsPrecision)
|> E.O.default("")
|> React.string}
</td>
<td className="px-4 py-2 border ">
{distPlus
|> DistPlus.T.toDiscrete
|> E.O.fmap(Discrete.T.Integral.sum)
|> ForetoldAppSquiggle.DistPlus.T.toDiscrete
|> E.O.fmap(ForetoldAppSquiggle.Discrete.T.Integral.sum)
|> E.O.fmap(E.Float.with2DigitsPrecision)
|> E.O.default("")
|> React.string}
@ -97,28 +97,28 @@ let percentiles = distPlus =>
<tbody>
<tr>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.01) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.01) |> showFloat}
</td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.05) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.05) |> showFloat}
</td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.25) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.25) |> showFloat}
</td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.5) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.5) |> showFloat}
</td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.75) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.75) |> showFloat}
</td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.95) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.95) |> showFloat}
</td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.99) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.99) |> showFloat}
</td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.Integral.yToX(0.99999) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.99999) |> showFloat}
</td>
</tr>
</tbody>
@ -133,11 +133,11 @@ let percentiles = distPlus =>
</thead>
<tbody>
<tr>
<td className="px-4 py-2 border"> {distPlus |> DistPlus.T.mean |> showFloat} </td>
<td className="px-4 py-2 border"> {distPlus |> ForetoldAppSquiggle.DistPlus.T.mean |> showFloat} </td>
<td className="px-4 py-2 border">
{distPlus |> DistPlus.T.variance |> (r => r ** 0.5) |> showFloat}
{distPlus |> ForetoldAppSquiggle.DistPlus.T.variance |> (r => r ** 0.5) |> showFloat}
</td>
<td className="px-4 py-2 border"> {distPlus |> DistPlus.T.variance |> showFloat} </td>
<td className="px-4 py-2 border"> {distPlus |> ForetoldAppSquiggle.DistPlus.T.variance |> showFloat} </td>
</tr>
</tbody>
</table>
@ -155,11 +155,11 @@ let adjustBoth = discreteProbabilityMassFraction => {
module DistPlusChart = {
@react.component
let make = (~distPlus: DistTypes.distPlus, ~config: chartConfig, ~onHover) => {
open DistPlus
let make = (~distPlus: ForetoldAppSquiggle.DistTypes.distPlus, ~config: chartConfig, ~onHover) => {
open ForetoldAppSquiggle.DistPlus
let discrete = distPlus |> T.toDiscrete |> E.O.fmap(Discrete.getShape)
let continuous = distPlus |> T.toContinuous |> E.O.fmap(Continuous.getShape)
let discrete = distPlus |> T.toDiscrete |> E.O.fmap(ForetoldAppSquiggle.Discrete.getShape)
let continuous = distPlus |> T.toContinuous |> E.O.fmap(ForetoldAppSquiggle.Continuous.getShape)
// // We subtract a bit from the range to make sure that it fits. Maybe this should be done in d3 instead.
// let minX =
@ -172,12 +172,12 @@ module DistPlusChart = {
// | _ => None
// };
let minX = distPlus |> DistPlus.T.Integral.yToX(0.00001)
let minX = distPlus |> T.Integral.yToX(0.00001)
let maxX = distPlus |> DistPlus.T.Integral.yToX(0.99999)
let maxX = distPlus |> T.Integral.yToX(0.99999)
let timeScale = distPlus.unit |> DistTypes.DistributionUnit.toJson
let discreteProbabilityMassFraction = distPlus |> DistPlus.T.toDiscreteProbabilityMassFraction
let timeScale = distPlus.unit |> ForetoldAppSquiggle.DistTypes.DistributionUnit.toJson
let discreteProbabilityMassFraction = distPlus |> T.toDiscreteProbabilityMassFraction
let (yMaxDiscreteDomainFactor, yMaxContinuousDomainFactor) = adjustBoth(
discreteProbabilityMassFraction,
@ -202,13 +202,13 @@ module DistPlusChart = {
module IntegralChart = {
@react.component
let make = (~distPlus: DistTypes.distPlus, ~config: chartConfig, ~onHover) => {
let make = (~distPlus: ForetoldAppSquiggle.DistTypes.distPlus, ~config: chartConfig, ~onHover) => {
let integral = distPlus.integralCache
let continuous = integral |> Continuous.toLinear |> E.O.fmap(Continuous.getShape)
let minX = distPlus |> DistPlus.T.Integral.yToX(0.00001)
let continuous = integral |> ForetoldAppSquiggle.Continuous.toLinear |> E.O.fmap(ForetoldAppSquiggle.Continuous.getShape)
let minX = distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.00001)
let maxX = distPlus |> DistPlus.T.Integral.yToX(0.99999)
let timeScale = distPlus.unit |> DistTypes.DistributionUnit.toJson
let maxX = distPlus |> ForetoldAppSquiggle.DistPlus.T.Integral.yToX(0.99999)
let timeScale = distPlus.unit |> ForetoldAppSquiggle.DistTypes.DistributionUnit.toJson
<DistributionPlot
xScale={config.xLog ? "log" : "linear"}
yScale={config.yLog ? "log" : "linear"}
@ -225,7 +225,7 @@ module IntegralChart = {
module Chart = {
@react.component
let make = (~distPlus: DistTypes.distPlus, ~config: chartConfig, ~onHover) => {
let make = (~distPlus: ForetoldAppSquiggle.DistTypes.distPlus, ~config: chartConfig, ~onHover) => {
let chart = React.useMemo2(
() =>
config.isCumulative
@ -246,7 +246,7 @@ module Chart = {
let button = "bg-gray-300 hover:bg-gray-500 text-grey-darkest text-xs px-4 py-1"
@react.component
let make = (~distPlus: DistTypes.distPlus) => {
let make = (~distPlus: ForetoldAppSquiggle.DistTypes.distPlus) => {
let (x, setX) = React.useState(() => 0.)
let (state, dispatch) = React.useReducer(DistPlusPlotReducer.reducer, DistPlusPlotReducer.init)

4
packages/playground/src/components/charts/DistributionPlot/DistributionPlot.res
View File

@ -95,12 +95,12 @@ let make = (
?xScale
?yScale
?timeScale
discrete={discrete |> E.O.fmap(XYShape.T.toJs)}
discrete={discrete |> E.O.fmap(ForetoldAppSquiggle.XYShape.T.toJs)}
height
marginBottom=50
marginTop=0
onHover
continuous={continuous |> E.O.fmap(XYShape.T.toJs)}
continuous={continuous |> E.O.fmap(ForetoldAppSquiggle.XYShape.T.toJs)}
showDistributionLines
showDistributionYAxis
showVerticalLine

3
packages/playground/src/components/charts/DistributionPlot/PercentilesChart.res
View File

@ -1,3 +1,4 @@
open ForetoldAppSquiggle
@module("./PercentilesChart.js")
external percentilesChart: React.element = "PercentilesChart"
@ -30,7 +31,7 @@ module Internal = {
@react.component
@module("./PercentilesChart.js")
let make = (~dists: array<(float, DistTypes.distPlus)>, ~children=React.null) => {
let data = dists |> E.A.fmap(((x, r)) =>
let data = dists -> Belt.Array.map(((x, r)) =>
{
"x": x,
"p1": r |> DistPlus.T.Integral.yToX(0.01),

4
packages/playground/src/components/charts/NumberShower.res
View File

@ -20,8 +20,8 @@ let make = (~number, ~precision) => {
let numberWithPresentation = JS.numberShow(number, precision)
<span>
{JS.valueGet(numberWithPresentation) |> React.string}
{JS.symbolGet(numberWithPresentation) |> E.O.React.fmapOrNull(React.string)}
{JS.powerGet(numberWithPresentation) |> E.O.React.fmapOrNull(e =>
{JS.symbolGet(numberWithPresentation) |> R.O.fmapOrNull(React.string)}
{JS.powerGet(numberWithPresentation) |> R.O.fmapOrNull(e =>
<span>
{j`\\u00b710` |> React.string}
<span style=sup> {e |> E.Float.toString |> React.string} </span>

171
packages/playground/src/distPlus/ProgramEvaluator.res
View File

@ -1,171 +0,0 @@
// TODO: This setup is more confusing than it should be, there's more work to do in cleanup here.
module Inputs = {
module SamplingInputs = {
type t = {
sampleCount: option<int>,
outputXYPoints: option<int>,
kernelWidth: option<float>,
shapeLength: option<int>,
}
}
let defaultRecommendedLength = 100
let defaultShouldDownsample = true
type inputs = {
squiggleString: string,
samplingInputs: SamplingInputs.t,
environment: ExpressionTypes.ExpressionTree.environment,
}
let empty: SamplingInputs.t = {
sampleCount: None,
outputXYPoints: None,
kernelWidth: None,
shapeLength: None,
}
let make = (
~samplingInputs=empty,
~squiggleString,
~environment=ExpressionTypes.ExpressionTree.Environment.empty,
(),
): inputs => {
samplingInputs: samplingInputs,
squiggleString: squiggleString,
environment: environment,
}
}
type \"export" = [
| #DistPlus(ProbExample.DistPlus.t)
| #Float(float)
| #Function(
(array<string>, ProbExample.ExpressionTypes.ExpressionTree.node),
ProbExample.ExpressionTypes.ExpressionTree.environment,
)
]
module Internals = {
let addVariable = (
{samplingInputs, squiggleString, environment}: Inputs.inputs,
str,
node,
): Inputs.inputs => {
samplingInputs: samplingInputs,
squiggleString: squiggleString,
environment: ExpressionTypes.ExpressionTree.Environment.update(environment, str, _ => Some(
node,
)),
}
type outputs = {
graph: ExpressionTypes.ExpressionTree.node,
shape: DistTypes.shape,
}
let makeOutputs = (graph, shape): outputs => {graph: graph, shape: shape}
let makeInputs = (inputs: Inputs.inputs): ExpressionTypes.ExpressionTree.samplingInputs => {
sampleCount: inputs.samplingInputs.sampleCount |> E.O.default(10000),
outputXYPoints: inputs.samplingInputs.outputXYPoints |> E.O.default(10000),
kernelWidth: inputs.samplingInputs.kernelWidth,
shapeLength: inputs.samplingInputs.shapeLength |> E.O.default(10000),
}
let runNode = (inputs, node) =>
ExpressionTree.toLeaf(makeInputs(inputs), inputs.environment, node)
let runProgram = (inputs: Inputs.inputs, p: ExpressionTypes.Program.program) => {
let ins = ref(inputs)
p
|> E.A.fmap(x =>
switch x {
| #Assignment(name, node) =>
ins := addVariable(ins.contents, name, node)
None
| #Expression(node) =>
Some(runNode(ins.contents, node) |> E.R.fmap(r => (ins.contents.environment, r)))
}
)
|> E.A.O.concatSomes
|> E.A.R.firstErrorOrOpen
}
let inputsToLeaf = (inputs: Inputs.inputs) =>
MathJsParser.fromString(inputs.squiggleString)
|> E.R.bind(_, g => runProgram(inputs, g))
let outputToDistPlus = (inputs: Inputs.inputs, shape: DistTypes.shape) =>
DistPlus.make(~shape, ~squiggleString=Some(inputs.squiggleString), ())
}
let renderIfNeeded = (inputs: Inputs.inputs, node: ExpressionTypes.ExpressionTree.node): result<
ExpressionTypes.ExpressionTree.node,
string,
> =>
node |> (
x =>
switch x {
| #Normalize(_) as n
| #SymbolicDist(_) as n =>
#Render(n)
|> Internals.runNode(inputs)
|> (
x =>
switch x {
| Ok(#RenderedDist(_)) as r => r
| Error(r) => Error(r)
| _ => Error("Didn't render, but intended to")
}
)
| n => Ok(n)
}
)
// TODO: Consider using ExpressionTypes.ExpressionTree.getFloat or similar in this function
let coersionToExportedTypes = (
inputs,
env: ProbExample.ExpressionTypes.ExpressionTree.environment,
node: ExpressionTypes.ExpressionTree.node,
): result<\"export", string> =>
node
|> renderIfNeeded(inputs)
|> E.R.bind(_, x =>
switch x {
| #RenderedDist(Discrete({xyShape: {xs: [x], ys: [1.0]}})) => Ok(#Float(x))
| #SymbolicDist(#Float(x)) => Ok(#Float(x))
| #RenderedDist(n) => Ok(#DistPlus(Internals.outputToDistPlus(inputs, n)))
| #Function(n) => Ok(#Function(n, env))
| n => Error("Didn't output a rendered distribution. Format:" ++ ExpressionTree.toString(n))
}
)
let rec mapM = (f, xs) =>
switch xs {
| list{} => Ok(list{})
| list{x, ...rest} =>
switch f(x) {
| Error(err) => Error(err)
| Ok(val) =>
switch mapM(f, rest) {
| Error(err) => Error(err)
| Ok(restList) => Ok(list{val, ...restList})
}
}
}
let evaluateProgram = (inputs: Inputs.inputs) =>
inputs |> Internals.inputsToLeaf |> E.R.bind(_, xs => mapM(((a, b)) => coersionToExportedTypes(inputs, a, b), (Array.to_list(xs))))
let evaluateFunction = (
inputs: Inputs.inputs,
fn: (array<string>, ExpressionTypes.ExpressionTree.node),
fnInputs,
) => {
let output = ExpressionTree.runFunction(
Internals.makeInputs(inputs),
inputs.environment,
fnInputs,
fn,
)
output |> E.R.bind(_, coersionToExportedTypes(inputs, inputs.environment))
}

298
packages/playground/src/distPlus/distribution/AlgebraicShapeCombination.re
View File

@ -1,298 +0,0 @@
type pointMassesWithMoments = {
n: int,
masses: array(float),
means: array(float),
variances: array(float),
};
/* This function takes a continuous distribution and efficiently approximates it as
point masses that have variances associated with them.
We estimate the means and variances from overlapping triangular distributions which we imagine are making up the
XYShape.
We can then use the algebra of random variables to "convolve" the point masses and their variances,
and finally reconstruct a new distribution from them, e.g. using a Fast Gauss Transform or Raykar et al. (2007). */
let toDiscretePointMassesFromTriangulars =
(~inverse=false, s: XYShape.T.t): pointMassesWithMoments => {
// TODO: what if there is only one point in the distribution?
let n = s |> XYShape.T.length;
// first, double up the leftmost and rightmost points:
let {xs, ys}: XYShape.T.t = s;
Js.Array.unshift(xs[0], xs) |> ignore;
Js.Array.unshift(ys[0], ys) |> ignore;
Js.Array.push(xs[n - 1], xs) |> ignore;
Js.Array.push(ys[n - 1], ys) |> ignore;
let n = E.A.length(xs);
// squares and neighbourly products of the xs
let xsSq: array(float) = Belt.Array.makeUninitializedUnsafe(n);
let xsProdN1: array(float) = Belt.Array.makeUninitializedUnsafe(n - 1);
let xsProdN2: array(float) = Belt.Array.makeUninitializedUnsafe(n - 2);
for (i in 0 to n - 1) {
Belt.Array.set(xsSq, i, xs[i] *. xs[i]) |> ignore;
();
};
for (i in 0 to n - 2) {
Belt.Array.set(xsProdN1, i, xs[i] *. xs[i + 1]) |> ignore;
();
};
for (i in 0 to n - 3) {
Belt.Array.set(xsProdN2, i, xs[i] *. xs[i + 2]) |> ignore;
();
};
// means and variances
let masses: array(float) = Belt.Array.makeUninitializedUnsafe(n - 2); // doesn't include the fake first and last points
let means: array(float) = Belt.Array.makeUninitializedUnsafe(n - 2);
let variances: array(float) = Belt.Array.makeUninitializedUnsafe(n - 2);
if (inverse) {
for (i in 1 to n - 2) {
Belt.Array.set(masses, i - 1, (xs[i + 1] -. xs[i - 1]) *. ys[i] /. 2.)
|> ignore;
// this only works when the whole triange is either on the left or on the right of zero
let a = xs[i - 1];
let c = xs[i];
let b = xs[i + 1];
// These are the moments of the reciprocal of a triangular distribution, as symbolically integrated by Mathematica.
// They're probably pretty close to invMean ~ 1/mean = 3/(a+b+c) and invVar. But I haven't worked out
// the worst case error, so for now let's use these monster equations
let inverseMean =
2.
*. (a *. log(a /. c) /. (a -. c) +. b *. log(c /. b) /. (b -. c))
/. (a -. b);
let inverseVar =
2.
*. (log(c /. a) /. (a -. c) +. b *. log(b /. c) /. (b -. c))
/. (a -. b)
-. inverseMean
** 2.;
Belt.Array.set(means, i - 1, inverseMean) |> ignore;
Belt.Array.set(variances, i - 1, inverseVar) |> ignore;
();
};
{n: n - 2, masses, means, variances};
} else {
for (i in 1 to n - 2) {
// area of triangle = width * height / 2
Belt.Array.set(masses, i - 1, (xs[i + 1] -. xs[i - 1]) *. ys[i] /. 2.)
|> ignore;
// means of triangle = (a + b + c) / 3
Belt.Array.set(means, i - 1, (xs[i - 1] +. xs[i] +. xs[i + 1]) /. 3.)
|> ignore;
// variance of triangle = (a^2 + b^2 + c^2 - ab - ac - bc) / 18
Belt.Array.set(
variances,
i - 1,
(
xsSq[i - 1]
+. xsSq[i]
+. xsSq[i + 1]
-. xsProdN1[i - 1]
-. xsProdN1[i]
-. xsProdN2[i - 1]
)
/. 18.,
)
|> ignore;
();
};
{n: n - 2, masses, means, variances};
};
};
let combineShapesContinuousContinuous =
(
op: ExpressionTypes.algebraicOperation,
s1: DistTypes.xyShape,
s2: DistTypes.xyShape,
)
: DistTypes.xyShape => {
let t1n = s1 |> XYShape.T.length;
let t2n = s2 |> XYShape.T.length;
// if we add the two distributions, we should probably use normal filters.
// if we multiply the two distributions, we should probably use lognormal filters.
let t1m = toDiscretePointMassesFromTriangulars(s1);
let t2m =
switch (op) {
| `Divide => toDiscretePointMassesFromTriangulars(~inverse=true, s2)
| _ => toDiscretePointMassesFromTriangulars(~inverse=false, s2)
};
let combineMeansFn =
switch (op) {
| `Add => ((m1, m2) => m1 +. m2)
| `Subtract => ((m1, m2) => m1 -. m2)
| `Multiply => ((m1, m2) => m1 *. m2)
| `Divide => ((m1, mInv2) => m1 *. mInv2)
| `Exponentiate => ((m1, mInv2) => m1 ** mInv2)
}; // note: here, mInv2 = mean(1 / t2) ~= 1 / mean(t2)
// TODO: I don't know what the variances are for exponentatiation
// converts the variances and means of the two inputs into the variance of the output
let combineVariancesFn =
switch (op) {
| `Add => ((v1, v2, _, _) => v1 +. v2)
| `Subtract => ((v1, v2, _, _) => v1 +. v2)
| `Multiply => (
(v1, v2, m1, m2) => v1 *. v2 +. v1 *. m2 ** 2. +. v2 *. m1 ** 2.
)
| `Exponentiate =>
((v1, v2, m1, m2) => v1 *. v2 +. v1 *. m2 ** 2. +. v2 *. m1 ** 2.);
| `Divide => (
(v1, vInv2, m1, mInv2) =>
v1 *. vInv2 +. v1 *. mInv2 ** 2. +. vInv2 *. m1 ** 2.
)
};
// TODO: If operating on two positive-domain distributions, we should take that into account
let outputMinX: ref(float) = ref(infinity);
let outputMaxX: ref(float) = ref(neg_infinity);
let masses: array(float) =
Belt.Array.makeUninitializedUnsafe(t1m.n * t2m.n);
let means: array(float) =
Belt.Array.makeUninitializedUnsafe(t1m.n * t2m.n);
let variances: array(float) =
Belt.Array.makeUninitializedUnsafe(t1m.n * t2m.n);
// then convolve the two sets of pointMassesWithMoments
for (i in 0 to t1m.n - 1) {
for (j in 0 to t2m.n - 1) {
let k = i * t2m.n + j;
Belt.Array.set(masses, k, t1m.masses[i] *. t2m.masses[j]) |> ignore;
let mean = combineMeansFn(t1m.means[i], t2m.means[j]);
let variance =
combineVariancesFn(
t1m.variances[i],
t2m.variances[j],
t1m.means[i],
t2m.means[j],
);
Belt.Array.set(means, k, mean) |> ignore;
Belt.Array.set(variances, k, variance) |> ignore;
// update bounds
let minX = mean -. 2. *. sqrt(variance) *. 1.644854;
let maxX = mean +. 2. *. sqrt(variance) *. 1.644854;
if (minX < outputMinX^) {
outputMinX := minX;
};
if (maxX > outputMaxX^) {
outputMaxX := maxX;
};
};
};
// we now want to create a set of target points. For now, let's just evenly distribute 200 points between
// between the outputMinX and outputMaxX
let nOut = 300;
let outputXs: array(float) =
E.A.Floats.range(outputMinX^, outputMaxX^, nOut);
let outputYs: array(float) = Belt.Array.make(nOut, 0.0);
// now, for each of the outputYs, accumulate from a Gaussian kernel over each input point.
for (j in 0 to E.A.length(masses) - 1) {
// go through all of the result points
if (variances[j] > 0. && masses[j] > 0.) {
for (i in 0 to E.A.length(outputXs) - 1) {
// go through all of the target points
let dx = outputXs[i] -. means[j];
let contribution =
masses[j]
*. exp(-. (dx ** 2.) /. (2. *. variances[j]))
/. sqrt(2. *. 3.14159276 *. variances[j]);
Belt.Array.set(outputYs, i, outputYs[i] +. contribution) |> ignore;
};
};
};
{xs: outputXs, ys: outputYs};
};
let toDiscretePointMassesFromDiscrete =
(s: DistTypes.xyShape): pointMassesWithMoments => {
let {xs, ys}: XYShape.T.t = s;
let n = E.A.length(xs);
let masses: array(float) = Belt.Array.makeBy(n, i => ys[i]);
let means: array(float) = Belt.Array.makeBy(n, i => xs[i]);
let variances: array(float) = Belt.Array.makeBy(n, i => 0.0);
{n, masses, means, variances};
};
let combineShapesContinuousDiscrete =
(
op: ExpressionTypes.algebraicOperation,
continuousShape: DistTypes.xyShape,
discreteShape: DistTypes.xyShape,
)
: DistTypes.xyShape => {
let t1n = continuousShape |> XYShape.T.length;
let t2n = discreteShape |> XYShape.T.length;
// each x pair is added/subtracted
let fn = Operation.Algebraic.toFn(op);
let outXYShapes: array(array((float, float))) =
Belt.Array.makeUninitializedUnsafe(t2n);
switch (op) {
| `Add
| `Subtract =>
for (j in 0 to t2n - 1) {
// creates a new continuous shape for each one of the discrete points, and collects them in outXYShapes.
let dxyShape: array((float, float)) =
Belt.Array.makeUninitializedUnsafe(t1n);
for (i in 0 to t1n - 1) {
Belt.Array.set(
dxyShape,
i,
(
fn(continuousShape.xs[i], discreteShape.xs[j]),
continuousShape.ys[i] *. discreteShape.ys[j],
),
)
|> ignore;
();
};
Belt.Array.set(outXYShapes, j, dxyShape) |> ignore;
();
}
| `Multiply
| `Exponentiate
| `Divide =>
for (j in 0 to t2n - 1) {
// creates a new continuous shape for each one of the discrete points, and collects them in outXYShapes.
let dxyShape: array((float, float)) =
Belt.Array.makeUninitializedUnsafe(t1n);
for (i in 0 to t1n - 1) {
Belt.Array.set(
dxyShape,
i,
(
fn(continuousShape.xs[i], discreteShape.xs[j]),
{continuousShape.ys[i] *. discreteShape.ys[j] /. discreteShape.xs[j]}
),
)
|> ignore;
();
};
Belt.Array.set(outXYShapes, j, dxyShape) |> ignore;
();
}
};
outXYShapes
|> E.A.fmap(XYShape.T.fromZippedArray)
|> E.A.fold_left(
XYShape.PointwiseCombination.combine(
(+.),
XYShape.XtoY.continuousInterpolator(`Linear, `UseZero),
),
XYShape.T.empty,
);
};

332
packages/playground/src/distPlus/distribution/Continuous.re
View File

@ -1,332 +0,0 @@
open Distributions;
type t = DistTypes.continuousShape;
let getShape = (t: t) => t.xyShape;
let interpolation = (t: t) => t.interpolation;
let make =
(
~interpolation=`Linear,
~integralSumCache=None,
~integralCache=None,
xyShape,
)
: t => {
xyShape,
interpolation,
integralSumCache,
integralCache,
};
let shapeMap =
(fn, {xyShape, interpolation, integralSumCache, integralCache}: t): t => {
xyShape: fn(xyShape),
interpolation,
integralSumCache,
integralCache,
};
let lastY = (t: t) => t |> getShape |> XYShape.T.lastY;
let oShapeMap =
(fn, {xyShape, interpolation, integralSumCache, integralCache}: t)
: option(DistTypes.continuousShape) =>
fn(xyShape)
|> E.O.fmap(make(~interpolation, ~integralSumCache, ~integralCache));
let emptyIntegral: DistTypes.continuousShape = {
xyShape: {
xs: [|neg_infinity|],
ys: [|0.0|],
},
interpolation: `Linear,
integralSumCache: Some(0.0),
integralCache: None,
};
let empty: DistTypes.continuousShape = {
xyShape: XYShape.T.empty,
interpolation: `Linear,
integralSumCache: Some(0.0),
integralCache: Some(emptyIntegral),
};
let stepwiseToLinear = (t: t): t =>
make(
~integralSumCache=t.integralSumCache,
~integralCache=t.integralCache,
XYShape.Range.stepwiseToLinear(t.xyShape),
);
// Note: This results in a distribution with as many points as the sum of those in t1 and t2.
let combinePointwise =
(
~integralSumCachesFn=(_, _) => None,
~integralCachesFn: (t, t) => option(t)=(_, _) => None,
~distributionType: DistTypes.distributionType=`PDF,
fn: (float, float) => float,
t1: DistTypes.continuousShape,
t2: DistTypes.continuousShape,
)
: DistTypes.continuousShape => {
// If we're adding the distributions, and we know the total of each, then we
// can just sum them up. Otherwise, all bets are off.
let combinedIntegralSum =
Common.combineIntegralSums(
integralSumCachesFn,
t1.integralSumCache,
t2.integralSumCache,
);
// TODO: does it ever make sense to pointwise combine the integrals here?
// It could be done for pointwise additions, but is that ever needed?
// If combining stepwise and linear, we must convert the stepwise to linear first,
// i.e. add a point at the bottom of each step
let (t1, t2) =
switch (t1.interpolation, t2.interpolation) {
| (`Linear, `Linear) => (t1, t2)
| (`Stepwise, `Stepwise) => (t1, t2)
| (`Linear, `Stepwise) => (t1, stepwiseToLinear(t2))
| (`Stepwise, `Linear) => (stepwiseToLinear(t1), t2)
};
let extrapolation =
switch (distributionType) {
| `PDF => `UseZero
| `CDF => `UseOutermostPoints
};
let interpolator =
XYShape.XtoY.continuousInterpolator(t1.interpolation, extrapolation);
make(
~integralSumCache=combinedIntegralSum,
XYShape.PointwiseCombination.combine(
fn,
interpolator,
t1.xyShape,
t2.xyShape,
),
);
};
let toLinear = (t: t): option(t) => {
switch (t) {
| {interpolation: `Stepwise, xyShape, integralSumCache, integralCache} =>
xyShape
|> XYShape.Range.stepsToContinuous
|> E.O.fmap(make(~integralSumCache, ~integralCache))
| {interpolation: `Linear} => Some(t)
};
};
let shapeFn = (fn, t: t) => t |> getShape |> fn;
let updateIntegralSumCache = (integralSumCache, t: t): t => {
...t,
integralSumCache,
};
let updateIntegralCache = (integralCache, t: t): t => {...t, integralCache};
let reduce =
(
~integralSumCachesFn: (float, float) => option(float)=(_, _) => None,
~integralCachesFn: (t, t) => option(t)=(_, _) => None,
fn,
continuousShapes,
) =>
continuousShapes
|> E.A.fold_left(
combinePointwise(~integralSumCachesFn, ~integralCachesFn, fn),
empty,
);
let mapY =
(~integralSumCacheFn=_ => None, ~integralCacheFn=_ => None, ~fn, t: t) => {
make(
~interpolation=t.interpolation,
~integralSumCache=t.integralSumCache |> E.O.bind(_, integralSumCacheFn),
~integralCache=t.integralCache |> E.O.bind(_, integralCacheFn),
t |> getShape |> XYShape.T.mapY(fn),
);
};
let rec scaleBy = (~scale=1.0, t: t): t => {
let scaledIntegralSumCache =
E.O.bind(t.integralSumCache, v => Some(scale *. v));
let scaledIntegralCache =
E.O.bind(t.integralCache, v => Some(scaleBy(~scale, v)));
t
|> mapY(~fn=(r: float) => r *. scale)
|> updateIntegralSumCache(scaledIntegralSumCache)
|> updateIntegralCache(scaledIntegralCache);
};
module T =
Dist({
type t = DistTypes.continuousShape;
type integral = DistTypes.continuousShape;
let minX = shapeFn(XYShape.T.minX);
let maxX = shapeFn(XYShape.T.maxX);
let mapY = mapY;
let updateIntegralCache = updateIntegralCache;
let toDiscreteProbabilityMassFraction = _ => 0.0;
let toShape = (t: t): DistTypes.shape => Continuous(t);
let xToY = (f, {interpolation, xyShape}: t) => {
(
switch (interpolation) {
| `Stepwise =>
xyShape |> XYShape.XtoY.stepwiseIncremental(f) |> E.O.default(0.0)
| `Linear => xyShape |> XYShape.XtoY.linear(f)
}
)
|> DistTypes.MixedPoint.makeContinuous;
};
let truncate =
(leftCutoff: option(float), rightCutoff: option(float), t: t) => {
let lc = E.O.default(neg_infinity, leftCutoff);
let rc = E.O.default(infinity, rightCutoff);
let truncatedZippedPairs =
t
|> getShape
|> XYShape.T.zip
|> XYShape.Zipped.filterByX(x => x >= lc && x <= rc);
let leftNewPoint =
leftCutoff
|> E.O.dimap(lc => [|(lc -. epsilon_float, 0.)|], _ => [||]);
let rightNewPoint =
rightCutoff
|> E.O.dimap(rc => [|(rc +. epsilon_float, 0.)|], _ => [||]);
let truncatedZippedPairsWithNewPoints =
E.A.concatMany([|leftNewPoint, truncatedZippedPairs, rightNewPoint|]);
let truncatedShape =
XYShape.T.fromZippedArray(truncatedZippedPairsWithNewPoints);
make(truncatedShape);
};
// TODO: This should work with stepwise plots.
let integral = t =>
switch (getShape(t) |> XYShape.T.isEmpty, t.integralCache) {
| (true, _) => emptyIntegral
| (false, Some(cache)) => cache
| (false, None) =>
t
|> getShape
|> XYShape.Range.integrateWithTriangles
|> E.O.toExt("This should not have happened")
|> make
};
let downsample = (length, t): t =>
t
|> shapeMap(
XYShape.XsConversion.proportionByProbabilityMass(
length,
integral(t).xyShape,
),
);
let integralEndY = (t: t) =>
t.integralSumCache |> E.O.default(t |> integral |> lastY);
let integralXtoY = (f, t: t) =>
t |> integral |> shapeFn(XYShape.XtoY.linear(f));
let integralYtoX = (f, t: t) =>
t |> integral |> shapeFn(XYShape.YtoX.linear(f));
let toContinuous = t => Some(t);
let toDiscrete = _ => None;
let normalize = (t: t): t => {
t
|> updateIntegralCache(Some(integral(t)))
|> scaleBy(~scale=1. /. integralEndY(t))
|> updateIntegralSumCache(Some(1.0));
};
let mean = (t: t) => {
let indefiniteIntegralStepwise = (p, h1) => h1 *. p ** 2.0 /. 2.0;
let indefiniteIntegralLinear = (p, a, b) =>
a *. p ** 2.0 /. 2.0 +. b *. p ** 3.0 /. 3.0;
XYShape.Analysis.integrateContinuousShape(
~indefiniteIntegralStepwise,
~indefiniteIntegralLinear,
t,
);
};
let variance = (t: t): float =>
XYShape.Analysis.getVarianceDangerously(
t,
mean,
XYShape.Analysis.getMeanOfSquaresContinuousShape,
);
});
/* This simply creates multiple copies of the continuous distribution, scaled and shifted according to
each discrete data point, and then adds them all together. */
let combineAlgebraicallyWithDiscrete =
(
op: ExpressionTypes.algebraicOperation,
t1: t,
t2: DistTypes.discreteShape,
) => {
let t1s = t1 |> getShape;
let t2s = t2.xyShape; // TODO would like to use Discrete.getShape here, but current file structure doesn't allow for that
if (XYShape.T.isEmpty(t1s) || XYShape.T.isEmpty(t2s)) {
empty;
} else {
let continuousAsLinear =
switch (t1.interpolation) {
| `Linear => t1
| `Stepwise => stepwiseToLinear(t1)
};
let combinedShape =
AlgebraicShapeCombination.combineShapesContinuousDiscrete(
op,
continuousAsLinear |> getShape,
t2s,
);
let combinedIntegralSum =
switch (op) {
| `Multiply
| `Divide =>
Common.combineIntegralSums(
(a, b) => Some(a *. b),
t1.integralSumCache,
t2.integralSumCache,
)
| _ => None
};
// TODO: It could make sense to automatically transform the integrals here (shift or scale)
make(
~interpolation=t1.interpolation,
~integralSumCache=combinedIntegralSum,
combinedShape,
);
};
};
let combineAlgebraically =
(op: ExpressionTypes.algebraicOperation, t1: t, t2: t) => {
let s1 = t1 |> getShape;
let s2 = t2 |> getShape;
let t1n = s1 |> XYShape.T.length;
let t2n = s2 |> XYShape.T.length;
if (t1n == 0 || t2n == 0) {
empty;
} else {
let combinedShape =
AlgebraicShapeCombination.combineShapesContinuousContinuous(op, s1, s2);
let combinedIntegralSum =
Common.combineIntegralSums(
(a, b) => Some(a *. b),
t1.integralSumCache,
t2.integralSumCache,
);
// return a new Continuous distribution
make(~integralSumCache=combinedIntegralSum, combinedShape);
};
};

232
packages/playground/src/distPlus/distribution/Discrete.re
View File

@ -1,232 +0,0 @@
open Distributions;
type t = DistTypes.discreteShape;
let make = (~integralSumCache=None, ~integralCache=None, xyShape): t => {xyShape, integralSumCache, integralCache};
let shapeMap = (fn, {xyShape, integralSumCache, integralCache}: t): t => {
xyShape: fn(xyShape),
integralSumCache,
integralCache
};
let getShape = (t: t) => t.xyShape;
let oShapeMap = (fn, {xyShape, integralSumCache, integralCache}: t): option(t) =>
fn(xyShape) |> E.O.fmap(make(~integralSumCache, ~integralCache));
let emptyIntegral: DistTypes.continuousShape = {
xyShape: {xs: [|neg_infinity|], ys: [|0.0|]},
interpolation: `Stepwise,
integralSumCache: Some(0.0),
integralCache: None,
};
let empty: DistTypes.discreteShape = {
xyShape: XYShape.T.empty,
integralSumCache: Some(0.0),
integralCache: Some(emptyIntegral),
};
let shapeFn = (fn, t: t) => t |> getShape |> fn;
let lastY = (t: t) => t |> getShape |> XYShape.T.lastY;
let combinePointwise =
(
~integralSumCachesFn = (_, _) => None,
~integralCachesFn: (DistTypes.continuousShape, DistTypes.continuousShape) => option(DistTypes.continuousShape) = (_, _) => None,
fn,
t1: DistTypes.discreteShape,
t2: DistTypes.discreteShape,
)
: DistTypes.discreteShape => {
let combinedIntegralSum =
Common.combineIntegralSums(
integralSumCachesFn,
t1.integralSumCache,
t2.integralSumCache,
);
// TODO: does it ever make sense to pointwise combine the integrals here?
// It could be done for pointwise additions, but is that ever needed?
make(
~integralSumCache=combinedIntegralSum,
XYShape.PointwiseCombination.combine(
(+.),
XYShape.XtoY.discreteInterpolator,
t1.xyShape,
t2.xyShape,
),
);
};
let reduce =
(~integralSumCachesFn=(_, _) => None,
~integralCachesFn=(_, _) => None,
fn, discreteShapes)
: DistTypes.discreteShape =>
discreteShapes
|> E.A.fold_left(combinePointwise(~integralSumCachesFn, ~integralCachesFn, fn), empty);
let updateIntegralSumCache = (integralSumCache, t: t): t => {
...t,
integralSumCache,
};
let updateIntegralCache = (integralCache, t: t): t => {
...t,
integralCache,
};
/* This multiples all of the data points together and creates a new discrete distribution from the results.
Data points at the same xs get added together. It may be a good idea to downsample t1 and t2 before and/or the result after. */
let combineAlgebraically =
(op: ExpressionTypes.algebraicOperation, t1: t, t2: t): t => {
let t1s = t1 |> getShape;
let t2s = t2 |> getShape;
let t1n = t1s |> XYShape.T.length;
let t2n = t2s |> XYShape.T.length;
let combinedIntegralSum =
Common.combineIntegralSums(
(s1, s2) => Some(s1 *. s2),
t1.integralSumCache,
t2.integralSumCache,
);
let fn = Operation.Algebraic.toFn(op);
let xToYMap = E.FloatFloatMap.empty();
for (i in 0 to t1n - 1) {
for (j in 0 to t2n - 1) {
let x = fn(t1s.xs[i], t2s.xs[j]);
let cv = xToYMap |> E.FloatFloatMap.get(x) |> E.O.default(0.);
let my = t1s.ys[i] *. t2s.ys[j];
let _ = Belt.MutableMap.set(xToYMap, x, cv +. my);
();
};
};
let rxys = xToYMap |> E.FloatFloatMap.toArray |> XYShape.Zipped.sortByX;
let combinedShape = XYShape.T.fromZippedArray(rxys);
make(~integralSumCache=combinedIntegralSum, combinedShape);
};
let mapY = (~integralSumCacheFn=_ => None,
~integralCacheFn=_ => None,
~fn, t: t) => {
make(
~integralSumCache=t.integralSumCache |> E.O.bind(_, integralSumCacheFn),
~integralCache=t.integralCache |> E.O.bind(_, integralCacheFn),
t |> getShape |> XYShape.T.mapY(fn),
);
};
let scaleBy = (~scale=1.0, t: t): t => {
let scaledIntegralSumCache = t.integralSumCache |> E.O.fmap((*.)(scale));
let scaledIntegralCache = t.integralCache |> E.O.fmap(Continuous.scaleBy(~scale));
t
|> mapY(~fn=(r: float) => r *. scale)
|> updateIntegralSumCache(scaledIntegralSumCache)
|> updateIntegralCache(scaledIntegralCache)
};
module T =
Dist({
type t = DistTypes.discreteShape;
type integral = DistTypes.continuousShape;
let integral = (t) =>
switch (getShape(t) |> XYShape.T.isEmpty, t.integralCache) {
| (true, _) => emptyIntegral
| (false, Some(c)) => c
| (false, None) => {
let ts = getShape(t);
// The first xy of this integral should always be the zero, to ensure nice plotting
let firstX = ts |> XYShape.T.minX;
let prependedZeroPoint: XYShape.T.t = {xs: [|firstX -. epsilon_float|], ys: [|0.|]};
let integralShape =
ts
|> XYShape.T.concat(prependedZeroPoint)
|> XYShape.T.accumulateYs((+.));
Continuous.make(~interpolation=`Stepwise, integralShape);
}
};
let integralEndY = (t: t) =>
t.integralSumCache
|> E.O.default(t |> integral |> Continuous.lastY);
let minX = shapeFn(XYShape.T.minX);
let maxX = shapeFn(XYShape.T.maxX);
let toDiscreteProbabilityMassFraction = _ => 1.0;
let mapY = mapY;
let updateIntegralCache = updateIntegralCache;
let toShape = (t: t): DistTypes.shape => Discrete(t);
let toContinuous = _ => None;
let toDiscrete = t => Some(t);
let normalize = (t: t): t => {
t
|> scaleBy(~scale=1. /. integralEndY(t))
|> updateIntegralSumCache(Some(1.0));
};
let downsample = (i, t: t): t => {
// It's not clear how to downsample a set of discrete points in a meaningful way.
// The best we can do is to clip off the smallest values.
let currentLength = t |> getShape |> XYShape.T.length;
if (i < currentLength && i >= 1 && currentLength > 1) {
t
|> getShape
|> XYShape.T.zip
|> XYShape.Zipped.sortByY
|> Belt.Array.reverse
|> Belt.Array.slice(_, ~offset=0, ~len=i)
|> XYShape.Zipped.sortByX
|> XYShape.T.fromZippedArray
|> make;
} else {
t;
};
};
let truncate =
(leftCutoff: option(float), rightCutoff: option(float), t: t): t => {
t
|> getShape
|> XYShape.T.zip
|> XYShape.Zipped.filterByX(x =>
x >= E.O.default(neg_infinity, leftCutoff)
&& x <= E.O.default(infinity, rightCutoff)
)
|> XYShape.T.fromZippedArray
|> make;
};
let xToY = (f, t) =>
t
|> getShape
|> XYShape.XtoY.stepwiseIfAtX(f)
|> E.O.default(0.0)
|> DistTypes.MixedPoint.makeDiscrete;
let integralXtoY = (f, t) =>
t |> integral |> Continuous.getShape |> XYShape.XtoY.linear(f);
let integralYtoX = (f, t) =>
t |> integral |> Continuous.getShape |> XYShape.YtoX.linear(f);
let mean = (t: t): float => {
let s = getShape(t);
E.A.reducei(s.xs, 0.0, (acc, x, i) => acc +. x *. s.ys[i]);
};
let variance = (t: t): float => {
let getMeanOfSquares = t =>
t |> shapeMap(XYShape.Analysis.squareXYShape) |> mean;
XYShape.Analysis.getVarianceDangerously(t, mean, getMeanOfSquares);
};
});

129
packages/playground/src/distPlus/distribution/DistPlus.re
View File

@ -1,129 +0,0 @@
open DistTypes;
type t = DistTypes.distPlus;
let shapeIntegral = shape => Shape.T.Integral.get(shape);
let make =
(
~shape,
~squiggleString,
~domain=Complete,
~unit=UnspecifiedDistribution,
(),
)
: t => {
let integral = shapeIntegral(shape);
{shape, domain, integralCache: integral, unit, squiggleString};
};
let update =
(
~shape=?,
~integralCache=?,
~domain=?,
~unit=?,
~squiggleString=?,
t: t,
) => {
shape: E.O.default(t.shape, shape),
integralCache: E.O.default(t.integralCache, integralCache),
domain: E.O.default(t.domain, domain),
unit: E.O.default(t.unit, unit),
squiggleString: E.O.default(t.squiggleString, squiggleString),
};
let updateShape = (shape, t) => {
let integralCache = shapeIntegral(shape);
update(~shape, ~integralCache, t);
};
let domainIncludedProbabilityMass = (t: t) =>
Domain.includedProbabilityMass(t.domain);
let domainIncludedProbabilityMassAdjustment = (t: t, f) =>
f *. Domain.includedProbabilityMass(t.domain);
let toShape = ({shape, _}: t) => shape;
let shapeFn = (fn, {shape}: t) => fn(shape);
module T =
Distributions.Dist({
type t = DistTypes.distPlus;
type integral = DistTypes.distPlus;
let toShape = toShape;
let toContinuous = shapeFn(Shape.T.toContinuous);
let toDiscrete = shapeFn(Shape.T.toDiscrete);
let normalize = (t: t): t => {
let normalizedShape = t |> toShape |> Shape.T.normalize;
t |> updateShape(normalizedShape);
};
let truncate = (leftCutoff, rightCutoff, t: t): t => {
let truncatedShape =
t
|> toShape
|> Shape.T.truncate(leftCutoff, rightCutoff);
t |> updateShape(truncatedShape);
};
let xToY = (f, t: t) =>
t
|> toShape
|> Shape.T.xToY(f)
|> MixedPoint.fmap(domainIncludedProbabilityMassAdjustment(t));
let minX = shapeFn(Shape.T.minX);
let maxX = shapeFn(Shape.T.maxX);
let toDiscreteProbabilityMassFraction =
shapeFn(Shape.T.toDiscreteProbabilityMassFraction);
// This bit is kind of awkward, could probably use rethinking.
let integral = (t: t) =>
updateShape(Continuous(t.integralCache), t);
let updateIntegralCache = (integralCache: option(DistTypes.continuousShape), t) =>
update(~integralCache=E.O.default(t.integralCache, integralCache), t);
let downsample = (i, t): t =>
updateShape(t |> toShape |> Shape.T.downsample(i), t);
// todo: adjust for limit, maybe?
let mapY =
(
~integralSumCacheFn=previousIntegralSum => None,
~integralCacheFn=previousIntegralCache => None,
~fn,
{shape, _} as t: t,
)
: t =>
Shape.T.mapY(~integralSumCacheFn, ~fn, shape)
|> updateShape(_, t);
// get the total of everything
let integralEndY = (t: t) => {
Shape.T.Integral.sum(
toShape(t),
);
};
// TODO: Fix this below, obviously. Adjust for limits
let integralXtoY = (f, t: t) => {
Shape.T.Integral.xToY(
f,
toShape(t),
)
|> domainIncludedProbabilityMassAdjustment(t);
};
// TODO: This part is broken when there is a limit, if this is supposed to be taken into account.
let integralYtoX = (f, t: t) => {
Shape.T.Integral.yToX(f, toShape(t));
};
let mean = (t: t) => {
Shape.T.mean(t.shape);
};
let variance = (t: t) => Shape.T.variance(t.shape);
});

28
packages/playground/src/distPlus/distribution/DistPlusTime.re
View File

@ -1,28 +0,0 @@
open DistTypes;
type t = DistTypes.distPlus;
let unitToJson = ({unit}: t) => unit |> DistTypes.DistributionUnit.toJson;
let timeVector = ({unit}: t) =>
switch (unit) {
| TimeDistribution(timeVector) => Some(timeVector)
| UnspecifiedDistribution => None
};
let timeInVectorToX = (f: TimeTypes.timeInVector, t: t) => {
let timeVector = t |> timeVector;
timeVector |> E.O.fmap(TimeTypes.RelativeTimePoint.toXValue(_, f));
};
let xToY = (f: TimeTypes.timeInVector, t: t) => {
timeInVectorToX(f, t) |> E.O.fmap(DistPlus.T.xToY(_, t));
};
module Integral = {
include DistPlus.T.Integral;
let xToY = (f: TimeTypes.timeInVector, t: t) => {
timeInVectorToX(f, t)
|> E.O.fmap(x => DistPlus.T.Integral.xToY(x, t));
};
};

179
packages/playground/src/distPlus/distribution/DistTypes.re
View File

@ -1,179 +0,0 @@
type domainLimit = {
xPoint: float,
excludingProbabilityMass: float,
};
type domain =
| Complete
| LeftLimited(domainLimit)
| RightLimited(domainLimit)
| LeftAndRightLimited(domainLimit, domainLimit);
type distributionType = [
| `PDF
| `CDF
];
type xyShape = {
xs: array(float),
ys: array(float),
};
type interpolationStrategy = [
| `Stepwise
| `Linear
];
type extrapolationStrategy = [
| `UseZero
| `UseOutermostPoints
];
type interpolator = (xyShape, int, float) => float;
type continuousShape = {
xyShape,
interpolation: interpolationStrategy,
integralSumCache: option(float),
integralCache: option(continuousShape),
};
type discreteShape = {
xyShape,
integralSumCache: option(float),
integralCache: option(continuousShape),
};
type mixedShape = {
continuous: continuousShape,
discrete: discreteShape,
integralSumCache: option(float),
integralCache: option(continuousShape),
};
type shapeMonad('a, 'b, 'c) =
| Mixed('a)
| Discrete('b)
| Continuous('c);
type shape = shapeMonad(mixedShape, discreteShape, continuousShape);
module ShapeMonad = {
let fmap =
(t: shapeMonad('a, 'b, 'c), (fn1, fn2, fn3)): shapeMonad('d, 'e, 'f) =>
switch (t) {
| Mixed(m) => Mixed(fn1(m))
| Discrete(m) => Discrete(fn2(m))
| Continuous(m) => Continuous(fn3(m))
};
};
type generationSource =
| SquiggleString(string)
| Shape(shape);
type distributionUnit =
| UnspecifiedDistribution
| TimeDistribution(TimeTypes.timeVector);
type distPlus = {
shape,
domain,
integralCache: continuousShape,
unit: distributionUnit,
squiggleString: option(string),
};
module DistributionUnit = {
let toJson = (distributionUnit: distributionUnit) =>
switch (distributionUnit) {
| TimeDistribution({zero, unit}) =>
Js.Null.fromOption(
Some({"zero": zero, "unit": unit |> TimeTypes.TimeUnit.toString}),
)
| _ => Js.Null.fromOption(None)
};
};
module Domain = {
let excludedProbabilityMass = (t: domain) => {
switch (t) {
| Complete => 0.0
| LeftLimited({excludingProbabilityMass}) => excludingProbabilityMass
| RightLimited({excludingProbabilityMass}) => excludingProbabilityMass
| LeftAndRightLimited(
{excludingProbabilityMass: l},
{excludingProbabilityMass: r},
) =>
l +. r
};
};
let includedProbabilityMass = (t: domain) =>
1.0 -. excludedProbabilityMass(t);
let initialProbabilityMass = (t: domain) => {
switch (t) {
| Complete
| RightLimited(_) => 0.0
| LeftLimited({excludingProbabilityMass}) => excludingProbabilityMass
| LeftAndRightLimited({excludingProbabilityMass}, _) => excludingProbabilityMass
};
};
let normalizeProbabilityMass = (t: domain) => {
1. /. excludedProbabilityMass(t);
};
let yPointToSubYPoint = (t: domain, yPoint) => {
switch (t) {
| Complete => Some(yPoint)
| LeftLimited({excludingProbabilityMass})
when yPoint < excludingProbabilityMass =>
None
| LeftLimited({excludingProbabilityMass})
when yPoint >= excludingProbabilityMass =>
Some(
(yPoint -. excludingProbabilityMass) /. includedProbabilityMass(t),
)
| RightLimited({excludingProbabilityMass})
when yPoint > 1. -. excludingProbabilityMass =>
None
| RightLimited({excludingProbabilityMass})
when yPoint <= 1. -. excludingProbabilityMass =>
Some(yPoint /. includedProbabilityMass(t))
| LeftAndRightLimited({excludingProbabilityMass: l}, _) when yPoint < l =>
None
| LeftAndRightLimited(_, {excludingProbabilityMass: r})
when yPoint > 1.0 -. r =>
None
| LeftAndRightLimited({excludingProbabilityMass: l}, _) =>
Some((yPoint -. l) /. includedProbabilityMass(t))
| _ => None
};
};
};
type mixedPoint = {
continuous: float,
discrete: float,
};
module MixedPoint = {
type t = mixedPoint;
let toContinuousValue = (t: t) => t.continuous;
let toDiscreteValue = (t: t) => t.discrete;
let makeContinuous = (continuous: float): t => {continuous, discrete: 0.0};
let makeDiscrete = (discrete: float): t => {continuous: 0.0, discrete};
let fmap = (fn: float => float, t: t) => {
continuous: fn(t.continuous),
discrete: fn(t.discrete),
};
let combine2 = (fn, c: t, d: t): t => {
continuous: fn(c.continuous, d.continuous),
discrete: fn(c.discrete, d.discrete),
};
let add = combine2((a, b) => a +. b);
};

84
packages/playground/src/distPlus/distribution/Distributions.re
View File

@ -1,84 +0,0 @@
module type dist = {
type t;
type integral;
let minX: t => float;
let maxX: t => float;
let mapY:
(~integralSumCacheFn: float => option(float)=?, ~integralCacheFn: DistTypes.continuousShape => option(DistTypes.continuousShape)=?, ~fn: float => float, t) => t;
let xToY: (float, t) => DistTypes.mixedPoint;
let toShape: t => DistTypes.shape;
let toContinuous: t => option(DistTypes.continuousShape);
let toDiscrete: t => option(DistTypes.discreteShape);
let normalize: t => t;
let toDiscreteProbabilityMassFraction: t => float;
let downsample: (int, t) => t;
let truncate: (option(float), option(float), t) => t;
let updateIntegralCache: (option(DistTypes.continuousShape), t) => t;
let integral: (t) => integral;
let integralEndY: (t) => float;
let integralXtoY: (float, t) => float;
let integralYtoX: (float, t) => float;
let mean: t => float;
let variance: t => float;
};
module Dist = (T: dist) => {
type t = T.t;
type integral = T.integral;
let minX = T.minX;
let maxX = T.maxX;
let integral = T.integral;
let xTotalRange = (t: t) => maxX(t) -. minX(t);
let mapY = T.mapY;
let xToY = T.xToY;
let downsample = T.downsample;
let toShape = T.toShape;
let toDiscreteProbabilityMassFraction = T.toDiscreteProbabilityMassFraction;
let toContinuous = T.toContinuous;
let toDiscrete = T.toDiscrete;
let normalize = T.normalize;
let truncate = T.truncate;
let mean = T.mean;
let variance = T.variance;
let updateIntegralCache = T.updateIntegralCache;
module Integral = {
type t = T.integral;
let get = T.integral;
let xToY = T.integralXtoY;
let yToX = T.integralYtoX;
let sum = T.integralEndY;
};
};
module Common = {
let combineIntegralSums =
(
combineFn: (float, float) => option(float),
t1IntegralSumCache: option(float),
t2IntegralSumCache: option(float),
) => {
switch (t1IntegralSumCache, t2IntegralSumCache) {
| (None, _)
| (_, None) => None
| (Some(s1), Some(s2)) => combineFn(s1, s2)
};
};
let combineIntegrals =
(
combineFn: (DistTypes.continuousShape, DistTypes.continuousShape) => option(DistTypes.continuousShape),
t1IntegralCache: option(DistTypes.continuousShape),
t2IntegralCache: option(DistTypes.continuousShape),
) => {
switch (t1IntegralCache, t2IntegralCache) {
| (None, _)
| (_, None) => None
| (Some(s1), Some(s2)) => combineFn(s1, s2)
};
};
};

332
packages/playground/src/distPlus/distribution/Mixed.re
View File

@ -1,332 +0,0 @@
open Distributions;
type t = DistTypes.mixedShape;
let make = (~integralSumCache=None, ~integralCache=None, ~continuous, ~discrete): t => {continuous, discrete, integralSumCache, integralCache};
let totalLength = (t: t): int => {
let continuousLength =
t.continuous |> Continuous.getShape |> XYShape.T.length;
let discreteLength = t.discrete |> Discrete.getShape |> XYShape.T.length;
continuousLength + discreteLength;
};
let scaleBy = (~scale=1.0, t: t): t => {
let scaledDiscrete = Discrete.scaleBy(~scale, t.discrete);
let scaledContinuous = Continuous.scaleBy(~scale, t.continuous);
let scaledIntegralCache = E.O.bind(t.integralCache, v => Some(Continuous.scaleBy(~scale, v)));
let scaledIntegralSumCache = E.O.bind(t.integralSumCache, s => Some(s *. scale));
make(~discrete=scaledDiscrete, ~continuous=scaledContinuous, ~integralSumCache=scaledIntegralSumCache, ~integralCache=scaledIntegralCache);
};
let toContinuous = ({continuous}: t) => Some(continuous);
let toDiscrete = ({discrete}: t) => Some(discrete);
let updateIntegralCache = (integralCache, t: t): t => {
...t,
integralCache,
};
module T =
Dist({
type t = DistTypes.mixedShape;
type integral = DistTypes.continuousShape;
let minX = ({continuous, discrete}: t) => {
min(Continuous.T.minX(continuous), Discrete.T.minX(discrete));
};
let maxX = ({continuous, discrete}: t) =>
max(Continuous.T.maxX(continuous), Discrete.T.maxX(discrete));
let toShape = (t: t): DistTypes.shape => Mixed(t);
let updateIntegralCache = updateIntegralCache;
let toContinuous = toContinuous;
let toDiscrete = toDiscrete;
let truncate =
(
leftCutoff: option(float),
rightCutoff: option(float),
{discrete, continuous}: t,
) => {
let truncatedContinuous =
Continuous.T.truncate(leftCutoff, rightCutoff, continuous);
let truncatedDiscrete =
Discrete.T.truncate(leftCutoff, rightCutoff, discrete);
make(~integralSumCache=None, ~integralCache=None, ~discrete=truncatedDiscrete, ~continuous=truncatedContinuous);
};
let normalize = (t: t): t => {
let continuousIntegral = Continuous.T.Integral.get(t.continuous);
let discreteIntegral = Discrete.T.Integral.get(t.discrete);
let continuous = t.continuous |> Continuous.updateIntegralCache(Some(continuousIntegral));
let discrete = t.discrete |> Discrete.updateIntegralCache(Some(discreteIntegral));
let continuousIntegralSum =
Continuous.T.Integral.sum(continuous);
let discreteIntegralSum =
Discrete.T.Integral.sum(discrete);
let totalIntegralSum = continuousIntegralSum +. discreteIntegralSum;
let newContinuousSum = continuousIntegralSum /. totalIntegralSum;
let newDiscreteSum = discreteIntegralSum /. totalIntegralSum;
let normalizedContinuous =
continuous
|> Continuous.scaleBy(~scale=newContinuousSum /. continuousIntegralSum)
|> Continuous.updateIntegralSumCache(Some(newContinuousSum));
let normalizedDiscrete =
discrete
|> Discrete.scaleBy(~scale=newDiscreteSum /. discreteIntegralSum)
|> Discrete.updateIntegralSumCache(Some(newDiscreteSum));
make(~integralSumCache=Some(1.0), ~integralCache=None, ~continuous=normalizedContinuous, ~discrete=normalizedDiscrete);
};
let xToY = (x, t: t) => {
// This evaluates the mixedShape at x, interpolating if necessary.
// Note that we normalize entire mixedShape first.
let {continuous, discrete}: t = normalize(t);
let c = Continuous.T.xToY(x, continuous);
let d = Discrete.T.xToY(x, discrete);
DistTypes.MixedPoint.add(c, d); // "add" here just combines the two values into a single MixedPoint.
};
let toDiscreteProbabilityMassFraction = ({discrete, continuous}: t) => {
let discreteIntegralSum =
Discrete.T.Integral.sum(discrete);
let continuousIntegralSum =
Continuous.T.Integral.sum(continuous);
let totalIntegralSum = discreteIntegralSum +. continuousIntegralSum;
discreteIntegralSum /. totalIntegralSum;
};
let downsample = (count, t: t): t => {
// We will need to distribute the new xs fairly between the discrete and continuous shapes.
// The easiest way to do this is to simply go by the previous probability masses.
let discreteIntegralSum =
Discrete.T.Integral.sum(t.discrete);
let continuousIntegralSum =
Continuous.T.Integral.sum(t.continuous);
let totalIntegralSum = discreteIntegralSum +. continuousIntegralSum;
// TODO: figure out what to do when the totalIntegralSum is zero.
let downsampledDiscrete =
Discrete.T.downsample(
int_of_float(
float_of_int(count) *. (discreteIntegralSum /. totalIntegralSum),
),
t.discrete,
);
let downsampledContinuous =
Continuous.T.downsample(
int_of_float(
float_of_int(count) *. (continuousIntegralSum /. totalIntegralSum),
),
t.continuous,
);
{...t, discrete: downsampledDiscrete, continuous: downsampledContinuous};
};
let integral = (t: t) => {
switch (t.integralCache) {
| Some(cache) => cache
| None =>
// note: if the underlying shapes aren't normalized, then these integrals won't be either -- but that's the way it should be.
let continuousIntegral = Continuous.T.Integral.get(t.continuous);
let discreteIntegral = Continuous.stepwiseToLinear(Discrete.T.Integral.get(t.discrete));
Continuous.make(
XYShape.PointwiseCombination.combine(
(+.),
XYShape.XtoY.continuousInterpolator(`Linear, `UseOutermostPoints),
Continuous.getShape(continuousIntegral),
Continuous.getShape(discreteIntegral),
),
);
};
};
let integralEndY = (t: t) => {
t |> integral |> Continuous.lastY;
};
let integralXtoY = (f, t) => {
t |> integral |> Continuous.getShape |> XYShape.XtoY.linear(f);
};
let integralYtoX = (f, t) => {
t |> integral |> Continuous.getShape |> XYShape.YtoX.linear(f);
};
// This pipes all ys (continuous and discrete) through fn.
// If mapY is a linear operation, we might be able to update the integralSumCaches as well;
// if not, they'll be set to None.
let mapY =
(
~integralSumCacheFn=previousIntegralSum => None,
~integralCacheFn=previousIntegral => None,
~fn,
t: t,
)
: t => {
let yMappedDiscrete: DistTypes.discreteShape =
t.discrete
|> Discrete.T.mapY(~fn)
|> Discrete.updateIntegralSumCache(E.O.bind(t.discrete.integralSumCache, integralSumCacheFn))
|> Discrete.updateIntegralCache(E.O.bind(t.discrete.integralCache, integralCacheFn));
let yMappedContinuous: DistTypes.continuousShape =
t.continuous
|> Continuous.T.mapY(~fn)
|> Continuous.updateIntegralSumCache(E.O.bind(t.continuous.integralSumCache, integralSumCacheFn))
|> Continuous.updateIntegralCache(E.O.bind(t.continuous.integralCache, integralCacheFn));
{
discrete: yMappedDiscrete,
continuous: yMappedContinuous,
integralSumCache: E.O.bind(t.integralSumCache, integralSumCacheFn),
integralCache: E.O.bind(t.integralCache, integralCacheFn),
};
};
let mean = ({discrete, continuous}: t): float => {
let discreteMean = Discrete.T.mean(discrete);
let continuousMean = Continuous.T.mean(continuous);
// the combined mean is the weighted sum of the two:
let discreteIntegralSum = Discrete.T.Integral.sum(discrete);
let continuousIntegralSum = Continuous.T.Integral.sum(continuous);
let totalIntegralSum = discreteIntegralSum +. continuousIntegralSum;
(
discreteMean
*. discreteIntegralSum
+. continuousMean
*. continuousIntegralSum
)
/. totalIntegralSum;
};
let variance = ({discrete, continuous} as t: t): float => {
// the combined mean is the weighted sum of the two:
let discreteIntegralSum = Discrete.T.Integral.sum(discrete);
let continuousIntegralSum = Continuous.T.Integral.sum(continuous);
let totalIntegralSum = discreteIntegralSum +. continuousIntegralSum;
let getMeanOfSquares = ({discrete, continuous}: t) => {
let discreteMean =
discrete
|> Discrete.shapeMap(XYShape.Analysis.squareXYShape)
|> Discrete.T.mean;
let continuousMean =
continuous |> XYShape.Analysis.getMeanOfSquaresContinuousShape;
(
discreteMean
*. discreteIntegralSum
+. continuousMean
*. continuousIntegralSum
)
/. totalIntegralSum;
};
switch (discreteIntegralSum /. totalIntegralSum) {
| 1.0 => Discrete.T.variance(discrete)
| 0.0 => Continuous.T.variance(continuous)
| _ =>
XYShape.Analysis.getVarianceDangerously(t, mean, getMeanOfSquares)
};
};
});
let combineAlgebraically =
(op: ExpressionTypes.algebraicOperation, t1: t, t2: t)
: t => {
// Discrete convolution can cause a huge increase in the number of samples,
// so we'll first downsample.
// An alternative (to be explored in the future) may be to first perform the full convolution and then to downsample the result;
// to use non-uniform fast Fourier transforms (for addition only), add web workers or gpu.js, etc. ...
// we have to figure out where to downsample, and how to effectively
//let downsampleIfTooLarge = (t: t) => {
// let sqtl = sqrt(float_of_int(totalLength(t)));
// sqtl > 10 ? T.downsample(int_of_float(sqtl), t) : t;
//};
let t1d = t1;
let t2d = t2;
// continuous (*) continuous => continuous, but also
// discrete (*) continuous => continuous (and vice versa). We have to take care of all combos and then combine them:
let ccConvResult =
Continuous.combineAlgebraically(
op,
t1.continuous,
t2.continuous,
);
let dcConvResult =
Continuous.combineAlgebraicallyWithDiscrete(
op,
t2.continuous,
t1.discrete,
);
let cdConvResult =
Continuous.combineAlgebraicallyWithDiscrete(
op,
t1.continuous,
t2.discrete,
);
let continuousConvResult =
Continuous.reduce((+.), [|ccConvResult, dcConvResult, cdConvResult|]);
// ... finally, discrete (*) discrete => discrete, obviously:
let discreteConvResult =
Discrete.combineAlgebraically(op, t1.discrete, t2.discrete);
let combinedIntegralSum =
Common.combineIntegralSums(
(a, b) => Some(a *. b),
t1.integralSumCache,
t2.integralSumCache,
);
{discrete: discreteConvResult, continuous: continuousConvResult, integralSumCache: combinedIntegralSum, integralCache: None};
};
let combinePointwise = (~integralSumCachesFn = (_, _) => None, ~integralCachesFn = (_, _) => None, fn, t1: t, t2: t): t => {
let reducedDiscrete =
[|t1, t2|]
|> E.A.fmap(toDiscrete)
|> E.A.O.concatSomes
|> Discrete.reduce(~integralSumCachesFn, ~integralCachesFn, fn);
let reducedContinuous =
[|t1, t2|]
|> E.A.fmap(toContinuous)
|> E.A.O.concatSomes
|> Continuous.reduce(~integralSumCachesFn, ~integralCachesFn, fn);
let combinedIntegralSum =
Common.combineIntegralSums(
integralSumCachesFn,
t1.integralSumCache,
t2.integralSumCache,
);
let combinedIntegral =
Common.combineIntegrals(
integralCachesFn,
t1.integralCache,
t2.integralCache,
);
make(~integralSumCache=combinedIntegralSum, ~integralCache=combinedIntegral, ~discrete=reducedDiscrete, ~continuous=reducedContinuous);
};

34
packages/playground/src/distPlus/distribution/MixedShapeBuilder.re
View File

@ -1,34 +0,0 @@
type assumption =
| ADDS_TO_1
| ADDS_TO_CORRECT_PROBABILITY;
type assumptions = {
continuous: assumption,
discrete: assumption,
discreteProbabilityMass: option(float),
};
let buildSimple = (~continuous: option(DistTypes.continuousShape), ~discrete: option(DistTypes.discreteShape)): option(DistTypes.shape) => {
let continuous = continuous |> E.O.default(Continuous.make(~integralSumCache=Some(0.0), {xs: [||], ys: [||]}));
let discrete = discrete |> E.O.default(Discrete.make(~integralSumCache=Some(0.0), {xs: [||], ys: [||]}));
let cLength =
continuous
|> Continuous.getShape
|> XYShape.T.xs
|> E.A.length;
let dLength = discrete |> Discrete.getShape |> XYShape.T.xs |> E.A.length;
switch (cLength, dLength) {
| (0 | 1, 0) => None
| (0 | 1, _) => Some(Discrete(discrete))
| (_, 0) => Some(Continuous(continuous))
| (_, _) =>
let mixedDist =
Mixed.make(
~integralSumCache=None,
~integralCache=None,
~continuous,
~discrete,
);
Some(Mixed(mixedDist));
};
};

240
packages/playground/src/distPlus/distribution/Shape.re
View File

@ -1,240 +0,0 @@
open Distributions;
type t = DistTypes.shape;
let mapToAll = ((fn1, fn2, fn3), t: t) =>
switch (t) {
| Mixed(m) => fn1(m)
| Discrete(m) => fn2(m)
| Continuous(m) => fn3(m)
};
let fmap = ((fn1, fn2, fn3), t: t): t =>
switch (t) {
| Mixed(m) => Mixed(fn1(m))
| Discrete(m) => Discrete(fn2(m))
| Continuous(m) => Continuous(fn3(m))
};
let toMixed =
mapToAll((
m => m,
d => Mixed.make(~integralSumCache=d.integralSumCache, ~integralCache=d.integralCache, ~discrete=d, ~continuous=Continuous.empty),
c => Mixed.make(~integralSumCache=c.integralSumCache, ~integralCache=c.integralCache, ~discrete=Discrete.empty, ~continuous=c),
));
let combineAlgebraically =
(op: ExpressionTypes.algebraicOperation, t1: t, t2: t): t => {
switch (t1, t2) {
| (Continuous(m1), Continuous(m2)) =>
Continuous.combineAlgebraically(op, m1, m2) |> Continuous.T.toShape;
| (Continuous(m1), Discrete(m2))
| (Discrete(m2), Continuous(m1)) =>
Continuous.combineAlgebraicallyWithDiscrete(op, m1, m2) |> Continuous.T.toShape
| (Discrete(m1), Discrete(m2)) =>
Discrete.combineAlgebraically(op, m1, m2) |> Discrete.T.toShape
| (m1, m2) =>
Mixed.combineAlgebraically(
op,
toMixed(m1),
toMixed(m2),
)
|> Mixed.T.toShape
};
};
let combinePointwise =
(~integralSumCachesFn: (float, float) => option(float) = (_, _) => None,
~integralCachesFn: (DistTypes.continuousShape, DistTypes.continuousShape) => option(DistTypes.continuousShape) = (_, _) => None,
fn,
t1: t,
t2: t) =>
switch (t1, t2) {
| (Continuous(m1), Continuous(m2)) =>
DistTypes.Continuous(
Continuous.combinePointwise(~integralSumCachesFn, ~integralCachesFn, fn, m1, m2),
)
| (Discrete(m1), Discrete(m2)) =>
DistTypes.Discrete(
Discrete.combinePointwise(~integralSumCachesFn, ~integralCachesFn, fn, m1, m2),
)
| (m1, m2) =>
DistTypes.Mixed(
Mixed.combinePointwise(
~integralSumCachesFn,
~integralCachesFn,
fn,
toMixed(m1),
toMixed(m2),
),
)
};
module T =
Dist({
type t = DistTypes.shape;
type integral = DistTypes.continuousShape;
let xToY = (f: float) =>
mapToAll((
Mixed.T.xToY(f),
Discrete.T.xToY(f),
Continuous.T.xToY(f),
));
let toShape = (t: t) => t;
let toContinuous = t => None;
let toDiscrete = t => None;
let downsample = (i, t) =>
fmap(
(
Mixed.T.downsample(i),
Discrete.T.downsample(i),
Continuous.T.downsample(i),
),
t,
);
let truncate = (leftCutoff, rightCutoff, t): t =>
fmap(
(
Mixed.T.truncate(leftCutoff, rightCutoff),
Discrete.T.truncate(leftCutoff, rightCutoff),
Continuous.T.truncate(leftCutoff, rightCutoff),
),
t,
);
let toDiscreteProbabilityMassFraction = t => 0.0;
let normalize =
fmap((
Mixed.T.normalize,
Discrete.T.normalize,
Continuous.T.normalize
));
let updateIntegralCache = (integralCache, t: t): t =>
fmap((
Mixed.T.updateIntegralCache(integralCache),
Discrete.T.updateIntegralCache(integralCache),
Continuous.T.updateIntegralCache(integralCache),
), t);
let toContinuous =
mapToAll((
Mixed.T.toContinuous,
Discrete.T.toContinuous,
Continuous.T.toContinuous,
));
let toDiscrete =
mapToAll((
Mixed.T.toDiscrete,
Discrete.T.toDiscrete,
Continuous.T.toDiscrete,
));
let toDiscreteProbabilityMassFraction =
mapToAll((
Mixed.T.toDiscreteProbabilityMassFraction,
Discrete.T.toDiscreteProbabilityMassFraction,
Continuous.T.toDiscreteProbabilityMassFraction,
));
let minX = mapToAll((Mixed.T.minX, Discrete.T.minX, Continuous.T.minX));
let integral =
mapToAll((
Mixed.T.Integral.get,
Discrete.T.Integral.get,
Continuous.T.Integral.get,
));
let integralEndY =
mapToAll((
Mixed.T.Integral.sum,
Discrete.T.Integral.sum,
Continuous.T.Integral.sum,
));
let integralXtoY = (f) => {
mapToAll((
Mixed.T.Integral.xToY(f),
Discrete.T.Integral.xToY(f),
Continuous.T.Integral.xToY(f),
));
};
let integralYtoX = (f) => {
mapToAll((
Mixed.T.Integral.yToX(f),
Discrete.T.Integral.yToX(f),
Continuous.T.Integral.yToX(f),
));
};
let maxX = mapToAll((Mixed.T.maxX, Discrete.T.maxX, Continuous.T.maxX));
let mapY = (~integralSumCacheFn=previousIntegralSum => None, ~integralCacheFn=previousIntegral=>None, ~fn) =>{
fmap((
Mixed.T.mapY(~integralSumCacheFn, ~integralCacheFn, ~fn),
Discrete.T.mapY(~integralSumCacheFn, ~integralCacheFn, ~fn),
Continuous.T.mapY(~integralSumCacheFn, ~integralCacheFn, ~fn),
));
}
let mean = (t: t): float =>
switch (t) {
| Mixed(m) => Mixed.T.mean(m)
| Discrete(m) => Discrete.T.mean(m)
| Continuous(m) => Continuous.T.mean(m)
};
let variance = (t: t): float =>
switch (t) {
| Mixed(m) => Mixed.T.variance(m)
| Discrete(m) => Discrete.T.variance(m)
| Continuous(m) => Continuous.T.variance(m)
};
});
let pdf = (f: float, t: t) => {
let mixedPoint: DistTypes.mixedPoint = T.xToY(f, t);
mixedPoint.continuous +. mixedPoint.discrete;
};
let inv = T.Integral.yToX;
let cdf = T.Integral.xToY;
let doN = (n, fn) => {
let items = Belt.Array.make(n, 0.0);
for (x in 0 to n - 1) {
let _ = Belt.Array.set(items, x, fn());
();
};
items;
};
let sample = (t: t): float => {
let randomItem = Random.float(1.);
let bar = t |> T.Integral.yToX(randomItem);
bar;
};
let isFloat = (t:t) => switch(t){
| Discrete({xyShape: {xs: [|_|], ys: [|1.0|]}}) => true
| _ => false
}
let sampleNRendered = (n, dist) => {
let integralCache = T.Integral.get(dist);
let distWithUpdatedIntegralCache = T.updateIntegralCache(Some(integralCache), dist);
doN(n, () => sample(distWithUpdatedIntegralCache));
};
let operate = (distToFloatOp: ExpressionTypes.distToFloatOperation, s): float =>
switch (distToFloatOp) {
| `Pdf(f) => pdf(f, s)
| `Cdf(f) => pdf(f, s)
| `Inv(f) => inv(f, s)
| `Sample => sample(s)
| `Mean => T.mean(s)
};

83
packages/playground/src/distPlus/distribution/TimeTypes.res
View File

@ -1,83 +0,0 @@
type timeUnit = [
| #days
| #hours
| #milliseconds
| #minutes
| #months
| #quarters
| #seconds
| #weeks
| #years
]
type timeVector = {
zero: MomentRe.Moment.t,
unit: timeUnit,
}
type timePoint = {
timeVector: timeVector,
value: float,
}
module TimeUnit = {
let toString = (timeUnit: timeUnit) =>
switch timeUnit {
| #days => "days"
| #hours => "hours"
| #milliseconds => "milliseconds"
| #minutes => "minutes"
| #months => "months"
| #quarters => "quarters"
| #seconds => "seconds"
| #weeks => "weeks"
| #years => "years"
}
let ofString = (timeUnit: string) =>
switch timeUnit {
| "days" => #days
| "hours" => #hours
| "milliseconds" => #milliseconds
| "minutes" => #minutes
| "months" => #months
| "quarters" => #quarters
| "seconds" => #seconds
| "weeks" => #weeks
| "years" => #years
| _ => Js.Exn.raiseError("TimeUnit is unknown")
}
}
module TimePoint = {
let fromTimeVector = (timeVector, value): timePoint => {timeVector: timeVector, value: value}
let toMoment = (timePoint: timePoint) =>
timePoint.timeVector.zero |> MomentRe.Moment.add(
~duration=MomentRe.duration(timePoint.value, timePoint.timeVector.unit),
)
let fromMoment = (timeVector: timeVector, moment: MomentRe.Moment.t) =>
MomentRe.diff(timeVector.zero, moment, timeVector.unit)
}
type timeInVector =
| Time(MomentRe.Moment.t)
| XValue(float)
module RelativeTimePoint = {
let toTime = (timeVector: timeVector, timeInVector: timeInVector) =>
switch timeInVector {
| Time(r) => r
| XValue(r) =>
timeVector.zero |> MomentRe.Moment.add(~duration=MomentRe.duration(r, timeVector.unit))
}
let _timeToX = (time, timeStart, timeUnit) => MomentRe.diff(time, timeStart, timeUnit)
let toXValue = (timeVector: timeVector, timeInVector: timeInVector) =>
switch timeInVector {
| Time(r) => _timeToX(r, timeVector.zero, timeVector.unit)
| XValue(r) => r
}
}

504
packages/playground/src/distPlus/distribution/XYShape.re
View File

@ -1,504 +0,0 @@
open DistTypes;
let interpolate =
(xMin: float, xMax: float, yMin: float, yMax: float, xIntended: float)
: float => {
let minProportion = (xMax -. xIntended) /. (xMax -. xMin);
let maxProportion = (xIntended -. xMin) /. (xMax -. xMin);
yMin *. minProportion +. yMax *. maxProportion;
};
// TODO: Make sure that shapes cannot be empty.
let extImp = E.O.toExt("Tried to perform an operation on an empty XYShape.");
module T = {
type t = xyShape;
let toXyShape = (t: t): xyShape => t;
type ts = array(xyShape);
let xs = (t: t) => t.xs;
let ys = (t: t) => t.ys;
let length = (t: t) => E.A.length(t.xs);
let empty = {xs: [||], ys: [||]};
let isEmpty = (t: t) => length(t) == 0;
let minX = (t: t) => t |> xs |> E.A.Sorted.min |> extImp;
let maxX = (t: t) => t |> xs |> E.A.Sorted.max |> extImp;
let firstY = (t: t) => t |> ys |> E.A.first |> extImp;
let lastY = (t: t) => t |> ys |> E.A.last |> extImp;
let xTotalRange = (t: t) => maxX(t) -. minX(t);
let mapX = (fn, t: t): t => {xs: E.A.fmap(fn, t.xs), ys: t.ys};
let mapY = (fn, t: t): t => {xs: t.xs, ys: E.A.fmap(fn, t.ys)};
let zip = ({xs, ys}: t) => Belt.Array.zip(xs, ys);
let fromArray = ((xs, ys)): t => {xs, ys};
let fromArrays = (xs, ys): t => {xs, ys};
let accumulateYs = (fn, p: t) => {
fromArray((p.xs, E.A.accumulate(fn, p.ys)));
};
let concat = (t1: t, t2: t) => {
let cxs = Array.concat([t1.xs, t2.xs]);
let cys = Array.concat([t1.ys, t2.ys]);
{xs: cxs, ys: cys};
};
let fromZippedArray = (pairs: array((float, float))): t =>
pairs |> Belt.Array.unzip |> fromArray;
let equallyDividedXs = (t: t, newLength) => {
E.A.Floats.range(minX(t), maxX(t), newLength);
};
let toJs = (t: t) => {
{"xs": t.xs, "ys": t.ys};
};
};
module Ts = {
type t = T.ts;
let minX = (t: t) => t |> E.A.fmap(T.minX) |> E.A.min |> extImp;
let maxX = (t: t) => t |> E.A.fmap(T.maxX) |> E.A.max |> extImp;
let equallyDividedXs = (t: t, newLength) => {
E.A.Floats.range(minX(t), maxX(t), newLength);
};
let allXs = (t: t) => t |> E.A.fmap(T.xs) |> E.A.Sorted.concatMany;
};
module Pairs = {
let x = fst;
let y = snd;
let first = (t: T.t) => (T.minX(t), T.firstY(t));
let last = (t: T.t) => (T.maxX(t), T.lastY(t));
let getBy = (t: T.t, fn) => t |> T.zip |> E.A.getBy(_, fn);
let firstAtOrBeforeXValue = (xValue, t: T.t) => {
let zipped = T.zip(t);
let firstIndex =
zipped |> Belt.Array.getIndexBy(_, ((x, _)) => x > xValue);
let previousIndex =
switch (firstIndex) {
| None => Some(Array.length(zipped) - 1)
| Some(0) => None
| Some(n) => Some(n - 1)
};
previousIndex |> Belt.Option.flatMap(_, Belt.Array.get(zipped));
};
};
module YtoX = {
let linear = (y: float, t: T.t): float => {
let firstHigherIndex =
E.A.Sorted.binarySearchFirstElementGreaterIndex(T.ys(t), y);
let foundX =
switch (firstHigherIndex) {
| `overMax => T.maxX(t)
| `underMin => T.minX(t)
| `firstHigher(firstHigherIndex) =>
let lowerOrEqualIndex =
firstHigherIndex - 1 < 0 ? 0 : firstHigherIndex - 1;
let (_xs, _ys) = (T.xs(t), T.ys(t));
let needsInterpolation = _ys[lowerOrEqualIndex] != y;
if (needsInterpolation) {
interpolate(
_ys[lowerOrEqualIndex],
_ys[firstHigherIndex],
_xs[lowerOrEqualIndex],
_xs[firstHigherIndex],
y,
);
} else {
_xs[lowerOrEqualIndex];
};
};
foundX;
};
};
module XtoY = {
let stepwiseIncremental = (f, t: T.t) =>
Pairs.firstAtOrBeforeXValue(f, t) |> E.O.fmap(Pairs.y);
let stepwiseIfAtX = (f: float, t: T.t) => {
Pairs.getBy(t, ((x: float, _)) => {x == f}) |> E.O.fmap(Pairs.y);
};
let linear = (x: float, t: T.t): float => {
let firstHigherIndex =
E.A.Sorted.binarySearchFirstElementGreaterIndex(T.xs(t), x);
let n =
switch (firstHigherIndex) {
| `overMax => T.lastY(t)
| `underMin => T.firstY(t)
| `firstHigher(firstHigherIndex) =>
let lowerOrEqualIndex =
firstHigherIndex - 1 < 0 ? 0 : firstHigherIndex - 1;
let (_xs, _ys) = (T.xs(t), T.ys(t));
let needsInterpolation = _xs[lowerOrEqualIndex] != x;
if (needsInterpolation) {
interpolate(
_xs[lowerOrEqualIndex],
_xs[firstHigherIndex],
_ys[lowerOrEqualIndex],
_ys[firstHigherIndex],
x,
);
} else {
_ys[lowerOrEqualIndex];
};
};
n;
};
/* Returns a between-points-interpolating function that can be used with PointwiseCombination.combine.
Interpolation can either be stepwise (using the value on the left) or linear. Extrapolation can be `UseZero or `UseOutermostPoints. */
let continuousInterpolator = (interpolation: DistTypes.interpolationStrategy, extrapolation: DistTypes.extrapolationStrategy): interpolator => {
switch (interpolation, extrapolation) {
| (`Linear, `UseZero) => (t: T.t, leftIndex: int, x: float) => {
if (leftIndex < 0) {
0.0
} else if (leftIndex >= T.length(t) - 1) {
0.0
} else {
let x1 = t.xs[leftIndex];
let x2 = t.xs[leftIndex + 1];
let y1 = t.ys[leftIndex];
let y2 = t.ys[leftIndex + 1];
let fraction = (x -. x1) /. (x2 -. x1);
y1 *. (1. -. fraction) +. y2 *. fraction;
};
}
| (`Linear, `UseOutermostPoints) => (t: T.t, leftIndex: int, x: float) => {
if (leftIndex < 0) {
t.ys[0];
} else if (leftIndex >= T.length(t) - 1) {
t.ys[T.length(t) - 1]
} else {
let x1 = t.xs[leftIndex];
let x2 = t.xs[leftIndex + 1];
let y1 = t.ys[leftIndex];
let y2 = t.ys[leftIndex + 1];
let fraction = (x -. x1) /. (x2 -. x1);
y1 *. (1. -. fraction) +. y2 *. fraction;
};
}
| (`Stepwise, `UseZero) => (t: T.t, leftIndex: int, x: float) => {
if (leftIndex < 0) {
0.0
} else if (leftIndex >= T.length(t) - 1) {
0.0
} else {
t.ys[leftIndex];
}
}
| (`Stepwise, `UseOutermostPoints) => (t: T.t, leftIndex: int, x: float) => {
if (leftIndex < 0) {
t.ys[0];
} else if (leftIndex >= T.length(t) - 1) {
t.ys[T.length(t) - 1]
} else {
t.ys[leftIndex];
}
}
}
};
/* Returns a between-points-interpolating function that can be used with PointwiseCombination.combine.
For discrete distributions, the probability density between points is zero, so we just return zero here. */
let discreteInterpolator: interpolator = (t: T.t, leftIndex: int, x: float) => 0.0;
};
module XsConversion = {
let _replaceWithXs = (newXs: array(float), t: T.t): T.t => {
let newYs = Belt.Array.map(newXs, XtoY.linear(_, t));
{xs: newXs, ys: newYs};
};
let equallyDivideXByMass = (newLength: int, integral: T.t) =>
E.A.Floats.range(0.0, 1.0, newLength)
|> E.A.fmap(YtoX.linear(_, integral));
let proportionEquallyOverX = (newLength: int, t: T.t): T.t => {
T.equallyDividedXs(t, newLength) |> _replaceWithXs(_, t);
};
let proportionByProbabilityMass =
(newLength: int, integral: T.t, t: T.t): T.t => {
integral
|> equallyDivideXByMass(newLength) // creates a new set of xs at evenly spaced percentiles
|> _replaceWithXs(_, t); // linearly interpolates new ys for the new xs
};
};
module Zipped = {
type zipped = array((float, float));
let compareYs = ((_, y1), (_, y2)) => y1 > y2 ? 1 : 0;
let compareXs = ((x1, _), (x2, _)) => x1 > x2 ? 1 : 0;
let sortByY = (t: zipped) => t |> E.A.stableSortBy(_, compareYs);
let sortByX = (t: zipped) => t |> E.A.stableSortBy(_, compareXs);
let filterByX = (testFn: (float => bool), t: zipped) => t |> E.A.filter(((x, _)) => testFn(x));
};
module PointwiseCombination = {
// t1Interpolator and t2Interpolator are functions from XYShape.XtoY, e.g. linearBetweenPointsExtrapolateFlat.
let combine = [%raw {| // : (float => float => float, T.t, T.t, bool) => T.t
// This function combines two xyShapes by looping through both of them simultaneously.
// It always moves on to the next smallest x, whether that's in the first or second input's xs,
// and interpolates the value on the other side, thus accumulating xs and ys.
// This is written in raw JS because this can still be a bottleneck, and using refs for the i and j indices is quite painful.
function(fn, interpolator, t1, t2) {
let t1n = t1.xs.length;
let t2n = t2.xs.length;
let outX = [];
let outY = [];
let i = -1;
let j = -1;
while (i <= t1n - 1 && j <= t2n - 1) {
let x, ya, yb;
if (j == t2n - 1 && i < t1n - 1 ||
t1.xs[i+1] < t2.xs[j+1]) { // if a has to catch up to b, or if b is already done
i++;
x = t1.xs[i];
ya = t1.ys[i];
yb = interpolator(t2, j, x);
} else if (i == t1n - 1 && j < t2n - 1 ||
t1.xs[i+1] > t2.xs[j+1]) { // if b has to catch up to a, or if a is already done
j++;
x = t2.xs[j];
yb = t2.ys[j];
ya = interpolator(t1, i, x);
} else if (i < t1n - 1 && j < t2n && t1.xs[i+1] === t2.xs[j+1]) { // if they happen to be equal, move both ahead
i++;
j++;
x = t1.xs[i];
ya = t1.ys[i];
yb = t2.ys[j];
} else if (i === t1n - 1 && j === t2n - 1) {
// finished!
i = t1n;
j = t2n;
continue;
} else {
console.log("Error!", i, j);
}
outX.push(x);
outY.push(fn(ya, yb));
}
return {xs: outX, ys: outY};
}
|}];
let combineEvenXs =
(
~fn,
~xToYSelection,
sampleCount,
t1: T.t,
t2: T.t,
) => {
switch ((E.A.length(t1.xs), E.A.length(t2.xs))) {
| (0, 0) => T.empty
| (0, _) => t2
| (_, 0) => t1
| (_, _) => {
let allXs = Ts.equallyDividedXs([|t1, t2|], sampleCount);
let allYs = allXs |> E.A.fmap(x => fn(xToYSelection(x, t1), xToYSelection(x, t2)));
T.fromArrays(allXs, allYs);
}
}
};
// TODO: I'd bet this is pretty slow. Maybe it would be faster to intersperse Xs and Ys separately.
let intersperse = (t1: T.t, t2: T.t) => {
E.A.intersperse(T.zip(t1), T.zip(t2)) |> T.fromZippedArray;
};
};
// I'm really not sure this part is actually what we want at this point.
module Range = {
// ((lastX, lastY), (nextX, nextY))
type zippedRange = ((float, float), (float, float));
let toT = T.fromZippedArray;
let nextX = ((_, (nextX, _)): zippedRange) => nextX;
let rangePointAssumingSteps = (((_, lastY), (nextX, _)): zippedRange) => (
nextX,
lastY,
);
let rangeAreaAssumingTriangles =
(((lastX, lastY), (nextX, nextY)): zippedRange) =>
(nextX -. lastX) *. (lastY +. nextY) /. 2.;
//Todo: figure out how to without making new array.
let rangeAreaAssumingTrapezoids =
(((lastX, lastY), (nextX, nextY)): zippedRange) =>
(nextX -. lastX)
*. (Js.Math.min_float(lastY, nextY) +. (lastY +. nextY) /. 2.);
let delta_y_over_delta_x =
(((lastX, lastY), (nextX, nextY)): zippedRange) =>
(nextY -. lastY) /. (nextX -. lastX);
let mapYsBasedOnRanges = (fn, t) =>
Belt.Array.zip(t.xs, t.ys)
|> E.A.toRanges
|> E.R.toOption
|> E.O.fmap(r => r |> Belt.Array.map(_, r => (nextX(r), fn(r))));
// This code is messy, in part because I'm trying to make things easy on garbage collection here.
// It's using triangles instead of trapezoids right now.
let integrateWithTriangles = ({xs, ys}) => {
let length = E.A.length(xs);
let cumulativeY = Belt.Array.make(length, 0.0);
for (x in 0 to E.A.length(xs) - 2) {
let _ =
Belt.Array.set(
cumulativeY,
x + 1,
(xs[x + 1] -. xs[x]) // dx
*. ((ys[x] +. ys[x + 1]) /. 2.) // (1/2) * (avgY)
+. cumulativeY[x],
);
();
};
Some({xs, ys: cumulativeY});
};
let derivative = mapYsBasedOnRanges(delta_y_over_delta_x);
let stepwiseToLinear = ({xs, ys}: T.t): T.t => {
// adds points at the bottom of each step.
let length = E.A.length(xs);
let newXs: array(float) = Belt.Array.makeUninitializedUnsafe(2 * length);
let newYs: array(float) = Belt.Array.makeUninitializedUnsafe(2 * length);
Belt.Array.set(newXs, 0, xs[0] -. epsilon_float) |> ignore;
Belt.Array.set(newYs, 0, 0.) |> ignore;
Belt.Array.set(newXs, 1, xs[0]) |> ignore;
Belt.Array.set(newYs, 1, ys[0]) |> ignore;
for (i in 1 to E.A.length(xs) - 1) {
Belt.Array.set(newXs, i * 2, xs[i] -. epsilon_float) |> ignore;
Belt.Array.set(newYs, i * 2, ys[i-1]) |> ignore;
Belt.Array.set(newXs, i * 2 + 1, xs[i]) |> ignore;
Belt.Array.set(newYs, i * 2 + 1, ys[i]) |> ignore;
();
};
{xs: newXs, ys: newYs};
};
// TODO: I think this isn't needed by any functions anymore.
let stepsToContinuous = t => {
// TODO: It would be nicer if this the diff didn't change the first element, and also maybe if there were a more elegant way of doing this.
let diff = T.xTotalRange(t) |> (r => r *. 0.00001);
let items =
switch (E.A.toRanges(Belt.Array.zip(t.xs, t.ys))) {
| Ok(items) =>
Some(
items
|> Belt.Array.map(_, rangePointAssumingSteps)
|> T.fromZippedArray
|> PointwiseCombination.intersperse(t |> T.mapX(e => e +. diff)),
)
| _ => Some(t)
};
let first = items |> E.O.fmap(T.zip) |> E.O.bind(_, E.A.get(_, 0));
switch (items, first) {
| (Some(items), Some((0.0, _))) => Some(items)
| (Some(items), Some((firstX, _))) =>
let all = E.A.append([|(firstX, 0.0)|], items |> T.zip);
all |> T.fromZippedArray |> E.O.some;
| _ => None
};
};
};
let pointLogScore = (prediction, answer) =>
switch (answer) {
| 0. => 0.0
| answer => answer *. Js.Math.log2(Js.Math.abs_float(prediction /. answer))
};
let logScorePoint = (sampleCount, t1, t2) =>
PointwiseCombination.combineEvenXs(
~fn=pointLogScore,
~xToYSelection=XtoY.linear,
sampleCount,
t1,
t2,
)
|> Range.integrateWithTriangles
|> E.O.fmap(T.accumulateYs((+.)))
|> E.O.fmap(Pairs.last)
|> E.O.fmap(Pairs.y);
module Analysis = {
let integrateContinuousShape =
(
~indefiniteIntegralStepwise=(p, h1) => h1 *. p,
~indefiniteIntegralLinear=(p, a, b) => a *. p +. b *. p ** 2.0 /. 2.0,
t: DistTypes.continuousShape,
)
: float => {
let xs = t.xyShape.xs;
let ys = t.xyShape.ys;
E.A.reducei(
xs,
0.0,
(acc, _x, i) => {
let areaUnderIntegral =
// TODO Take this switch statement out of the loop body
switch (t.interpolation, i) {
| (_, 0) => 0.0
| (`Stepwise, _) =>
indefiniteIntegralStepwise(xs[i], ys[i - 1])
-. indefiniteIntegralStepwise(xs[i - 1], ys[i - 1])
| (`Linear, _) =>
let x1 = xs[i - 1];
let x2 = xs[i];
if (x1 == x2) {
0.0
} else {
let h1 = ys[i - 1];
let h2 = ys[i];
let b = (h1 -. h2) /. (x1 -. x2);
let a = h1 -. b *. x1;
indefiniteIntegralLinear(x2, a, b)
-. indefiniteIntegralLinear(x1, a, b);
};
};
acc +. areaUnderIntegral;
},
);
};
let getMeanOfSquaresContinuousShape = (t: DistTypes.continuousShape) => {
let indefiniteIntegralLinear = (p, a, b) =>
a *. p ** 3.0 /. 3.0 +. b *. p ** 4.0 /. 4.0;
let indefiniteIntegralStepwise = (p, h1) => h1 *. p ** 3.0 /. 3.0;
integrateContinuousShape(
~indefiniteIntegralStepwise,
~indefiniteIntegralLinear,
t,
);
};
let getVarianceDangerously =
(t: 't, mean: 't => float, getMeanOfSquares: 't => float): float => {
let meanSquared = mean(t) ** 2.0;
let meanOfSquares = getMeanOfSquares(t);
meanOfSquares -. meanSquared;
};
let squareXYShape = T.mapX(x => x ** 2.0)
};

21
packages/playground/src/distPlus/expressionTree/ExpressionTree.re
View File

@ -1,21 +0,0 @@
open ExpressionTypes.ExpressionTree;
let toString = ExpressionTreeBasic.toString;
let envs = (samplingInputs, environment) => {
{samplingInputs, environment, evaluateNode: ExpressionTreeEvaluator.toLeaf};
};
let toLeaf = (samplingInputs, environment, node: node) =>
ExpressionTreeEvaluator.toLeaf(envs(samplingInputs, environment), node);
let toShape = (samplingInputs, environment, node: node) => {
switch (toLeaf(samplingInputs, environment, node)) {
| Ok(`RenderedDist(shape)) => Ok(shape)
| Ok(_) => Error("Rendering failed.")
| Error(e) => Error(e)
};
};
let runFunction = (samplingInputs, environment, inputs, fn: PTypes.Function.t) => {
let params = envs(samplingInputs, environment);
PTypes.Function.run(params, inputs, fn);
};

35
packages/playground/src/distPlus/expressionTree/ExpressionTreeBasic.re
View File

@ -1,35 +0,0 @@
open ExpressionTypes.ExpressionTree;
let rec toString: node => string =
fun
| `SymbolicDist(d) => SymbolicDist.T.toString(d)
| `RenderedDist(_) => "[renderedShape]"
| `AlgebraicCombination(op, t1, t2) =>
Operation.Algebraic.format(op, toString(t1), toString(t2))
| `PointwiseCombination(op, t1, t2) =>
Operation.Pointwise.format(op, toString(t1), toString(t2))
| `Normalize(t) => "normalize(k" ++ toString(t) ++ ")"
| `Truncate(lc, rc, t) =>
Operation.T.truncateToString(lc, rc, toString(t))
| `Render(t) => toString(t)
| `Symbol(t) => "Symbol: " ++ t
| `FunctionCall(name, args) =>
"[Function call: ("
++ name
++ (args |> E.A.fmap(toString) |> Js.String.concatMany(_, ","))
++ ")]"
| `Function(args, internal) =>
"[Function: ("
++ (args |> Js.String.concatMany(_, ","))
++ toString(internal)
++ ")]"
| `Array(a) =>
"[" ++ (a |> E.A.fmap(toString) |> Js.String.concatMany(_, ",")) ++ "]"
| `Hash(h) =>
"{"
++ (
h
|> E.A.fmap(((name, value)) => name ++ ":" ++ toString(value))
|> Js.String.concatMany(_, ",")
)
++ "}";

332
packages/playground/src/distPlus/expressionTree/ExpressionTreeEvaluator.re
View File

@ -1,332 +0,0 @@
open ExpressionTypes;
open ExpressionTypes.ExpressionTree;
type t = node;
type tResult = node => result(node, string);
/* Given two random variables A and B, this returns the distribution
of a new variable that is the result of the operation on A and B.
For instance, normal(0, 1) + normal(1, 1) -> normal(1, 2).
In general, this is implemented via convolution. */
module AlgebraicCombination = {
let tryAnalyticalSimplification = (operation, t1: t, t2: t) =>
switch (operation, t1, t2) {
| (operation, `SymbolicDist(d1), `SymbolicDist(d2)) =>
switch (SymbolicDist.T.tryAnalyticalSimplification(d1, d2, operation)) {
| `AnalyticalSolution(symbolicDist) => Ok(`SymbolicDist(symbolicDist))
| `Error(er) => Error(er)
| `NoSolution => Ok(`AlgebraicCombination((operation, t1, t2)))
}
| _ => Ok(`AlgebraicCombination((operation, t1, t2)))
};
let combinationByRendering =
(evaluationParams, algebraicOp, t1: node, t2: node)
: result(node, string) => {
E.R.merge(
Render.ensureIsRenderedAndGetShape(evaluationParams, t1),
Render.ensureIsRenderedAndGetShape(evaluationParams, t2),
)
|> E.R.fmap(((a, b)) =>
`RenderedDist(Shape.combineAlgebraically(algebraicOp, a, b))
);
};
let nodeScore: node => int =
fun
| `SymbolicDist(`Float(_)) => 1
| `SymbolicDist(_) => 1000
| `RenderedDist(Discrete(m)) => m.xyShape |> XYShape.T.length
| `RenderedDist(Mixed(_)) => 1000
| `RenderedDist(Continuous(_)) => 1000
| _ => 1000;
let choose = (t1: node, t2: node) => {
nodeScore(t1) * nodeScore(t2) > 10000 ? `Sampling : `Analytical;
};
let combine =
(evaluationParams, algebraicOp, t1: node, t2: node)
: result(node, string) => {
E.R.merge(
PTypes.SamplingDistribution.renderIfIsNotSamplingDistribution(
evaluationParams,
t1,
),
PTypes.SamplingDistribution.renderIfIsNotSamplingDistribution(
evaluationParams,
t2,
),
)
|> E.R.bind(_, ((a, b)) =>
switch (choose(a, b)) {
| `Sampling =>
PTypes.SamplingDistribution.combineShapesUsingSampling(
evaluationParams,
algebraicOp,
a,
b,
)
| `Analytical =>
combinationByRendering(evaluationParams, algebraicOp, a, b)
}
);
};
let operationToLeaf =
(
evaluationParams: evaluationParams,
algebraicOp: ExpressionTypes.algebraicOperation,
t1: t,
t2: t,
)
: result(node, string) =>
algebraicOp
|> tryAnalyticalSimplification(_, t1, t2)
|> E.R.bind(
_,
fun
| `SymbolicDist(_) as t => Ok(t)
| _ => combine(evaluationParams, algebraicOp, t1, t2),
);
};
module PointwiseCombination = {
let pointwiseAdd = (evaluationParams: evaluationParams, t1: t, t2: t) => {
switch (
Render.render(evaluationParams, t1),
Render.render(evaluationParams, t2),
) {
| (Ok(`RenderedDist(rs1)), Ok(`RenderedDist(rs2))) =>
Ok(
`RenderedDist(
Shape.combinePointwise(
~integralSumCachesFn=(a, b) => Some(a +. b),
~integralCachesFn=
(a, b) =>
Some(
Continuous.combinePointwise(
~distributionType=`CDF,
(+.),
a,
b,
),
),
(+.),
rs1,
rs2,
),
),
)
| (Error(e1), _) => Error(e1)
| (_, Error(e2)) => Error(e2)
| _ => Error("Pointwise combination: rendering failed.")
};
};
let pointwiseCombine =
(fn, evaluationParams: evaluationParams, t1: t, t2: t) => {
// TODO: construct a function that we can easily sample from, to construct
// a RenderedDist. Use the xMin and xMax of the rendered shapes to tell the sampling function where to look.
// TODO: This should work for symbolic distributions too!
switch (
Render.render(evaluationParams, t1),
Render.render(evaluationParams, t2),
) {
| (Ok(`RenderedDist(rs1)), Ok(`RenderedDist(rs2))) =>
Ok(`RenderedDist(Shape.combinePointwise(fn, rs1, rs2)))
| (Error(e1), _) => Error(e1)
| (_, Error(e2)) => Error(e2)
| _ => Error("Pointwise combination: rendering failed.")
};
};
let operationToLeaf =
(
evaluationParams: evaluationParams,
pointwiseOp: pointwiseOperation,
t1: t,
t2: t,
) => {
switch (pointwiseOp) {
| `Add => pointwiseAdd(evaluationParams, t1, t2)
| `Multiply => pointwiseCombine(( *. ), evaluationParams, t1, t2)
| `Exponentiate => pointwiseCombine(( ** ), evaluationParams, t1, t2)
};
};
};
module Truncate = {
let trySimplification = (leftCutoff, rightCutoff, t): simplificationResult => {
switch (leftCutoff, rightCutoff, t) {
| (None, None, t) => `Solution(t)
| (Some(lc), Some(rc), _) when lc > rc =>
`Error(
"Left truncation bound must be smaller than right truncation bound.",
)
| (lc, rc, `SymbolicDist(`Uniform(u))) =>
`Solution(
`SymbolicDist(`Uniform(SymbolicDist.Uniform.truncate(lc, rc, u))),
)
| _ => `NoSolution
};
};
let truncateAsShape =
(evaluationParams: evaluationParams, leftCutoff, rightCutoff, t) => {
// TODO: use named args for xMin/xMax in renderToShape; if we're lucky we can at least get the tail
// of a distribution we otherwise wouldn't get at all
switch (Render.ensureIsRendered(evaluationParams, t)) {
| Ok(`RenderedDist(rs)) =>
Ok(`RenderedDist(Shape.T.truncate(leftCutoff, rightCutoff, rs)))
| Error(e) => Error(e)
| _ => Error("Could not truncate distribution.")
};
};
let operationToLeaf =
(
evaluationParams,
leftCutoff: option(float),
rightCutoff: option(float),
t: node,
)
: result(node, string) => {
t
|> trySimplification(leftCutoff, rightCutoff)
|> (
fun
| `Solution(t) => Ok(t)
| `Error(e) => Error(e)
| `NoSolution =>
truncateAsShape(evaluationParams, leftCutoff, rightCutoff, t)
);
};
};
module Normalize = {
let rec operationToLeaf = (evaluationParams, t: node): result(node, string) => {
Js.log2("normalize", t);
switch (t) {
| `RenderedDist(s) => Ok(`RenderedDist(Shape.T.normalize(s)))
| `SymbolicDist(_) => Ok(t)
| _ => evaluateAndRetry(evaluationParams, operationToLeaf, t)
};
};
};
module FunctionCall = {
let _runHardcodedFunction = (name, evaluationParams, args) =>
TypeSystem.Function.Ts.findByNameAndRun(
HardcodedFunctions.all,
name,
evaluationParams,
args,
);
let _runLocalFunction = (name, evaluationParams: evaluationParams, args) => {
Environment.getFunction(evaluationParams.environment, name)
|> E.R.bind(_, ((argNames, fn)) =>
PTypes.Function.run(evaluationParams, args, (argNames, fn))
);
};
let _runWithEvaluatedInputs =
(
evaluationParams: ExpressionTypes.ExpressionTree.evaluationParams,
name,
args: array(ExpressionTypes.ExpressionTree.node),
) => {
_runHardcodedFunction(name, evaluationParams, args)
|> E.O.default(_runLocalFunction(name, evaluationParams, args));
};
// TODO: This forces things to be floats
let run = (evaluationParams, name, args) => {
args
|> E.A.fmap(a => evaluationParams.evaluateNode(evaluationParams, a))
|> E.A.R.firstErrorOrOpen
|> E.R.bind(_, _runWithEvaluatedInputs(evaluationParams, name));
};
};
module Render = {
let rec operationToLeaf =
(evaluationParams: evaluationParams, t: node): result(t, string) => {
switch (t) {
| `Function(_) => Error("Cannot render a function")
| `SymbolicDist(d) =>
Ok(
`RenderedDist(
SymbolicDist.T.toShape(
evaluationParams.samplingInputs.shapeLength,
d,
),
),
)
| `RenderedDist(_) as t => Ok(t) // already a rendered shape, we're done here
| _ => evaluateAndRetry(evaluationParams, operationToLeaf, t)
};
};
};
/* This function recursively goes through the nodes of the parse tree,
replacing each Operation node and its subtree with a Data node.
Whenever possible, the replacement produces a new Symbolic Data node,
but most often it will produce a RenderedDist.
This function is used mainly to turn a parse tree into a single RenderedDist
that can then be displayed to the user. */
let rec toLeaf =
(
evaluationParams: ExpressionTypes.ExpressionTree.evaluationParams,
node: t,
)
: result(t, string) => {
switch (node) {
// Leaf nodes just stay leaf nodes
| `SymbolicDist(_)
| `Function(_)
| `RenderedDist(_) => Ok(node)
| `Array(args) =>
args
|> E.A.fmap(toLeaf(evaluationParams))
|> E.A.R.firstErrorOrOpen
|> E.R.fmap(r => `Array(r))
// Operations nevaluationParamsd to be turned into leaves
| `AlgebraicCombination(algebraicOp, t1, t2) =>
AlgebraicCombination.operationToLeaf(
evaluationParams,
algebraicOp,
t1,
t2,
)
| `PointwiseCombination(pointwiseOp, t1, t2) =>
PointwiseCombination.operationToLeaf(
evaluationParams,
pointwiseOp,
t1,
t2,
)
| `Truncate(leftCutoff, rightCutoff, t) =>
Truncate.operationToLeaf(evaluationParams, leftCutoff, rightCutoff, t)
| `Normalize(t) => Normalize.operationToLeaf(evaluationParams, t)
| `Render(t) => Render.operationToLeaf(evaluationParams, t)
| `Hash(t) =>
t
|> E.A.fmap(((name: string, node: node)) =>
toLeaf(evaluationParams, node) |> E.R.fmap(r => (name, r))
)
|> E.A.R.firstErrorOrOpen
|> E.R.fmap(r => `Hash(r))
| `Symbol(r) =>
ExpressionTypes.ExpressionTree.Environment.get(
evaluationParams.environment,
r,
)
|> E.O.toResult("Undeclared variable " ++ r)
|> E.R.bind(_, toLeaf(evaluationParams))
| `FunctionCall(name, args) =>
FunctionCall.run(evaluationParams, name, args)
|> E.R.bind(_, toLeaf(evaluationParams))
}
};

180
packages/playground/src/distPlus/expressionTree/ExpressionTypes.re
View File

@ -1,180 +0,0 @@
type algebraicOperation = [
| `Add
| `Multiply
| `Subtract
| `Divide
| `Exponentiate
];
type pointwiseOperation = [ | `Add | `Multiply | `Exponentiate];
type scaleOperation = [ | `Multiply | `Exponentiate | `Log];
type distToFloatOperation = [
| `Pdf(float)
| `Cdf(float)
| `Inv(float)
| `Mean
| `Sample
];
module ExpressionTree = {
type hash = array((string, node))
and node = [
| `SymbolicDist(SymbolicTypes.symbolicDist)
| `RenderedDist(DistTypes.shape)
| `Symbol(string)
| `Hash(hash)
| `Array(array(node))
| `Function(array(string), node)
| `AlgebraicCombination(algebraicOperation, node, node)
| `PointwiseCombination(pointwiseOperation, node, node)
| `Normalize(node)
| `Render(node)
| `Truncate(option(float), option(float), node)
| `FunctionCall(string, array(node))
];
module Hash = {
type t('a) = array((string, 'a));
let getByName = (t: t('a), name) =>
E.A.getBy(t, ((n, _)) => n == name) |> E.O.fmap(((_, r)) => r);
let getByNameResult = (t: t('a), name) =>
getByName(t, name) |> E.O.toResult(name ++ " expected and not found");
let getByNames = (hash: t('a), names: array(string)) =>
names |> E.A.fmap(name => (name, getByName(hash, name)));
};
// Have nil as option
let getFloat = (node: node) =>
node
|> (
fun
| `RenderedDist(Discrete({xyShape: {xs: [|x|], ys: [|1.0|]}})) =>
Some(x)
| `SymbolicDist(`Float(x)) => Some(x)
| _ => None
);
let toFloatIfNeeded = (node: node) =>
switch (node |> getFloat) {
| Some(float) => `SymbolicDist(`Float(float))
| None => node
};
type samplingInputs = {
sampleCount: int,
outputXYPoints: int,
kernelWidth: option(float),
shapeLength: int,
};
module SamplingInputs = {
type t = {
sampleCount: option(int),
outputXYPoints: option(int),
kernelWidth: option(float),
shapeLength: option(int),
};
let withDefaults = (t: t): samplingInputs => {
sampleCount: t.sampleCount |> E.O.default(10000),
outputXYPoints: t.outputXYPoints |> E.O.default(10000),
kernelWidth: t.kernelWidth,
shapeLength: t.shapeLength |> E.O.default(10000),
};
};
type environment = Belt.Map.String.t(node);
module Environment = {
type t = environment;
module MS = Belt.Map.String;
let fromArray = MS.fromArray;
let empty: t = [||]->fromArray;
let mergeKeepSecond = (a: t, b: t) =>
MS.merge(a, b, (_, a, b) =>
switch (a, b) {
| (_, Some(b)) => Some(b)
| (Some(a), _) => Some(a)
| _ => None
}
);
let update = (t, str, fn) => MS.update(t, str, fn);
let get = (t: t, str) => MS.get(t, str);
let getFunction = (t: t, str) =>
switch (get(t, str)) {
| Some(`Function(argNames, fn)) => Ok((argNames, fn))
| _ => Error("Function " ++ str ++ " not found")
};
};
type evaluationParams = {
samplingInputs,
environment,
evaluateNode: (evaluationParams, node) => Belt.Result.t(node, string),
};
let evaluateNode = (evaluationParams: evaluationParams) =>
evaluationParams.evaluateNode(evaluationParams);
let evaluateAndRetry = (evaluationParams, fn, node) =>
node
|> evaluationParams.evaluateNode(evaluationParams)
|> E.R.bind(_, fn(evaluationParams));
module Render = {
type t = node;
let render = (evaluationParams: evaluationParams, r) =>
`Render(r) |> evaluateNode(evaluationParams);
let ensureIsRendered = (params, t) =>
switch (t) {
| `RenderedDist(_) => Ok(t)
| _ =>
switch (render(params, t)) {
| Ok(`RenderedDist(r)) => Ok(`RenderedDist(r))
| Ok(_) => Error("Did not render as requested")
| Error(e) => Error(e)
}
};
let ensureIsRenderedAndGetShape = (params, t) =>
switch (ensureIsRendered(params, t)) {
| Ok(`RenderedDist(r)) => Ok(r)
| Ok(_) => Error("Did not render as requested")
| Error(e) => Error(e)
};
let getShape = (item: node) =>
switch (item) {
| `RenderedDist(r) => Some(r)
| _ => None
};
let _toFloat = (t: DistTypes.shape) =>
switch (t) {
| Discrete({xyShape: {xs: [|x|], ys: [|1.0|]}}) =>
Some(`SymbolicDist(`Float(x)))
| _ => None
};
let toFloat = (item: node): result(node, string) =>
item
|> getShape
|> E.O.bind(_, _toFloat)
|> E.O.toResult("Not valid shape");
};
};
type simplificationResult = [
| `Solution(ExpressionTree.node)
| `Error(string)
| `NoSolution
];
module Program = {
type statement = [
| `Assignment(string, ExpressionTree.node)
| `Expression(ExpressionTree.node)
];
type program = array(statement);
};

353
packages/playground/src/distPlus/expressionTree/MathJsParser.re
View File

@ -1,353 +0,0 @@
module MathJsonToMathJsAdt = {
type arg =
| Symbol(string)
| Value(float)
| Fn(fn)
| Array(array(arg))
| Blocks(array(arg))
| Object(Js.Dict.t(arg))
| Assignment(arg, arg)
| FunctionAssignment(fnAssignment)
and fn = {
name: string,
args: array(arg),
}
and fnAssignment = {
name: string,
args: array(string),
expression: arg,
};
let rec run = (j: Js.Json.t) =>
Json.Decode.(
switch (field("mathjs", string, j)) {
| "FunctionNode" =>
let args = j |> field("args", array(run));
let name = j |> optional(field("fn", field("name", string)));
name |> E.O.fmap(name => Fn({name, args: args |> E.A.O.concatSomes}));
| "OperatorNode" =>
let args = j |> field("args", array(run));
Some(
Fn({
name: j |> field("fn", string),
args: args |> E.A.O.concatSomes,
}),
);
| "ConstantNode" =>
optional(field("value", Json.Decode.float), j)
|> E.O.fmap(r => Value(r))
| "ParenthesisNode" => j |> field("content", run)
| "ObjectNode" =>
let properties = j |> field("properties", dict(run));
Js.Dict.entries(properties)
|> E.A.fmap(((key, value)) => value |> E.O.fmap(v => (key, v)))
|> E.A.O.concatSomes
|> Js.Dict.fromArray
|> (r => Some(Object(r)));
| "ArrayNode" =>
let items = field("items", array(run), j);
Some(Array(items |> E.A.O.concatSomes));
| "SymbolNode" => Some(Symbol(field("name", string, j)))
| "AssignmentNode" =>
let object_ = j |> field("object", run);
let value_ = j |> field("value", run);
switch (object_, value_) {
| (Some(o), Some(v)) => Some(Assignment(o, v))
| _ => None
};
| "BlockNode" =>
let block = r => r |> field("node", run);
let args = j |> field("blocks", array(block)) |> E.A.O.concatSomes;
Some(Blocks(args));
| "FunctionAssignmentNode" =>
let name = j |> field("name", string);
let args = j |> field("params", array(field("name", string)));
let expression = j |> field("expr", run);
expression
|> E.O.fmap(expression =>
FunctionAssignment({name, args, expression})
);
| n =>
Js.log3("Couldn't parse mathjs node", j, n);
None;
}
);
};
module MathAdtToDistDst = {
open MathJsonToMathJsAdt;
let handleSymbol = sym => {
Ok(`Symbol(sym));
};
// TODO: This only works on the top level, which needs to be refactored. Also, I think functions don't need to be done like this anymore.
module MathAdtCleaner = {
let transformWithSymbol = (f: float, s: string) =>
switch (s) {
| "K" => Some(f *. 1000.)
| "M" => Some(f *. 1000000.)
| "B" => Some(f *. 1000000000.)
| "T" => Some(f *. 1000000000000.)
| _ => None
};
let rec run =
fun
| Fn({name: "multiply", args: [|Value(f), Symbol(s)|]}) as doNothing =>
transformWithSymbol(f, s)
|> E.O.fmap(r => Value(r))
|> E.O.default(doNothing)
| Fn({name: "unaryMinus", args: [|Value(f)|]}) => Value((-1.0) *. f)
| Fn({name, args}) => Fn({name, args: args |> E.A.fmap(run)})
| Array(args) => Array(args |> E.A.fmap(run))
| Symbol(s) => Symbol(s)
| Value(v) => Value(v)
| Blocks(args) => Blocks(args |> E.A.fmap(run))
| Assignment(a, b) => Assignment(a, run(b))
| FunctionAssignment(a) => FunctionAssignment(a)
| Object(v) =>
Object(
v
|> Js.Dict.entries
|> E.A.fmap(((key, value)) => (key, run(value)))
|> Js.Dict.fromArray,
);
};
let lognormal = (args, parseArgs, nodeParser) =>
switch (args) {
| [|Object(o)|] =>
let g = s =>
Js.Dict.get(o, s)
|> E.O.toResult("Variable was empty")
|> E.R.bind(_, nodeParser);
switch (g("mean"), g("stdev"), g("mu"), g("sigma")) {
| (Ok(mean), Ok(stdev), _, _) =>
Ok(`FunctionCall(("lognormalFromMeanAndStdDev", [|mean, stdev|])))
| (_, _, Ok(mu), Ok(sigma)) =>
Ok(`FunctionCall(("lognormal", [|mu, sigma|])))
| _ =>
Error(
"Lognormal distribution needs either mean and stdev or mu and sigma",
)
};
| _ =>
parseArgs()
|> E.R.fmap((args: array(ExpressionTypes.ExpressionTree.node)) =>
`FunctionCall(("lognormal", args))
)
};
// Error("Dotwise exponentiation needs two operands")
let operationParser =
(
name: string,
args: result(array(ExpressionTypes.ExpressionTree.node), string),
)
: result(ExpressionTypes.ExpressionTree.node, string) => {
let toOkAlgebraic = r => Ok(`AlgebraicCombination(r));
let toOkPointwise = r => Ok(`PointwiseCombination(r));
let toOkTruncate = r => Ok(`Truncate(r));
args
|> E.R.bind(_, args => {
switch (name, args) {
| ("add", [|l, r|]) => toOkAlgebraic((`Add, l, r))
| ("add", _) => Error("Addition needs two operands")
| ("unaryMinus", [|l|]) =>
toOkAlgebraic((`Multiply, `SymbolicDist(`Float(-1.0)), l))
| ("subtract", [|l, r|]) => toOkAlgebraic((`Subtract, l, r))
| ("subtract", _) => Error("Subtraction needs two operands")
| ("multiply", [|l, r|]) => toOkAlgebraic((`Multiply, l, r))
| ("multiply", _) => Error("Multiplication needs two operands")
| ("pow", [|l, r|]) => toOkAlgebraic((`Exponentiate, l, r))
| ("pow", _) => Error("Exponentiation needs two operands")
| ("dotMultiply", [|l, r|]) => toOkPointwise((`Multiply, l, r))
| ("dotMultiply", _) =>
Error("Dotwise multiplication needs two operands")
| ("dotPow", [|l, r|]) => toOkPointwise((`Exponentiate, l, r))
| ("dotPow", _) =>
Error("Dotwise exponentiation needs two operands")
| ("rightLogShift", [|l, r|]) => toOkPointwise((`Add, l, r))
| ("rightLogShift", _) =>
Error("Dotwise addition needs two operands")
| ("divide", [|l, r|]) => toOkAlgebraic((`Divide, l, r))
| ("divide", _) => Error("Division needs two operands")
| ("leftTruncate", [|d, `SymbolicDist(`Float(lc))|]) =>
toOkTruncate((Some(lc), None, d))
| ("leftTruncate", _) =>
Error(
"leftTruncate needs two arguments: the expression and the cutoff",
)
| ("rightTruncate", [|d, `SymbolicDist(`Float(rc))|]) =>
toOkTruncate((None, Some(rc), d))
| ("rightTruncate", _) =>
Error(
"rightTruncate needs two arguments: the expression and the cutoff",
)
| (
"truncate",
[|d, `SymbolicDist(`Float(lc)), `SymbolicDist(`Float(rc))|],
) =>
toOkTruncate((Some(lc), Some(rc), d))
| ("truncate", _) =>
Error(
"truncate needs three arguments: the expression and both cutoffs",
)
| _ => Error("This type not currently supported")
}
});
};
let functionParser =
(
nodeParser:
MathJsonToMathJsAdt.arg =>
Belt.Result.t(
ProbExample.ExpressionTypes.ExpressionTree.node,
string,
),
name: string,
args: array(MathJsonToMathJsAdt.arg),
)
: result(ExpressionTypes.ExpressionTree.node, string) => {
let parseArray = ags =>
ags |> E.A.fmap(nodeParser) |> E.A.R.firstErrorOrOpen;
let parseArgs = () => parseArray(args);
switch (name) {
| "lognormal" => lognormal(args, parseArgs, nodeParser)
| "multimodal"
| "add"
| "subtract"
| "multiply"
| "unaryMinus"
| "dotMultiply"
| "dotPow"
| "rightLogShift"
| "divide"
| "pow"
| "leftTruncate"
| "rightTruncate"
| "truncate" => operationParser(name, parseArgs())
| "mm" =>
let weights =
args
|> E.A.last
|> E.O.bind(
_,
fun
| Array(values) => Some(parseArray(values))
| _ => None,
);
let possibleDists =
E.O.isSome(weights)
? Belt.Array.slice(args, ~offset=0, ~len=E.A.length(args) - 1)
: args;
let dists = parseArray(possibleDists);
switch (weights, dists) {
| (Some(Error(r)), _) => Error(r)
| (_, Error(r)) => Error(r)
| (None, Ok(dists)) =>
let hash: ExpressionTypes.ExpressionTree.node =
`FunctionCall(("multimodal", [|`Hash(
[|
("dists", `Array(dists)),
("weights", `Array([||]))
|]
)|]));
Ok(hash);
| (Some(Ok(weights)), Ok(dists)) =>
let hash: ExpressionTypes.ExpressionTree.node =
`FunctionCall(("multimodal", [|`Hash(
[|
("dists", `Array(dists)),
("weights", `Array(weights))
|]
)|]));
Ok(hash);
};
| name =>
parseArgs()
|> E.R.fmap((args: array(ExpressionTypes.ExpressionTree.node)) =>
`FunctionCall((name, args))
)
};
};
let rec nodeParser:
MathJsonToMathJsAdt.arg =>
result(ExpressionTypes.ExpressionTree.node, string) =
fun
| Value(f) => Ok(`SymbolicDist(`Float(f)))
| Symbol(sym) => Ok(`Symbol(sym))
| Fn({name, args}) => functionParser(nodeParser, name, args)
| _ => {
Error("This type not currently supported");
};
// | FunctionAssignment({name, args, expression}) => {
// let evaluatedExpression = run(expression);
// `Function(_ => Ok(evaluatedExpression));
// }
let rec topLevel = (r): result(ExpressionTypes.Program.program, string) =>
switch (r) {
| FunctionAssignment({name, args, expression}) =>
switch (nodeParser(expression)) {
| Ok(r) => Ok([|`Assignment((name, `Function((args, r))))|])
| Error(r) => Error(r)
}
| Value(_) as r => nodeParser(r) |> E.R.fmap(r => [|`Expression(r)|])
| Fn(_) as r => nodeParser(r) |> E.R.fmap(r => [|`Expression(r)|])
| Array(_) => Error("Array not valid as top level")
| Symbol(s) => handleSymbol(s) |> E.R.fmap(r => [|`Expression(r)|])
| Object(_) => Error("Object not valid as top level")
| Assignment(name, value) =>
switch (name) {
| Symbol(symbol) =>
nodeParser(value) |> E.R.fmap(r => [|`Assignment((symbol, r))|])
| _ => Error("Symbol not a string")
}
| Blocks(blocks) =>
blocks
|> E.A.fmap(b => topLevel(b))
|> E.A.R.firstErrorOrOpen
|> E.R.fmap(E.A.concatMany)
};
let run = (r): result(ExpressionTypes.Program.program, string) =>
r |> MathAdtCleaner.run |> topLevel;
};
/* The MathJs parser doesn't support '.+' syntax, but we want it because it
would make sense with '.*'. Our workaround is to change this to >>>, which is
logShift in mathJS. We don't expect to use logShift anytime soon, so this tradeoff
seems fine.
*/
let pointwiseToRightLogShift = Js.String.replaceByRe([%re "/\.\+/g"], ">>>");
let fromString2 = str => {
/* We feed the user-typed string into Mathjs.parseMath,
which returns a JSON with (hopefully) a single-element array.
This array element is the top-level node of a nested-object tree
representing the functions/arguments/values/etc. in the string.
The function MathJsonToMathJsAdt then recursively unpacks this JSON into a typed data structure we can use.
Inside of this function, MathAdtToDistDst is called whenever a distribution function is encountered.
*/
let mathJsToJson = str |> pointwiseToRightLogShift |> Mathjs.parseMath;
let mathJsParse =
E.R.bind(mathJsToJson, r => {
switch (MathJsonToMathJsAdt.run(r)) {
| Some(r) => Ok(r)
| None => Error("MathJsParse Error")
}
});
let value = E.R.bind(mathJsParse, MathAdtToDistDst.run);
Js.log2(mathJsParse, value);
value;
};
let fromString = str => {
fromString2(str);
};

10
packages/playground/src/distPlus/expressionTree/Mathjs.re
View File

@ -1,10 +0,0 @@
[@bs.module "./MathjsWrapper.js"]
external parseMathExt: string => Js.Json.t = "parseMath";
let parseMath = (str: string): result(Js.Json.t, string) =>
switch (parseMathExt(str)) {
| exception (Js.Exn.Error(err)) =>
Error(Js.Exn.message(err) |> E.O.default("MathJS Parse Error"))
| exception _ => Error("MathJS Parse Error")
| j => Ok(j)
};

9
packages/playground/src/distPlus/expressionTree/MathjsWrapper.js
View File

@ -1,9 +0,0 @@
const math = require("mathjs");
function parseMath(f) {
return JSON.parse(JSON.stringify(math.parse(f)))
};
module.exports = {
parseMath,
};

105
packages/playground/src/distPlus/expressionTree/Operation.re
View File

@ -1,105 +0,0 @@
open ExpressionTypes;
module Algebraic = {
type t = algebraicOperation;
let toFn: (t, float, float) => float =
fun
| `Add => (+.)
| `Subtract => (-.)
| `Multiply => ( *. )
| `Exponentiate => ( ** )
| `Divide => (/.);
let applyFn = (t, f1, f2) => {
switch (t, f1, f2) {
| (`Divide, _, 0.) => Error("Cannot divide $v1 by zero.")
| _ => Ok(toFn(t, f1, f2))
};
};
let toString =
fun
| `Add => "+"
| `Subtract => "-"
| `Multiply => "*"
| `Exponentiate => ( "**" )
| `Divide => "/";
let format = (a, b, c) => b ++ " " ++ toString(a) ++ " " ++ c;
};
module Pointwise = {
type t = pointwiseOperation;
let toString =
fun
| `Add => "+"
| `Exponentiate => "^"
| `Multiply => "*";
let format = (a, b, c) => b ++ " " ++ toString(a) ++ " " ++ c;
};
module DistToFloat = {
type t = distToFloatOperation;
let format = (operation, value) =>
switch (operation) {
| `Cdf(f) => {j|cdf(x=$f,$value)|j}
| `Pdf(f) => {j|pdf(x=$f,$value)|j}
| `Inv(f) => {j|inv(x=$f,$value)|j}
| `Sample => "sample($value)"
| `Mean => "mean($value)"
};
};
// Note that different logarithms don't really do anything.
module Scale = {
type t = scaleOperation;
let toFn =
fun
| `Multiply => ( *. )
| `Exponentiate => ( ** )
| `Log => ((a, b) => log(a) /. log(b));
let format = (operation: t, value, scaleBy) =>
switch (operation) {
| `Multiply => {j|verticalMultiply($value, $scaleBy) |j}
| `Exponentiate => {j|verticalExponentiate($value, $scaleBy) |j}
| `Log => {j|verticalLog($value, $scaleBy) |j}
};
let toIntegralSumCacheFn =
fun
| `Multiply => ((a, b) => Some(a *. b))
| `Exponentiate => ((_, _) => None)
| `Log => ((_, _) => None);
let toIntegralCacheFn =
fun
| `Multiply => ((a, b) => None) // TODO: this could probably just be multiplied out (using Continuous.scaleBy)
| `Exponentiate => ((_, _) => None)
| `Log => ((_, _) => None);
};
module T = {
let truncateToString =
(left: option(float), right: option(float), nodeToString) => {
let left = left |> E.O.dimap(Js.Float.toString, () => "-inf");
let right = right |> E.O.dimap(Js.Float.toString, () => "inf");
{j|truncate($nodeToString, $left, $right)|j};
};
let toString = nodeToString =>
fun
| `AlgebraicCombination(op, t1, t2) =>
Algebraic.format(op, nodeToString(t1), nodeToString(t2))
| `PointwiseCombination(op, t1, t2) =>
Pointwise.format(op, nodeToString(t1), nodeToString(t2))
| `VerticalScaling(scaleOp, t, scaleBy) =>
Scale.format(scaleOp, nodeToString(t), nodeToString(scaleBy))
| `Normalize(t) => "normalize(k" ++ nodeToString(t) ++ ")"
| `FloatFromDist(floatFromDistOp, t) =>
DistToFloat.format(floatFromDistOp, nodeToString(t))
| `Truncate(lc, rc, t) => truncateToString(lc, rc, nodeToString(t))
| `Render(t) => nodeToString(t)
| _ => ""; // SymbolicDist and RenderedDist are handled in ExpressionTree.toString.
};

143
packages/playground/src/distPlus/expressionTree/PTypes.re
View File

@ -1,143 +0,0 @@
open ExpressionTypes.ExpressionTree;
module Function = {
type t = (array(string), node);
let fromNode: node => option(t) =
node =>
switch (node) {
| `Function(r) => Some(r)
| _ => None
};
let argumentNames = ((a, _): t) => a;
let internals = ((_, b): t) => b;
let run =
(
evaluationParams: ExpressionTypes.ExpressionTree.evaluationParams,
args: array(node),
t: t,
) =>
if (E.A.length(args) == E.A.length(argumentNames(t))) {
let newEnvironment =
Belt.Array.zip(argumentNames(t), args)
|> ExpressionTypes.ExpressionTree.Environment.fromArray;
let newEvaluationParams: ExpressionTypes.ExpressionTree.evaluationParams = {
samplingInputs: evaluationParams.samplingInputs,
environment:
ExpressionTypes.ExpressionTree.Environment.mergeKeepSecond(
evaluationParams.environment,
newEnvironment,
),
evaluateNode: evaluationParams.evaluateNode,
};
evaluationParams.evaluateNode(newEvaluationParams, internals(t));
} else {
Error("Wrong number of variables");
};
};
module Primative = {
type t = [
| `SymbolicDist(SymbolicTypes.symbolicDist)
| `RenderedDist(DistTypes.shape)
| `Function(array(string), node)
];
let isPrimative: node => bool =
fun
| `SymbolicDist(_)
| `RenderedDist(_)
| `Function(_) => true
| _ => false;
let fromNode: node => option(t) =
fun
| `SymbolicDist(_) as n
| `RenderedDist(_) as n
| `Function(_) as n => Some(n)
| _ => None;
};
module SamplingDistribution = {
type t = [
| `SymbolicDist(SymbolicTypes.symbolicDist)
| `RenderedDist(DistTypes.shape)
];
let isSamplingDistribution: node => bool =
fun
| `SymbolicDist(_) => true
| `RenderedDist(_) => true
| _ => false;
let fromNode: node => result(t, string) =
fun
| `SymbolicDist(n) => Ok(`SymbolicDist(n))
| `RenderedDist(n) => Ok(`RenderedDist(n))
| _ => Error("Not valid type");
let renderIfIsNotSamplingDistribution = (params, t): result(node, string) =>
!isSamplingDistribution(t)
? switch (Render.render(params, t)) {
| Ok(r) => Ok(r)
| Error(e) => Error(e)
}
: Ok(t);
let map = (~renderedDistFn, ~symbolicDistFn, node: node) =>
node
|> (
fun
| `RenderedDist(r) => Some(renderedDistFn(r))
| `SymbolicDist(s) => Some(symbolicDistFn(s))
| _ => None
);
let sampleN = n =>
map(
~renderedDistFn=Shape.sampleNRendered(n),
~symbolicDistFn=SymbolicDist.T.sampleN(n),
);
let getCombinationSamples = (n, algebraicOp, t1: node, t2: node) => {
switch (sampleN(n, t1), sampleN(n, t2)) {
| (Some(a), Some(b)) =>
Some(
Belt.Array.zip(a, b)
|> E.A.fmap(((a, b)) => Operation.Algebraic.toFn(algebraicOp, a, b)),
)
| _ => None
};
};
let combineShapesUsingSampling =
(evaluationParams: evaluationParams, algebraicOp, t1: node, t2: node) => {
let i1 = renderIfIsNotSamplingDistribution(evaluationParams, t1);
let i2 = renderIfIsNotSamplingDistribution(evaluationParams, t2);
E.R.merge(i1, i2)
|> E.R.bind(
_,
((a, b)) => {
let samples =
getCombinationSamples(
evaluationParams.samplingInputs.sampleCount,
algebraicOp,
a,
b,
);
// todo: This bottom part should probably be somewhere else.
let shape =
samples
|> E.O.fmap(
SamplesToShape.fromSamples(
~samplingInputs=evaluationParams.samplingInputs,
),
)
|> E.O.bind(_, r => r.shape)
|> E.O.toResult("No response");
shape |> E.R.fmap(r => `Normalize(`RenderedDist(r)));
},
);
};
};

5
packages/playground/src/distPlus/expressionTree/Program.re
View File

@ -1,5 +0,0 @@
type t = ExpressionTypes.Program.program;
let last = (r:t) => E.A.last(r) |> E.O.toResult("No rendered lines");
// let run = (p:program) => p |> E.A.last |> E.O.fmap(r =>
// )

30
packages/playground/src/distPlus/samplesRenderer/Bandwidth.re
View File

@ -1,30 +0,0 @@
//The math here was taken from https://github.com/jasondavies/science.js/blob/master/src/stats/bandwidth.js
let len = x => E.A.length(x) |> float_of_int;
let iqr = x => {
Jstat.percentile(x, 0.75, true) -. Jstat.percentile(x, 0.25, true);
};
// Silverman, B. W. (1986) Density Estimation. London: Chapman and Hall.
let nrd0 = x => {
let hi = Js_math.sqrt(Jstat.variance(x));
let lo = Js_math.minMany_float([|hi, iqr(x) /. 1.34|]);
let e = Js_math.abs_float(x[1]);
let lo' =
switch (lo, hi, e) {
| (lo, _, _) when !Js.Float.isNaN(lo) => lo
| (_, hi, _) when !Js.Float.isNaN(hi) => hi
| (_, _, e) when !Js.Float.isNaN(e) => e
| _ => 1.0
};
0.9 *. lo' *. Js.Math.pow_float(~base=len(x), ~exp=-0.2);
};
// Scott, D. W. (1992) Multivariate Density Estimation: Theory, Practice, and Visualization. Wiley.
let nrd = x => {
let h = iqr(x) /. 1.34;
1.06
*. Js.Math.min_float(Js.Math.sqrt(Jstat.variance(x)), h)
*. Js.Math.pow_float(~base=len(x), ~exp=(-1.0) /. 5.0);
};

21
packages/playground/src/distPlus/samplesRenderer/KdeLibrary.js
View File

@ -1,21 +0,0 @@
const pdfast = require('pdfast');
const _ = require("lodash");
const samplesToContinuousPdf = (
samples,
size,
width,
min = false,
max = false,
) => {
let _samples = _.filter(samples, _.isFinite);
if (_.isFinite(min)) { _samples = _.filter(_samples, r => r > min) };
if (_.isFinite(max)) { _samples = _.filter(_samples, r => r < max) };
let pdf = pdfast.create(_samples, { size, width });
return {xs: pdf.map(r => r.x), ys: pdf.map(r => r.y)};
};
module.exports = {
samplesToContinuousPdf,
};

164
packages/playground/src/distPlus/samplesRenderer/SamplesToShape.re
View File

@ -1,164 +0,0 @@
module Internals = {
module Types = {
type samplingStats = {
sampleCount: int,
outputXYPoints: int,
bandwidthXSuggested: float,
bandwidthUnitSuggested: float,
bandwidthXImplemented: float,
bandwidthUnitImplemented: float,
};
type outputs = {
continuousParseParams: option(samplingStats),
shape: option(DistTypes.shape),
};
};
module JS = {
[@bs.deriving abstract]
type distJs = {
xs: array(float),
ys: array(float),
};
let jsToDist = (d: distJs): DistTypes.xyShape => {
xs: xsGet(d),
ys: ysGet(d),
};
[@bs.module "./KdeLibrary.js"]
external samplesToContinuousPdf: (array(float), int, int) => distJs =
"samplesToContinuousPdf";
};
module KDE = {
let normalSampling = (samples, outputXYPoints, kernelWidth) => {
samples
|> JS.samplesToContinuousPdf(_, outputXYPoints, kernelWidth)
|> JS.jsToDist;
};
};
module T = {
type t = array(float);
let splitContinuousAndDiscrete = (sortedArray: t) => {
let continuous = [||];
let discrete = E.FloatFloatMap.empty();
Belt.Array.forEachWithIndex(
sortedArray,
(index, element) => {
let maxIndex = (sortedArray |> Array.length) - 1;
let possiblySimilarElements =
(
switch (index) {
| 0 => [|index + 1|]
| n when n == maxIndex => [|index - 1|]
| _ => [|index - 1, index + 1|]
}
)
|> Belt.Array.map(_, r => sortedArray[r]);
let hasSimilarElement =
Belt.Array.some(possiblySimilarElements, r => r == element);
hasSimilarElement
? E.FloatFloatMap.increment(element, discrete)
: {
let _ = Js.Array.push(element, continuous);
();
};
();
},
);
(continuous, discrete);
};
let xWidthToUnitWidth = (samples, outputXYPoints, xWidth) => {
let xyPointRange = E.A.Sorted.range(samples) |> E.O.default(0.0);
let xyPointWidth = xyPointRange /. float_of_int(outputXYPoints);
xWidth /. xyPointWidth;
};
let formatUnitWidth = w => Jstat.max([|w, 1.0|]) |> int_of_float;
let suggestedUnitWidth = (samples, outputXYPoints) => {
let suggestedXWidth = Bandwidth.nrd0(samples);
xWidthToUnitWidth(samples, outputXYPoints, suggestedXWidth);
};
let kde = (~samples, ~outputXYPoints, width) => {
KDE.normalSampling(samples, outputXYPoints, width);
};
};
};
let toShape =
(
~samples: Internals.T.t,
~samplingInputs: ExpressionTypes.ExpressionTree.samplingInputs,
(),
) => {
Array.fast_sort(compare, samples);
let (continuousPart, discretePart) = E.A.Sorted.Floats.split(samples);
let length = samples |> E.A.length |> float_of_int;
let discrete: DistTypes.discreteShape =
discretePart
|> E.FloatFloatMap.fmap(r => r /. length)
|> E.FloatFloatMap.toArray
|> XYShape.T.fromZippedArray
|> Discrete.make;
let pdf =
continuousPart |> E.A.length > 5
? {
let _suggestedXWidth = Bandwidth.nrd0(continuousPart);
// todo: This does some recalculating from the last step.
let _suggestedUnitWidth =
Internals.T.suggestedUnitWidth(
continuousPart,
samplingInputs.outputXYPoints,
);
let usedWidth =
samplingInputs.kernelWidth |> E.O.default(_suggestedXWidth);
let usedUnitWidth =
Internals.T.xWidthToUnitWidth(
samples,
samplingInputs.outputXYPoints,
usedWidth,
);
let samplingStats: Internals.Types.samplingStats = {
sampleCount: samplingInputs.sampleCount,
outputXYPoints: samplingInputs.outputXYPoints,
bandwidthXSuggested: _suggestedXWidth,
bandwidthUnitSuggested: _suggestedUnitWidth,
bandwidthXImplemented: usedWidth,
bandwidthUnitImplemented: usedUnitWidth,
};
continuousPart
|> Internals.T.kde(
~samples=_,
~outputXYPoints=samplingInputs.outputXYPoints,
Internals.T.formatUnitWidth(usedUnitWidth),
)
|> Continuous.make
|> (r => Some((r, samplingStats)));
}
: None;
let shape =
MixedShapeBuilder.buildSimple(
~continuous=pdf |> E.O.fmap(fst),
~discrete=Some(discrete),
);
let samplesParse: Internals.Types.outputs = {
continuousParseParams: pdf |> E.O.fmap(snd),
shape,
};
samplesParse;
};
let fromSamples = (~samplingInputs, samples) => {
toShape(~samples, ~samplingInputs, ());
};

346
packages/playground/src/distPlus/symbolic/SymbolicDist.re
View File

@ -1,346 +0,0 @@
open SymbolicTypes;
module Exponential = {
type t = exponential;
let make = (rate:float): symbolicDist =>
`Exponential(
{
rate:rate
},
);
let pdf = (x, t: t) => Jstat.exponential##pdf(x, t.rate);
let cdf = (x, t: t) => Jstat.exponential##cdf(x, t.rate);
let inv = (p, t: t) => Jstat.exponential##inv(p, t.rate);
let sample = (t: t) => Jstat.exponential##sample(t.rate);
let mean = (t: t) => Ok(Jstat.exponential##mean(t.rate));
let toString = ({rate}: t) => {j|Exponential($rate)|j};
};
module Cauchy = {
type t = cauchy;
let make = (local, scale): symbolicDist => `Cauchy({local, scale});
let pdf = (x, t: t) => Jstat.cauchy##pdf(x, t.local, t.scale);
let cdf = (x, t: t) => Jstat.cauchy##cdf(x, t.local, t.scale);
let inv = (p, t: t) => Jstat.cauchy##inv(p, t.local, t.scale);
let sample = (t: t) => Jstat.cauchy##sample(t.local, t.scale);
let mean = (_: t) => Error("Cauchy distributions have no mean value.");
let toString = ({local, scale}: t) => {j|Cauchy($local, $scale)|j};
};
module Triangular = {
type t = triangular;
let make = (low, medium, high): result(symbolicDist, string) =>
low < medium && medium < high
? Ok(`Triangular({low, medium, high}))
: Error("Triangular values must be increasing order");
let pdf = (x, t: t) => Jstat.triangular##pdf(x, t.low, t.high, t.medium);
let cdf = (x, t: t) => Jstat.triangular##cdf(x, t.low, t.high, t.medium);
let inv = (p, t: t) => Jstat.triangular##inv(p, t.low, t.high, t.medium);
let sample = (t: t) => Jstat.triangular##sample(t.low, t.high, t.medium);
let mean = (t: t) => Ok(Jstat.triangular##mean(t.low, t.high, t.medium));
let toString = ({low, medium, high}: t) => {j|Triangular($low, $medium, $high)|j};
};
module Normal = {
type t = normal;
let make = (mean, stdev): symbolicDist => `Normal({mean, stdev});
let pdf = (x, t: t) => Jstat.normal##pdf(x, t.mean, t.stdev);
let cdf = (x, t: t) => Jstat.normal##cdf(x, t.mean, t.stdev);
let from90PercentCI = (low, high) => {
let mean = E.A.Floats.mean([|low, high|]);
let stdev = (high -. low) /. (2. *. 1.644854);
`Normal({mean, stdev});
};
let inv = (p, t: t) => Jstat.normal##inv(p, t.mean, t.stdev);
let sample = (t: t) => Jstat.normal##sample(t.mean, t.stdev);
let mean = (t: t) => Ok(Jstat.normal##mean(t.mean, t.stdev));
let toString = ({mean, stdev}: t) => {j|Normal($mean,$stdev)|j};
let add = (n1: t, n2: t) => {
let mean = n1.mean +. n2.mean;
let stdev = sqrt(n1.stdev ** 2. +. n2.stdev ** 2.);
`Normal({mean, stdev});
};
let subtract = (n1: t, n2: t) => {
let mean = n1.mean -. n2.mean;
let stdev = sqrt(n1.stdev ** 2. +. n2.stdev ** 2.);
`Normal({mean, stdev});
};
// TODO: is this useful here at all? would need the integral as well ...
let pointwiseProduct = (n1: t, n2: t) => {
let mean =
(n1.mean *. n2.stdev ** 2. +. n2.mean *. n1.stdev ** 2.)
/. (n1.stdev ** 2. +. n2.stdev ** 2.);
let stdev = 1. /. (1. /. n1.stdev ** 2. +. 1. /. n2.stdev ** 2.);
`Normal({mean, stdev});
};
let operate = (operation: Operation.Algebraic.t, n1: t, n2: t) =>
switch (operation) {
| `Add => Some(add(n1, n2))
| `Subtract => Some(subtract(n1, n2))
| _ => None
};
};
module Beta = {
type t = beta;
let make = (alpha, beta) => `Beta({alpha, beta});
let pdf = (x, t: t) => Jstat.beta##pdf(x, t.alpha, t.beta);
let cdf = (x, t: t) => Jstat.beta##cdf(x, t.alpha, t.beta);
let inv = (p, t: t) => Jstat.beta##inv(p, t.alpha, t.beta);
let sample = (t: t) => Jstat.beta##sample(t.alpha, t.beta);
let mean = (t: t) => Ok(Jstat.beta##mean(t.alpha, t.beta));
let toString = ({alpha, beta}: t) => {j|Beta($alpha,$beta)|j};
};
module Lognormal = {
type t = lognormal;
let make = (mu, sigma) => `Lognormal({mu, sigma});
let pdf = (x, t: t) => Jstat.lognormal##pdf(x, t.mu, t.sigma);
let cdf = (x, t: t) => Jstat.lognormal##cdf(x, t.mu, t.sigma);
let inv = (p, t: t) => Jstat.lognormal##inv(p, t.mu, t.sigma);
let mean = (t: t) => Ok(Jstat.lognormal##mean(t.mu, t.sigma));
let sample = (t: t) => Jstat.lognormal##sample(t.mu, t.sigma);
let toString = ({mu, sigma}: t) => {j|Lognormal($mu,$sigma)|j};
let from90PercentCI = (low, high) => {
let logLow = Js.Math.log(low);
let logHigh = Js.Math.log(high);
let mu = E.A.Floats.mean([|logLow, logHigh|]);
let sigma = (logHigh -. logLow) /. (2.0 *. 1.645);
`Lognormal({mu, sigma});
};
let fromMeanAndStdev = (mean, stdev) => {
let variance = Js.Math.pow_float(~base=stdev, ~exp=2.0);
let meanSquared = Js.Math.pow_float(~base=mean, ~exp=2.0);
let mu =
Js.Math.log(mean) -. 0.5 *. Js.Math.log(variance /. meanSquared +. 1.0);
let sigma =
Js.Math.pow_float(
~base=Js.Math.log(variance /. meanSquared +. 1.0),
~exp=0.5,
);
`Lognormal({mu, sigma});
};
let multiply = (l1, l2) => {
let mu = l1.mu +. l2.mu;
let sigma = l1.sigma +. l2.sigma;
`Lognormal({mu, sigma});
};
let divide = (l1, l2) => {
let mu = l1.mu -. l2.mu;
let sigma = l1.sigma +. l2.sigma;
`Lognormal({mu, sigma});
};
let operate = (operation: Operation.Algebraic.t, n1: t, n2: t) =>
switch (operation) {
| `Multiply => Some(multiply(n1, n2))
| `Divide => Some(divide(n1, n2))
| _ => None
};
};
module Uniform = {
type t = uniform;
let make = (low, high) => `Uniform({low, high});
let pdf = (x, t: t) => Jstat.uniform##pdf(x, t.low, t.high);
let cdf = (x, t: t) => Jstat.uniform##cdf(x, t.low, t.high);
let inv = (p, t: t) => Jstat.uniform##inv(p, t.low, t.high);
let sample = (t: t) => Jstat.uniform##sample(t.low, t.high);
let mean = (t: t) => Ok(Jstat.uniform##mean(t.low, t.high));
let toString = ({low, high}: t) => {j|Uniform($low,$high)|j};
let truncate = (low, high, t: t): t => {
let newLow = max(E.O.default(neg_infinity, low), t.low);
let newHigh = min(E.O.default(infinity, high), t.high);
{low: newLow, high: newHigh};
};
};
module Float = {
type t = float;
let make = t => `Float(t);
let pdf = (x, t: t) => x == t ? 1.0 : 0.0;
let cdf = (x, t: t) => x >= t ? 1.0 : 0.0;
let inv = (p, t: t) => p < t ? 0.0 : 1.0;
let mean = (t: t) => Ok(t);
let sample = (t: t) => t;
let toString = Js.Float.toString;
};
module T = {
let minCdfValue = 0.0001;
let maxCdfValue = 0.9999;
let pdf = (x, dist) =>
switch (dist) {
| `Normal(n) => Normal.pdf(x, n)
| `Triangular(n) => Triangular.pdf(x, n)
| `Exponential(n) => Exponential.pdf(x, n)
| `Cauchy(n) => Cauchy.pdf(x, n)
| `Lognormal(n) => Lognormal.pdf(x, n)
| `Uniform(n) => Uniform.pdf(x, n)
| `Beta(n) => Beta.pdf(x, n)
| `Float(n) => Float.pdf(x, n)
};
let cdf = (x, dist) =>
switch (dist) {
| `Normal(n) => Normal.cdf(x, n)
| `Triangular(n) => Triangular.cdf(x, n)
| `Exponential(n) => Exponential.cdf(x, n)
| `Cauchy(n) => Cauchy.cdf(x, n)
| `Lognormal(n) => Lognormal.cdf(x, n)
| `Uniform(n) => Uniform.cdf(x, n)
| `Beta(n) => Beta.cdf(x, n)
| `Float(n) => Float.cdf(x, n)
};
let inv = (x, dist) =>
switch (dist) {
| `Normal(n) => Normal.inv(x, n)
| `Triangular(n) => Triangular.inv(x, n)
| `Exponential(n) => Exponential.inv(x, n)
| `Cauchy(n) => Cauchy.inv(x, n)
| `Lognormal(n) => Lognormal.inv(x, n)
| `Uniform(n) => Uniform.inv(x, n)
| `Beta(n) => Beta.inv(x, n)
| `Float(n) => Float.inv(x, n)
};
let sample: symbolicDist => float =
fun
| `Normal(n) => Normal.sample(n)
| `Triangular(n) => Triangular.sample(n)
| `Exponential(n) => Exponential.sample(n)
| `Cauchy(n) => Cauchy.sample(n)
| `Lognormal(n) => Lognormal.sample(n)
| `Uniform(n) => Uniform.sample(n)
| `Beta(n) => Beta.sample(n)
| `Float(n) => Float.sample(n);
let doN = (n, fn) => {
let items = Belt.Array.make(n, 0.0);
for (x in 0 to n - 1) {
let _ = Belt.Array.set(items, x, fn());
();
};
items;
};
let sampleN = (n, dist) => {
doN(n, () => sample(dist));
};
let toString: symbolicDist => string =
fun
| `Triangular(n) => Triangular.toString(n)
| `Exponential(n) => Exponential.toString(n)
| `Cauchy(n) => Cauchy.toString(n)
| `Normal(n) => Normal.toString(n)
| `Lognormal(n) => Lognormal.toString(n)
| `Uniform(n) => Uniform.toString(n)
| `Beta(n) => Beta.toString(n)
| `Float(n) => Float.toString(n);
let min: symbolicDist => float =
fun
| `Triangular({low}) => low
| `Exponential(n) => Exponential.inv(minCdfValue, n)
| `Cauchy(n) => Cauchy.inv(minCdfValue, n)
| `Normal(n) => Normal.inv(minCdfValue, n)
| `Lognormal(n) => Lognormal.inv(minCdfValue, n)
| `Uniform({low}) => low
| `Beta(n) => Beta.inv(minCdfValue, n)
| `Float(n) => n;
let max: symbolicDist => float =
fun
| `Triangular(n) => n.high
| `Exponential(n) => Exponential.inv(maxCdfValue, n)
| `Cauchy(n) => Cauchy.inv(maxCdfValue, n)
| `Normal(n) => Normal.inv(maxCdfValue, n)
| `Lognormal(n) => Lognormal.inv(maxCdfValue, n)
| `Beta(n) => Beta.inv(maxCdfValue, n)
| `Uniform({high}) => high
| `Float(n) => n;
let mean: symbolicDist => result(float, string) =
fun
| `Triangular(n) => Triangular.mean(n)
| `Exponential(n) => Exponential.mean(n)
| `Cauchy(n) => Cauchy.mean(n)
| `Normal(n) => Normal.mean(n)
| `Lognormal(n) => Lognormal.mean(n)
| `Beta(n) => Beta.mean(n)
| `Uniform(n) => Uniform.mean(n)
| `Float(n) => Float.mean(n);
let operate = (distToFloatOp: ExpressionTypes.distToFloatOperation, s) =>
switch (distToFloatOp) {
| `Cdf(f) => Ok(cdf(f, s))
| `Pdf(f) => Ok(pdf(f, s))
| `Inv(f) => Ok(inv(f, s))
| `Sample => Ok(sample(s))
| `Mean => mean(s)
};
let interpolateXs =
(~xSelection: [ | `Linear | `ByWeight]=`Linear, dist: symbolicDist, n) => {
switch (xSelection, dist) {
| (`Linear, _) => E.A.Floats.range(min(dist), max(dist), n)
| (`ByWeight, `Uniform(n)) =>
// In `ByWeight mode, uniform distributions get special treatment because we need two x's
// on either side for proper rendering (just left and right of the discontinuities).
let dx = 0.00001 *. (n.high -. n.low);
[|n.low -. dx, n.low +. dx, n.high -. dx, n.high +. dx|];
| (`ByWeight, _) =>
let ys = E.A.Floats.range(minCdfValue, maxCdfValue, n);
ys |> E.A.fmap(y => inv(y, dist));
};
};
/* Calling e.g. "Normal.operate" returns an optional that wraps a result.
If the optional is None, there is no valid analytic solution. If it Some, it
can still return an error if there is a serious problem,
like in the case of a divide by 0.
*/
let tryAnalyticalSimplification =
(
d1: symbolicDist,
d2: symbolicDist,
op: ExpressionTypes.algebraicOperation,
)
: analyticalSimplificationResult =>
switch (d1, d2) {
| (`Float(v1), `Float(v2)) =>
switch (Operation.Algebraic.applyFn(op, v1, v2)) {
| Ok(r) => `AnalyticalSolution(`Float(r))
| Error(n) => `Error(n)
}
| (`Normal(v1), `Normal(v2)) =>
Normal.operate(op, v1, v2)
|> E.O.dimap(r => `AnalyticalSolution(r), () => `NoSolution)
| (`Lognormal(v1), `Lognormal(v2)) =>
Lognormal.operate(op, v1, v2)
|> E.O.dimap(r => `AnalyticalSolution(r), () => `NoSolution)
| _ => `NoSolution
};
let toShape = (sampleCount, d: symbolicDist): DistTypes.shape =>
switch (d) {
| `Float(v) =>
Discrete(
Discrete.make(
~integralSumCache=Some(1.0),
{xs: [|v|], ys: [|1.0|]},
),
)
| _ =>
let xs = interpolateXs(~xSelection=`ByWeight, d, sampleCount);
let ys = xs |> E.A.fmap(x => pdf(x, d));
Continuous(Continuous.make(~integralSumCache=Some(1.0), {xs, ys}));
};
};

49
packages/playground/src/distPlus/symbolic/SymbolicTypes.re
View File

@ -1,49 +0,0 @@
type normal = {
mean: float,
stdev: float,
};
type lognormal = {
mu: float,
sigma: float,
};
type uniform = {
low: float,
high: float,
};
type beta = {
alpha: float,
beta: float,
};
type exponential = {rate: float};
type cauchy = {
local: float,
scale: float,
};
type triangular = {
low: float,
medium: float,
high: float,
};
type symbolicDist = [
| `Normal(normal)
| `Beta(beta)
| `Lognormal(lognormal)
| `Uniform(uniform)
| `Exponential(exponential)
| `Cauchy(cauchy)
| `Triangular(triangular)
| `Float(float) // Dirac delta at x. Practically useful only in the context of multimodals.
];
type analyticalSimplificationResult = [
| `AnalyticalSolution(symbolicDist)
| `Error(string)
| `NoSolution
];

258
packages/playground/src/distPlus/typeSystem/HardcodedFunctions.re
View File

@ -1,258 +0,0 @@
open TypeSystem;
let wrongInputsError = (r: array(typedValue)) => {
let inputs = r |> E.A.fmap(TypedValue.toString) |>Js.String.concatMany(_, ",");
Js.log3("Inputs were", inputs, r);
Error("Wrong inputs. The inputs were:" ++ inputs);
};
let to_: (float, float) => result(node, string) =
(low, high) =>
switch (low, high) {
| (low, high) when low <= 0.0 && low < high =>
Ok(`SymbolicDist(SymbolicDist.Normal.from90PercentCI(low, high)))
| (low, high) when low < high =>
Ok(`SymbolicDist(SymbolicDist.Lognormal.from90PercentCI(low, high)))
| (_, _) => Error("Low value must be less than high value.")
};
let makeSymbolicFromTwoFloats = (name, fn) =>
Function.T.make(
~name,
~outputType=`SamplingDistribution,
~inputTypes=[|`Float, `Float|],
~run=
fun
| [|`Float(a), `Float(b)|] => Ok(`SymbolicDist(fn(a, b)))
| e => wrongInputsError(e),
(),
);
let makeSymbolicFromOneFloat = (name, fn) =>
Function.T.make(
~name,
~outputType=`SamplingDistribution,
~inputTypes=[|`Float|],
~run=
fun
| [|`Float(a)|] => Ok(`SymbolicDist(fn(a)))
| e => wrongInputsError(e),
(),
);
let makeDistFloat = (name, fn) =>
Function.T.make(
~name,
~outputType=`SamplingDistribution,
~inputTypes=[|`SamplingDistribution, `Float|],
~run=
fun
| [|`SamplingDist(a), `Float(b)|] => fn(a, b)
| [|`RenderedDist(a), `Float(b)|] => fn(`RenderedDist(a), b)
| e => wrongInputsError(e),
(),
);
let makeRenderedDistFloat = (name, fn) =>
Function.T.make(
~name,
~outputType=`RenderedDistribution,
~inputTypes=[|`RenderedDistribution, `Float|],
~shouldCoerceTypes=true,
~run=
fun
| [|`RenderedDist(a), `Float(b)|] => fn(a, b)
| e => wrongInputsError(e),
(),
);
let makeDist = (name, fn) =>
Function.T.make(
~name,
~outputType=`SamplingDistribution,
~inputTypes=[|`SamplingDistribution|],
~run=
fun
| [|`SamplingDist(a)|] => fn(a)
| [|`RenderedDist(a)|] => fn(`RenderedDist(a))
| e => wrongInputsError(e),
(),
);
let floatFromDist =
(
distToFloatOp: ExpressionTypes.distToFloatOperation,
t: TypeSystem.samplingDist,
)
: result(node, string) => {
switch (t) {
| `SymbolicDist(s) =>
SymbolicDist.T.operate(distToFloatOp, s)
|> E.R.bind(_, v => Ok(`SymbolicDist(`Float(v))))
| `RenderedDist(rs) =>
Shape.operate(distToFloatOp, rs) |> (v => Ok(`SymbolicDist(`Float(v))))
};
};
let verticalScaling = (scaleOp, rs, scaleBy) => {
// scaleBy has to be a single float, otherwise we'll return an error.
let fn = (secondary, main) =>
Operation.Scale.toFn(scaleOp, main, secondary);
let integralSumCacheFn = Operation.Scale.toIntegralSumCacheFn(scaleOp);
let integralCacheFn = Operation.Scale.toIntegralCacheFn(scaleOp);
Ok(
`RenderedDist(
Shape.T.mapY(
~integralSumCacheFn=integralSumCacheFn(scaleBy),
~integralCacheFn=integralCacheFn(scaleBy),
~fn=fn(scaleBy),
rs,
),
),
);
};
module Multimodal = {
let getByNameResult = ExpressionTypes.ExpressionTree.Hash.getByNameResult;
let _paramsToDistsAndWeights = (r: array(typedValue)) =>
switch (r) {
| [|`Hash(r)|] =>
let dists =
getByNameResult(r, "dists")
->E.R.bind(TypeSystem.TypedValue.toArray)
->E.R.bind(r =>
r
|> E.A.fmap(TypeSystem.TypedValue.toDist)
|> E.A.R.firstErrorOrOpen
);
let weights =
getByNameResult(r, "weights")
->E.R.bind(TypeSystem.TypedValue.toArray)
->E.R.bind(r =>
r
|> E.A.fmap(TypeSystem.TypedValue.toFloat)
|> E.A.R.firstErrorOrOpen
);
E.R.merge(dists, weights)
|> E.R.fmap(((a, b)) =>
E.A.zipMaxLength(a, b)
|> E.A.fmap(((a, b)) =>
(a |> E.O.toExn(""), b |> E.O.default(1.0))
)
);
| _ => Error("Needs items")
};
let _runner: array(typedValue) => result(node, string) =
r => {
let paramsToDistsAndWeights =
_paramsToDistsAndWeights(r)
|> E.R.fmap(
E.A.fmap(((dist, weight)) =>
`FunctionCall((
"scaleMultiply",
[|dist, `SymbolicDist(`Float(weight))|],
))
),
);
let pointwiseSum: result(node, string) =
paramsToDistsAndWeights->E.R.bind(
E.R.errorIfCondition(E.A.isEmpty, "Needs one input"),
)
|> E.R.fmap(r =>
r
|> Js.Array.sliceFrom(1)
|> E.A.fold_left(
(acc, x) => {`PointwiseCombination((`Add, acc, x))},
E.A.unsafe_get(r, 0),
)
);
pointwiseSum;
};
let _function =
Function.T.make(
~name="multimodal",
~outputType=`SamplingDistribution,
~inputTypes=[|
`Hash([|
("dists", `Array(`SamplingDistribution)),
("weights", `Array(`Float)),
|]),
|],
~run=_runner,
(),
);
};
let all = [|
makeSymbolicFromTwoFloats("normal", SymbolicDist.Normal.make),
makeSymbolicFromTwoFloats("uniform", SymbolicDist.Uniform.make),
makeSymbolicFromTwoFloats("beta", SymbolicDist.Beta.make),
makeSymbolicFromTwoFloats("lognormal", SymbolicDist.Lognormal.make),
makeSymbolicFromTwoFloats(
"lognormalFromMeanAndStdDev",
SymbolicDist.Lognormal.fromMeanAndStdev,
),
makeSymbolicFromOneFloat("exponential", SymbolicDist.Exponential.make),
Function.T.make(
~name="to",
~outputType=`SamplingDistribution,
~inputTypes=[|`Float, `Float|],
~run=
fun
| [|`Float(a), `Float(b)|] => to_(a, b)
| e => wrongInputsError(e),
(),
),
Function.T.make(
~name="triangular",
~outputType=`SamplingDistribution,
~inputTypes=[|`Float, `Float, `Float|],
~run=
fun
| [|`Float(a), `Float(b), `Float(c)|] =>
SymbolicDist.Triangular.make(a, b, c)
|> E.R.fmap(r => `SymbolicDist(r))
| e => wrongInputsError(e),
(),
),
makeDistFloat("pdf", (dist, float) => floatFromDist(`Pdf(float), dist)),
makeDistFloat("inv", (dist, float) => floatFromDist(`Inv(float), dist)),
makeDistFloat("cdf", (dist, float) => floatFromDist(`Cdf(float), dist)),
makeDist("mean", dist => floatFromDist(`Mean, dist)),
makeDist("sample", dist => floatFromDist(`Sample, dist)),
Function.T.make(
~name="render",
~outputType=`RenderedDistribution,
~inputTypes=[|`RenderedDistribution|],
~run=
fun
| [|`RenderedDist(c)|] => Ok(`RenderedDist(c))
| e => wrongInputsError(e),
(),
),
Function.T.make(
~name="normalize",
~outputType=`SamplingDistribution,
~inputTypes=[|`SamplingDistribution|],
~run=
fun
| [|`SamplingDist(`SymbolicDist(c))|] => Ok(`SymbolicDist(c))
| [|`SamplingDist(`RenderedDist(c))|] =>
Ok(`RenderedDist(Shape.T.normalize(c)))
| e => wrongInputsError(e),
(),
),
makeRenderedDistFloat("scaleExp", (dist, float) =>
verticalScaling(`Exponentiate, dist, float)
),
makeRenderedDistFloat("scaleMultiply", (dist, float) =>
verticalScaling(`Multiply, dist, float)
),
makeRenderedDistFloat("scaleLog", (dist, float) =>
verticalScaling(`Log, dist, float)
),
Multimodal._function
|];

228
packages/playground/src/distPlus/typeSystem/TypeSystem.re
View File

@ -1,228 +0,0 @@
type node = ExpressionTypes.ExpressionTree.node;
let getFloat = ExpressionTypes.ExpressionTree.getFloat;
type samplingDist = [
| `SymbolicDist(SymbolicTypes.symbolicDist)
| `RenderedDist(DistTypes.shape)
];
type hashType = array((string, _type))
and _type = [
| `Float
| `SamplingDistribution
| `RenderedDistribution
| `Array(_type)
| `Hash(hashType)
];
type hashTypedValue = array((string, typedValue))
and typedValue = [
| `Float(float)
| `RenderedDist(DistTypes.shape)
| `SamplingDist(samplingDist)
| `Array(array(typedValue))
| `Hash(hashTypedValue)
];
type _function = {
name: string,
inputTypes: array(_type),
outputType: _type,
run: array(typedValue) => result(node, string),
shouldCoerceTypes: bool,
};
type functions = array(_function);
type inputNodes = array(node);
module TypedValue = {
let rec toString: typedValue => string =
fun
| `SamplingDist(_) => "[sampling dist]"
| `RenderedDist(_) => "[rendered Shape]"
| `Float(f) => "Float: " ++ Js.Float.toString(f)
| `Array(a) =>
"[" ++ (a |> E.A.fmap(toString) |> Js.String.concatMany(_, ",")) ++ "]"
| `Hash(v) =>
"{"
++ (
v
|> E.A.fmap(((name, value)) => name ++ ":" ++ toString(value))
|> Js.String.concatMany(_, ",")
)
++ "}";
let rec fromNode = (node: node): result(typedValue, string) =>
switch (node) {
| `SymbolicDist(`Float(r)) => Ok(`Float(r))
| `SymbolicDist(s) => Ok(`SamplingDist(`SymbolicDist(s)))
| `RenderedDist(s) => Ok(`RenderedDist(s))
| `Array(r) =>
r
|> E.A.fmap(fromNode)
|> E.A.R.firstErrorOrOpen
|> E.R.fmap(r => `Array(r))
| `Hash(hash) =>
hash
|> E.A.fmap(((name, t)) => fromNode(t) |> E.R.fmap(r => (name, r)))
|> E.A.R.firstErrorOrOpen
|> E.R.fmap(r => `Hash(r))
| e => Error("Wrong type: " ++ ExpressionTreeBasic.toString(e))
};
// todo: Arrays and hashes
let rec fromNodeWithTypeCoercion = (evaluationParams, _type: _type, node) => {
switch (_type, node) {
| (`Float, _) =>
switch (getFloat(node)) {
| Some(a) => Ok(`Float(a))
| _ => Error("Type Error: Expected float.")
}
| (`SamplingDistribution, _) =>
PTypes.SamplingDistribution.renderIfIsNotSamplingDistribution(
evaluationParams,
node,
)
|> E.R.bind(_, fromNode)
| (`RenderedDistribution, _) =>{
ExpressionTypes.ExpressionTree.Render.render(evaluationParams, node)
|> E.R.bind(_, fromNode);
}
| (`Array(_type), `Array(b)) =>
b
|> E.A.fmap(fromNodeWithTypeCoercion(evaluationParams, _type))
|> E.A.R.firstErrorOrOpen
|> E.R.fmap(r => `Array(r))
| (`Hash(named), `Hash(r)) =>
let keyValues =
named
|> E.A.fmap(((name, intendedType)) =>
(
name,
intendedType,
ExpressionTypes.ExpressionTree.Hash.getByName(r, name),
)
);
let typedHash =
keyValues
|> E.A.fmap(((name, intendedType, optionNode)) =>
switch (optionNode) {
| Some(node) =>
fromNodeWithTypeCoercion(evaluationParams, intendedType, node)
|> E.R.fmap(node => (name, node))
| None => Error("Hash parameter not present in hash.")
}
)
|> E.A.R.firstErrorOrOpen
|> E.R.fmap(r => `Hash(r));
typedHash;
| _ => Error("fromNodeWithTypeCoercion error, sorry.")
};
};
let toFloat: typedValue => result(float, string) =
fun
| `Float(x) => Ok(x)
| _ => Error("Not a float");
let toArray: typedValue => result(array('a), string) =
fun
| `Array(x) => Ok(x)
| _ => Error("Not an array");
let toNamed: typedValue => result(hashTypedValue, string) =
fun
| `Hash(x) => Ok(x)
| _ => Error("Not a named item");
let toDist: typedValue => result(node,string) =
fun
| `SamplingDist(`SymbolicDist(c)) => Ok(`SymbolicDist(c))
| `SamplingDist(`RenderedDist(c)) => Ok(`RenderedDist(c))
| `RenderedDist(c) => Ok(`RenderedDist(c))
| `Float(x) => Ok(`SymbolicDist(`Float(x)))
| x => Error("Cannot be converted into a distribution: " ++ toString(x));
};
module Function = {
type t = _function;
type ts = functions;
module T = {
let make =
(~name, ~inputTypes, ~outputType, ~run, ~shouldCoerceTypes=true, _): t => {
name,
inputTypes,
outputType,
run,
shouldCoerceTypes,
};
let _inputLengthCheck = (inputNodes: inputNodes, t: t) => {
let expectedLength = E.A.length(t.inputTypes);
let actualLength = E.A.length(inputNodes);
expectedLength == actualLength
? Ok(inputNodes)
: Error(
"Wrong number of inputs. Expected"
++ (expectedLength |> E.I.toString)
++ ". Got:"
++ (actualLength |> E.I.toString),
);
};
let _coerceInputNodes =
(evaluationParams, inputTypes, shouldCoerce, inputNodes) =>
Belt.Array.zip(inputTypes, inputNodes)
|> E.A.fmap(((def, input)) =>
shouldCoerce
? TypedValue.fromNodeWithTypeCoercion(
evaluationParams,
def,
input,
)
: TypedValue.fromNode(input)
)
|> E.A.R.firstErrorOrOpen;
let inputsToTypedValues =
(
evaluationParams: ExpressionTypes.ExpressionTree.evaluationParams,
inputNodes: inputNodes,
t: t,
) => {
_inputLengthCheck(inputNodes, t)
->E.R.bind(
_coerceInputNodes(
evaluationParams,
t.inputTypes,
t.shouldCoerceTypes,
),
)
};
let run =
(
evaluationParams: ExpressionTypes.ExpressionTree.evaluationParams,
inputNodes: inputNodes,
t: t,
) => {
inputsToTypedValues(evaluationParams, inputNodes, t)->E.R.bind(t.run)
|> (
fun
| Ok(i) => Ok(i)
| Error(r) => {
Error("Function " ++ t.name ++ " error: " ++ r);
}
);
};
};
module Ts = {
let findByName = (ts: ts, n: string) =>
ts |> Belt.Array.getBy(_, ({name}) => name == n);
let findByNameAndRun = (ts: ts, n: string, evaluationParams, inputTypes) =>
findByName(ts, n) |> E.O.fmap(T.run(evaluationParams, inputTypes));
};
};

112
packages/playground/src/distPlus/utility/Jstat.re
View File

@ -1,112 +0,0 @@
// Todo: Another way of doing this is with [@bs.scope "normal"], which may be more elegant
type normal = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
[@bs.meth] "inv": (float, float, float) => float,
[@bs.meth] "sample": (float, float) => float,
[@bs.meth] "mean": (float, float) => float,
};
type lognormal = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
[@bs.meth] "inv": (float, float, float) => float,
[@bs.meth] "sample": (float, float) => float,
[@bs.meth] "mean": (float, float) => float,
};
type uniform = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
[@bs.meth] "inv": (float, float, float) => float,
[@bs.meth] "sample": (float, float) => float,
[@bs.meth] "mean": (float, float) => float,
};
type beta = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
[@bs.meth] "inv": (float, float, float) => float,
[@bs.meth] "sample": (float, float) => float,
[@bs.meth] "mean": (float, float) => float,
};
type exponential = {
.
[@bs.meth] "pdf": (float, float) => float,
[@bs.meth] "cdf": (float, float) => float,
[@bs.meth] "inv": (float, float) => float,
[@bs.meth] "sample": float => float,
[@bs.meth] "mean": float => float,
};
type cauchy = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
[@bs.meth] "inv": (float, float, float) => float,
[@bs.meth] "sample": (float, float) => float,
};
type triangular = {
.
[@bs.meth] "pdf": (float, float, float, float) => float,
[@bs.meth] "cdf": (float, float, float, float) => float,
[@bs.meth] "inv": (float, float, float, float) => float,
[@bs.meth] "sample": (float, float, float) => float,
[@bs.meth] "mean": (float, float, float) => float,
};
// Pareto doesn't have sample for some reason
type pareto = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
[@bs.meth] "inv": (float, float, float) => float,
};
type poisson = {
.
[@bs.meth] "pdf": (float, float) => float,
[@bs.meth] "cdf": (float, float) => float,
[@bs.meth] "sample": float => float,
[@bs.meth] "mean": float => float,
};
type weibull = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
[@bs.meth] "inv": (float, float, float) => float,
[@bs.meth] "sample": (float, float) => float,
[@bs.meth] "mean": (float, float) => float,
};
type binomial = {
.
[@bs.meth] "pdf": (float, float, float) => float,
[@bs.meth] "cdf": (float, float, float) => float,
};
[@bs.module "jstat"] external normal: normal = "normal";
[@bs.module "jstat"] external lognormal: lognormal = "lognormal";
[@bs.module "jstat"] external uniform: uniform = "uniform";
[@bs.module "jstat"] external beta: beta = "beta";
[@bs.module "jstat"] external exponential: exponential = "exponential";
[@bs.module "jstat"] external cauchy: cauchy = "cauchy";
[@bs.module "jstat"] external triangular: triangular = "triangular";
[@bs.module "jstat"] external poisson: poisson = "poisson";
[@bs.module "jstat"] external pareto: pareto = "pareto";
[@bs.module "jstat"] external weibull: weibull = "weibull";
[@bs.module "jstat"] external binomial: binomial = "binomial";
[@bs.module "jstat"] external sum: array(float) => float = "sum";
[@bs.module "jstat"] external product: array(float) => float = "product";
[@bs.module "jstat"] external min: array(float) => float = "min";
[@bs.module "jstat"] external max: array(float) => float = "max";
[@bs.module "jstat"] external mean: array(float) => float = "mean";
[@bs.module "jstat"] external geomean: array(float) => float = "geomean";
[@bs.module "jstat"] external mode: array(float) => float = "mode";
[@bs.module "jstat"] external variance: array(float) => float = "variance";
[@bs.module "jstat"] external deviation: array(float) => float = "deviation";
[@bs.module "jstat"] external stdev: array(float) => float = "stdev";
[@bs.module "jstat"]
external quartiles: (array(float)) => array(float) = "quartiles";
[@bs.module "jstat"]
external quantiles: (array(float), array(float)) => array(float) = "quantiles";
[@bs.module "jstat"]
external percentile: (array(float), float, bool) => float = "percentile";

5
packages/playground/src/distPlus/utility/Lodash.re
View File

@ -1,5 +0,0 @@
[@bs.module "lodash"] external min: array('a) => 'a = "min";
[@bs.module "lodash"] external max: array('a) => 'a = "max";
[@bs.module "lodash"] external uniq: array('a) => array('a) = "uniq";
[@bs.module "lodash"]
external countBy: (array('a), 'a => 'b) => Js.Dict.t(int) = "countBy";

17941
packages/playground/yarn.lock
View File

File diff suppressed because it is too large Load Diff

2
packages/squiggle-lang/bsconfig.json
View File

@ -1,5 +1,5 @@
{
"name": "SquiggleExperimental",
"name": "@foretold-app/squiggle",
"reason": {},
"sources": [
{

45518
packages/squiggle-lang/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
packages/squiggle-lang/package.json
View File

@ -1,5 +1,5 @@
{
"name": "squiggle-experimental",
"name": "@foretold-app/squiggle",
"version": "0.1.9",
"homepage": "https://foretold-app.github.io/estiband/",
"private": false,

8
packages/squiggle-lang/src/distPlus/ProgramEvaluator.res
View File

@ -37,11 +37,11 @@ module Inputs = {
}
type \"export" = [
| #DistPlus(SquiggleExperimental.DistPlus.t)
| #DistPlus(DistPlus.t)
| #Float(float)
| #Function(
(array<string>, SquiggleExperimental.ExpressionTypes.ExpressionTree.node),
SquiggleExperimental.ExpressionTypes.ExpressionTree.environment,
(array<string>, ExpressionTypes.ExpressionTree.node),
ExpressionTypes.ExpressionTree.environment,
)
]
@ -124,7 +124,7 @@ let renderIfNeeded = (inputs: Inputs.inputs, node: ExpressionTypes.ExpressionTre
// TODO: Consider using ExpressionTypes.ExpressionTree.getFloat or similar in this function
let coersionToExportedTypes = (
inputs,
env: SquiggleExperimental.ExpressionTypes.ExpressionTree.environment,
env: ExpressionTypes.ExpressionTree.environment,
node: ExpressionTypes.ExpressionTree.node,
): result<\"export", string> =>
node

2
packages/squiggle-lang/src/distPlus/expressionTree/MathJsParser.res
View File

@ -170,7 +170,7 @@ module MathAdtToDistDst = {
let functionParser = (
nodeParser: MathJsonToMathJsAdt.arg => Belt.Result.t<
SquiggleExperimental.ExpressionTypes.ExpressionTree.node,
ExpressionTypes.ExpressionTree.node,
string,
>,
name: string,

96
packages/squiggle-lang/src/distPlus/utility/Jstat.res
View File

@ -1,85 +1,85 @@
// Todo: Another way of doing this is with [@bs.scope "normal"], which may be more elegant
module Normal = {
@module("jStat") @scope("normal") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("normal") external cdf: (float, float, float) => float = "cdf"
@module("jStat") @scope("normal") external inv: (float, float, float) => float = "inv"
@module("jStat") @scope("normal") external sample: (float, float) => float = "sample"
@module("jStat") @scope("normal") external mean: (float, float) => float = "mean"
@module("jstat") @scope("normal") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("normal") external cdf: (float, float, float) => float = "cdf"
@module("jstat") @scope("normal") external inv: (float, float, float) => float = "inv"
@module("jstat") @scope("normal") external sample: (float, float) => float = "sample"
@module("jstat") @scope("normal") external mean: (float, float) => float = "mean"
}
module Lognormal = {
@module("jStat") @scope("lognormal") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("lognormal") external cdf: (float, float, float) => float = "cdf"
@module("jStat") @scope("lognormal") external inv: (float, float, float) => float = "inv"
@module("jStat") @scope("lognormal") external sample: (float, float) => float = "sample"
@module("jStat") @scope("lognormal") external mean: (float, float) => float = "mean"
@module("jstat") @scope("lognormal") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("lognormal") external cdf: (float, float, float) => float = "cdf"
@module("jstat") @scope("lognormal") external inv: (float, float, float) => float = "inv"
@module("jstat") @scope("lognormal") external sample: (float, float) => float = "sample"
@module("jstat") @scope("lognormal") external mean: (float, float) => float = "mean"
}
module Uniform = {
@module("jStat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jStat") @scope("uniform") external inv: (float, float, float) => float = "inv"
@module("jStat") @scope("uniform") external sample: (float, float) => float = "sample"
@module("jStat") @scope("uniform") external mean: (float, float) => float = "mean"
@module("jstat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jstat") @scope("uniform") external inv: (float, float, float) => float = "inv"
@module("jstat") @scope("uniform") external sample: (float, float) => float = "sample"
@module("jstat") @scope("uniform") external mean: (float, float) => float = "mean"
}
type beta
module Beta = {
@module("jStat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jStat") @scope("uniform") external inv: (float, float, float) => float = "inv"
@module("jStat") @scope("uniform") external sample: (float, float) => float = "sample"
@module("jStat") @scope("uniform") external mean: (float, float) => float = "mean"
@module("jstat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jstat") @scope("uniform") external inv: (float, float, float) => float = "inv"
@module("jstat") @scope("uniform") external sample: (float, float) => float = "sample"
@module("jstat") @scope("uniform") external mean: (float, float) => float = "mean"
}
module Exponential = {
@module("jStat") @scope("uniform") external pdf: (float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float) => float = "cdf"
@module("jStat") @scope("uniform") external inv: (float, float) => float = "inv"
@module("jStat") @scope("uniform") external sample: (float) => float = "sample"
@module("jStat") @scope("uniform") external mean: (float) => float = "mean"
@module("jstat") @scope("uniform") external pdf: (float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float) => float = "cdf"
@module("jstat") @scope("uniform") external inv: (float, float) => float = "inv"
@module("jstat") @scope("uniform") external sample: (float) => float = "sample"
@module("jstat") @scope("uniform") external mean: (float) => float = "mean"
}
module Cauchy = {
@module("jStat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jStat") @scope("uniform") external inv: (float, float, float) => float = "inv"
@module("jStat") @scope("uniform") external sample: (float, float) => float = "sample"
@module("jStat") @scope("uniform") external mean: (float, float) => float = "mean"
@module("jstat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jstat") @scope("uniform") external inv: (float, float, float) => float = "inv"
@module("jstat") @scope("uniform") external sample: (float, float) => float = "sample"
@module("jstat") @scope("uniform") external mean: (float, float) => float = "mean"
}
module Triangular = {
@module("jStat") @scope("uniform") external pdf: (float, float, float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float, float, float) => float = "cdf"
@module("jStat") @scope("uniform") external inv: (float, float, float, float) => float = "inv"
@module("jStat") @scope("uniform") external sample: (float, float, float) => float = "sample"
@module("jStat") @scope("uniform") external mean: (float, float, float) => float = "mean"
@module("jstat") @scope("uniform") external pdf: (float, float, float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float, float, float) => float = "cdf"
@module("jstat") @scope("uniform") external inv: (float, float, float, float) => float = "inv"
@module("jstat") @scope("uniform") external sample: (float, float, float) => float = "sample"
@module("jstat") @scope("uniform") external mean: (float, float, float) => float = "mean"
}
module Pareto = {
@module("jStat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jStat") @scope("uniform") external inv: (float, float, float) => float = "inv"
@module("jstat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float, float) => float = "cdf"
@module("jstat") @scope("uniform") external inv: (float, float, float) => float = "inv"
}
module Poisson = {
@module("jStat") @scope("uniform") external pdf: (float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float) => float = "cdf"
@module("jStat") @scope("uniform") external sample: (float) => float = "sample"
@module("jStat") @scope("uniform") external mean: (float) => float = "mean"
@module("jstat") @scope("uniform") external pdf: (float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float) => float = "cdf"
@module("jstat") @scope("uniform") external sample: (float) => float = "sample"
@module("jstat") @scope("uniform") external mean: (float) => float = "mean"
}
module Weibull = {
@module("jStat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float,float ) => float = "cdf"
@module("jStat") @scope("uniform") external sample: (float,float) => float = "sample"
@module("jStat") @scope("uniform") external mean: (float,float) => float = "mean"
@module("jstat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float,float ) => float = "cdf"
@module("jstat") @scope("uniform") external sample: (float,float) => float = "sample"
@module("jstat") @scope("uniform") external mean: (float,float) => float = "mean"
}
module Binomial = {
@module("jStat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jStat") @scope("uniform") external cdf: (float, float,float ) => float = "cdf"
@module("jstat") @scope("uniform") external pdf: (float, float, float) => float = "pdf"
@module("jstat") @scope("uniform") external cdf: (float, float,float ) => float = "cdf"
}
@module("jstat") external sum: array<float> => float = "sum"

16950
packages/squiggle-lang/yarn.lock
View File

File diff suppressed because it is too large Load Diff

1573
packages/website/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

7864
packages/website/yarn.lock
View File

File diff suppressed because it is too large Load Diff

4478
yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff