quantified.uncertainty/squiggle
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed tests after pair; error'd out mixed case

Browse Source
This commit is contained in:
Quinn Dougherty 2022-06-21 12:23:58 -04:00
parent 95adc67701
commit d80ea676c5
4 changed files with 121 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
packages/squiggle-lang/__tests__/Distributions/GenericDist_Fixtures.res
View File

@ -19,7 +19,6 @@ exception MixtureFailed
let float1 = 1.0
let float2 = 2.0
let float3 = 3.0
let {mkDelta} = module(TestHelpers)
let point1 = mkDelta(float1)
let point2 = mkDelta(float2)
let point3 = mkDelta(float3)
let point1 = TestHelpers.mkDelta(float1)
let point2 = TestHelpers.mkDelta(float2)
let point3 = TestHelpers.mkDelta(float3)

90
packages/squiggle-lang/__tests__/Distributions/Scoring/TwoScalars_test.res Normal file
View File

@ -0,0 +1,90 @@
open Jest
open Expect
open GenericDist_Fixtures
exception ScoreFailed
describe("TwoScalars: scalar -> scalar -> score", () => {
test("score: infinity", () => {
let scalar1 = 1.0 // 100% of probability mass 1.0
let scalar2 = 2.0 // 100% of probability mass to 2.0
let score = PointSetDist_Scoring.TwoScalars.score(~estimate=scalar1, ~answer=scalar2)
switch score {
| Ok(x) => x->expect->toEqual(infinity)
| _ => raise(MixtureFailed)
}
})
test("score: 0.0", () => {
let scalar1 = 1.5 // 100% of probability mass 1.5
let scalar2 = 1.5 // 100% of probability mass to 1.5
let score = PointSetDist_Scoring.TwoScalars.score(~estimate=scalar1, ~answer=scalar2)
switch score {
| Ok(x) => x->expect->toEqual(0.0)
| _ => raise(MixtureFailed)
}
})
test("scoreWithPrior: minus infinity", () => {
let scalar1 = 1.5 // 100% of probability mass 1.5
let scalar2 = 1.5 // 100% of probability mass to 1.5
let scalar3 = 1.0 // 100% of probability mass to 1.0
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual(-.infinity)
| _ => raise(MixtureFailed)
}
})
test("scoreWithPrior: 0.0", () => {
let scalar1 = 1.5 // 100% of probability mass 1.5
let scalar2 = 1.5 // 100% of probability mass to 1.5
let scalar3 = 1.5 // 100% of probability mass to 1.5
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual(0.0)
| _ => raise(ScoreFailed)
}
})
test("scoreWithPrior: really dumb forecasters", () => {
let scalar1 = 1.0 // 100% of probability mass 1.0
let scalar2 = 1.5 // 100% of probability mass to 1.5
let scalar3 = 1.0 // 100% of probability mass to 1.0
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual(infinity -. infinity) // "Error: Really dumb forecasters"
| _ => raise(ScoreFailed)
}
})
test("scoreWithPrior: 0.0", () => {
let scalar1 = 1.0 // 100% of probability mass 1.0
let scalar2 = 1.0 // 100% of probability mass to 1.0
let scalar3 = 1.0 // 100% of probability mass to 1.0
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual(0.0)
| _ => raise(ScoreFailed)
}
})
})

121
packages/squiggle-lang/__tests__/Distributions/Scoring/WithScalarAnswer_test.res
View File

@ -1,23 +1,20 @@
// Bring up a discrete distribution
open Jest
open Expect
open TestHelpers
open GenericDist_Fixtures
exception ScoreFailed
// WithDistAnswer -> in the KL divergence test file.
// WithScalarAnswer
describe("WithScalarAnswer: discrete -> discrete -> float", () => {
describe("WithScalarAnswer: discrete -> scalar -> score", () => {
let mixture = a => DistributionTypes.DistributionOperation.Mixture(a)
let pointA = mkDelta(3.0)
let pointB = mkDelta(2.0)
let pointC = mkDelta(1.0)
let pointD = mkDelta(0.0)
test("WithScalarAnswer.score: agrees with analytical answer when finite", () => {
test("score: agrees with analytical answer when finite", () => {
let prediction' = [(pointA, 0.25), (pointB, 0.25), (pointC, 0.25), (pointD, 0.25)]->mixture->run
let prediction = switch prediction' {
| Dist(PointSet(a'')) => a''
| Dist(PointSet(p)) => p
| _ => raise(MixtureFailed)
}
@ -25,35 +22,35 @@ describe("WithScalarAnswer: discrete -> discrete -> float", () => {
let result = PointSetDist_Scoring.WithScalarAnswer.score(~estimate=prediction, ~answer)
switch result {
| Ok(x) => x->expect->toEqual(-.Js.Math.log(0.25 /. 1.0))
| _ => raise(MixtureFailed)
| _ => raise(ScoreFailed)
}
})
test("WithScalarAnswer.score: agrees with analytical answer when finite", () => {
test("score: agrees with analytical answer when finite", () => {
let prediction' = [(pointA, 0.75), (pointB, 0.25)]->mixture->run
let prediction = switch prediction' {
| Dist(PointSet(a'')) => a''
| Dist(PointSet(p)) => p
| _ => raise(MixtureFailed)
}
let answer = 3.0 // So this is: assigning 100% probability to 2.0
let result = PointSetDist_Scoring.WithScalarAnswer.score(~estimate=prediction, ~answer)
switch result {
| Ok(x) => x->expect->toEqual(-.Js.Math.log(0.75 /. 1.0))
| _ => raise(MixtureFailed)
| _ => raise(ScoreFailed)
}
})
test("WithScalarAnswer.scoreWithPrior: ", () => {
test("scoreWithPrior: agrees with analytical answer when finite", () => {
let prior' = [(pointA, 0.5), (pointB, 0.5)]->mixture->run
let prediction' = [(pointA, 0.75), (pointB, 0.25)]->mixture->run
let prediction = switch prediction' {
| Dist(PointSet(a'')) => a''
| Dist(PointSet(p)) => p
| _ => raise(MixtureFailed)
}
let prior = switch prior' {
| Dist(PointSet(a'')) => a''
| Dist(PointSet(p)) => p
| _ => raise(MixtureFailed)
}
@ -65,101 +62,7 @@ describe("WithScalarAnswer: discrete -> discrete -> float", () => {
)
switch result {
| Ok(x) => x->expect->toEqual(-.Js.Math.log(0.75 /. 1.0) -. -.Js.Math.log(0.5 /. 1.0))
| _ => raise(MixtureFailed)
| _ => raise(ScoreFailed)
}
})
})
describe("TwoScalars: float -> float -> float", () => {
test("TwoScalars.score: ", () => {
let scalar1 = 1.0 // 100% of probability mass 1.0
let scalar2 = 2.0 // 100% of probability mass to 2.0
let score = PointSetDist_Scoring.TwoScalars.score(~estimate=scalar1, ~answer=scalar2)
switch score {
| Ok(x) => x->expect->toEqual(infinity)
| _ => raise(MixtureFailed)
}
})
test("TwoScalars.score: ", () => {
let scalar1 = 1.5 // 100% of probability mass 1.0
let scalar2 = 1.5 // 100% of probability mass to 2.0
let score = PointSetDist_Scoring.TwoScalars.score(~estimate=scalar1, ~answer=scalar2)
switch score {
| Ok(x) => x->expect->toEqual(0.0)
| _ => raise(MixtureFailed)
}
})
test("TwoScalars.scoreWithPrior: ", () => {
let scalar1 = 1.5 // 100% of probability mass 1.0
let scalar2 = 1.5 // 100% of probability mass to 2.0
let scalar3 = 1.0 // 100% of probability mass to 2.0
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual(-.infinity)
| _ => raise(MixtureFailed)
}
})
test("TwoScalars.scoreWithPrior: ", () => {
let scalar1 = 1.5 // 100% of probability mass 1.0
let scalar2 = 1.5 // 100% of probability mass to 2.0
let scalar3 = 1.5 // 100% of probability mass to 2.0
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual(0.0)
| _ => raise(MixtureFailed)
}
})
test("TwoScalars.scoreWithPrior: ", () => {
let scalar1 = 1.0 // 100% of probability mass 1.0
let scalar2 = 1.5 // 100% of probability mass to 2.0
let scalar3 = 1.0 // 100% of probability mass to 2.0
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual("Error: Really dumb forecasters") // unclear what this case should give; could be smth else, or undefined
| _ => raise(MixtureFailed)
}
})
test("TwoScalars.scoreWithPrior: ", () => {
let scalar1 = 1.0 // 100% of probability mass 1.0
let scalar2 = 1.0 // 100% of probability mass to 2.0
let scalar3 = 1.0 // 100% of probability mass to 2.0
let score = PointSetDist_Scoring.TwoScalars.scoreWithPrior(
~estimate=scalar1,
~answer=scalar2,
~prior=scalar3,
)
switch score {
| Ok(x) => x->expect->toEqual(0.0)
| _ => raise(MixtureFailed)
}
})
})
/*
describe("WithScalarAnswer: discrete -> discrete -> float", () => {
})
// TwoScalars
describe("WithScalarAnswer: discrete -> discrete -> float", () => {
})
*/

81
packages/squiggle-lang/src/rescript/Distributions/PointSetDist/PointSetDist_Scoring.res
View File

@ -93,33 +93,32 @@ module WithDistAnswer = {
module WithScalarAnswer = {
let sum = (mp: PointSetTypes.MixedPoint.t): float => mp.continuous +. mp.discrete
let score = (~estimate: pointSetDist, ~answer: scalar): result<score, Operation.Error.t> => {
let _score = (~estimatePdf: float => float, ~answer: float): result<
let _score = (~estimatePdf: float => option<float>, ~answer: float): result<
score,
Operation.Error.t,
> => {
let density = answer->estimatePdf
if density < 0.0 {
switch density {
| None => Operation.PdfInvalidError->Error
| Some(density') =>
if density' < 0.0 {
Operation.PdfInvalidError->Error
} else if density == 0.0 {
} else if density' == 0.0 {
infinity->Ok
} else {
density->logFn->(x => -.x)->Ok
density'->logFn->(x => -.x)->Ok
}
}
}
let estimatePdf = x =>
switch estimate {
| Continuous(esti) => Continuous.T.xToY(x, esti)->sum
| Discrete(esti) => Discrete.T.xToY(x, esti)->sum
| Mixed(esti) => Mixed.T.xToY(x, esti)->sum
| Continuous(esti) => Continuous.T.xToY(x, esti)->sum->Some
| Discrete(esti) => Discrete.T.xToY(x, esti)->sum->Some
| Mixed(_) => None
}
_score(~estimatePdf, ~answer)
}
/*
let score1 = (~estimate: pointSetDist, ~answer: scalar): result<score, Operation.Error.t> => {
let probabilityAssignedToAnswer = Ok(1.0)
}
*/
let scoreWithPrior = (~estimate: pointSetDist, ~answer: scalar, ~prior: pointSetDist): result<
score,
@ -128,47 +127,13 @@ module WithScalarAnswer = {
E.R.merge(score(~estimate, ~answer), score(~estimate=prior, ~answer))->E.R2.fmap(((s1, s2)) =>
s1 -. s2
)
/*
let _scoreWithPrior = (
~estimatePdf: float => float,
~answer: scalar,
~priorPdf: float => float,
): result<score, Operation.Error.t> => {
let numerator = answer->estimatePdf
let priorDensityOfAnswer = answer->priorPdf
if numerator < 0.0 || priorDensityOfAnswer < 0.0 {
Operation.PdfInvalidError->Error
} else if numerator == 0.0 || priorDensityOfAnswer == 0.0 {
infinity->Ok
} else {
//
}
}
let estimatePdf = x =>
switch estimate {
| Continuous(esti) => Continuous.T.xToY(x, esti)->sum
| Discrete(esti) => Discrete.T.xToY(x, esti)->sum
| Mixed(esti) => Mixed.T.xToY(x, esti)->sum
}
let priorPdf = x =>
switch prior {
| Continuous(prio) => Continuous.T.xToY(x, prio)->sum
| Discrete(prio) => Discrete.T.xToY(x, prio)->sum
| Mixed(prio) => Mixed.T.xToY(x, prio)->sum
}
_scoreWithPrior(~estimatePdf, ~answer, ~priorPdf)
*/
}
}
// For mixed discrete answer
// (prediction, answer) => sum(answer.map(a => a.probability * WithScalarAnswer.score(prediction, a.value)))
module TwoScalars = {
// You will almost never want to use this.
let score = (~estimate: scalar, ~answer: scalar) => {
if estimate == answer {
if Js.Math.abs_float(estimate -. answer) < MagicNumbers.Epsilon.ten {
0.0->Ok
} else {
infinity->Ok // - log(0)
@ -178,28 +143,8 @@ module TwoScalars = {
let scoreWithPrior = (~estimate: scalar, ~answer: scalar, ~prior: scalar) => {
E.R.merge(score(~estimate, ~answer), score(~estimate=prior, ~answer))->E.R2.fmap(((s1, s2)) =>
s1 -. s2
)
// unclear what this should give if both are wrong: infinity-infinity. Maybe some warning??
) // This will presently NaN if both are wrong: infinity-infinity.
}
/*
let score = (~estimate: scalar, ~answer: scalar) =>
if answer == 0.0 {
0.0->Ok
} else if estimate == 0.0 {
infinity->Ok
} else {
minusScaledLogOfQuotient(~esti=estimate, ~answ=answer)
}
let scoreWithPrior = (~estimate: scalar, ~answer: scalar, ~prior: scalar) =>
if answer == 0.0 {
0.0->Ok
} else if estimate == 0.0 || prior == 0.0 {
infinity->Ok
} else {
minusScaledLogOfQuotient(~esti=estimate /. prior, ~answ=answer)
}
*/
}
let twoGenericDistsToTwoPointSetDists = (~toPointSetFn, estimate, answer): result<