Merge pull request #501 from quantified-uncertainty/kldivergence-discrete

`klDivergence` on discrete distributions
This commit is contained in:
Ozzie Gooen 2022-05-10 15:35:49 -04:00 committed by GitHub
commit 396bf5bf00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 63 additions and 18 deletions

View File

@ -12,3 +12,13 @@ let triangularDist: DistributionTypes.genericDist = Symbolic(
let exponentialDist: DistributionTypes.genericDist = Symbolic(#Exponential({rate: 2.0}))
let uniformDist: DistributionTypes.genericDist = Symbolic(#Uniform({low: 9.0, high: 10.0}))
let floatDist: DistributionTypes.genericDist = Symbolic(#Float(1e1))
exception KlFailed
exception MixtureFailed
let float1 = 1.0
let float2 = 2.0
let float3 = 3.0
let {mkDelta} = module(TestHelpers)
let point1 = mkDelta(float1)
let point2 = mkDelta(float2)
let point3 = mkDelta(float3)

View File

@ -1,8 +1,9 @@
open Jest
open Expect
open TestHelpers
open GenericDist_Fixtures
describe("kl divergence", () => {
describe("klDivergence: continuous -> continuous -> float", () => {
let klDivergence = DistributionOperation.Constructors.klDivergence(~env)
exception KlFailed
@ -19,7 +20,7 @@ describe("kl divergence", () => {
let analyticalKl = Js.Math.log((highPrediction -. lowPrediction) /. (highAnswer -. lowAnswer))
let kl = E.R.liftJoin2(klDivergence, prediction, answer)
switch kl {
| Ok(kl') => kl'->expect->toBeCloseTo(analyticalKl)
| Ok(kl') => kl'->expect->toBeSoCloseTo(analyticalKl, ~digits=7)
| Error(err) => {
Js.Console.log(DistributionTypes.Error.toString(err))
raise(KlFailed)
@ -51,7 +52,7 @@ describe("kl divergence", () => {
let kl = E.R.liftJoin2(klDivergence, prediction, answer)
switch kl {
| Ok(kl') => kl'->expect->toBeCloseTo(analyticalKl)
| Ok(kl') => kl'->expect->toBeSoCloseTo(analyticalKl, ~digits=3)
| Error(err) => {
Js.Console.log(DistributionTypes.Error.toString(err))
raise(KlFailed)
@ -60,9 +61,44 @@ describe("kl divergence", () => {
})
})
describe("combine along support test", () => {
describe("klDivergence: discrete -> discrete -> float", () => {
let klDivergence = DistributionOperation.Constructors.klDivergence(~env)
let mixture = a => DistributionTypes.DistributionOperation.Mixture(a)
let a' = [(point1, 1e0), (point2, 1e0)]->mixture->run
let b' = [(point1, 1e0), (point2, 1e0), (point3, 1e0)]->mixture->run
let (a, b) = switch (a', b') {
| (Dist(a''), Dist(b'')) => (a'', b'')
| _ => raise(MixtureFailed)
}
test("agrees with analytical answer when finite", () => {
let prediction = b
let answer = a
let kl = klDivergence(prediction, answer)
// Sigma_{i \in 1..2} 0.5 * log(0.5 / 0.33333)
let analyticalKl = Js.Math.log(3.0 /. 2.0)
switch kl {
| Ok(kl') => kl'->expect->toBeSoCloseTo(analyticalKl, ~digits=7)
| Error(err) =>
Js.Console.log(DistributionTypes.Error.toString(err))
raise(KlFailed)
}
})
test("returns infinity when infinite", () => {
let prediction = a
let answer = b
let kl = klDivergence(prediction, answer)
switch kl {
| Ok(kl') => kl'->expect->toEqual(infinity)
| Error(err) =>
Js.Console.log(DistributionTypes.Error.toString(err))
raise(KlFailed)
}
})
})
describe("combineAlongSupportOfSecondArgument0", () => {
// This tests the version of the function that we're NOT using. Haven't deleted the test in case we use the code later.
test("combine along support test", _ => {
test("test on two uniforms", _ => {
let combineAlongSupportOfSecondArgument = XYShape.PointwiseCombination.combineAlongSupportOfSecondArgument0
let lowAnswer = 0.0
let highAnswer = 1.0
@ -97,6 +133,7 @@ describe("combine along support test", () => {
2.0 *. MagicNumbers.Epsilon.ten,
1.0 -. MagicNumbers.Epsilon.ten,
1.0,
1.0 +. MagicNumbers.Epsilon.ten,
],
ys: [
-0.34657359027997264,
@ -104,6 +141,7 @@ describe("combine along support test", () => {
-0.34657359027997264,
-0.34657359027997264,
-0.34657359027997264,
infinity,
],
}),
),

View File

@ -51,6 +51,7 @@ let mkExponential = rate => DistributionTypes.Symbolic(#Exponential({rate: rate}
let mkUniform = (low, high) => DistributionTypes.Symbolic(#Uniform({low: low, high: high}))
let mkCauchy = (local, scale) => DistributionTypes.Symbolic(#Cauchy({local: local, scale: scale}))
let mkLognormal = (mu, sigma) => DistributionTypes.Symbolic(#Lognormal({mu: mu, sigma: sigma}))
let mkDelta = x => DistributionTypes.Symbolic(#Float(x))
let normalMake = SymbolicDist.Normal.make
let betaMake = SymbolicDist.Beta.make

View File

@ -48,12 +48,7 @@ let combinePointwise = (
// TODO: does it ever make sense to pointwise combine the integrals here?
// It could be done for pointwise additions, but is that ever needed?
make(
combiner(fn, XYShape.XtoY.discreteInterpolator, t1.xyShape, t2.xyShape)->E.R.toExn(
"Addition operation should never fail",
_,
),
)->Ok
combiner(fn, XYShape.XtoY.discreteInterpolator, t1.xyShape, t2.xyShape)->E.R2.fmap(make)
}
let reduce = (
@ -163,7 +158,6 @@ module T = Dist({
}
let integralEndY = (t: t) => t.integralSumCache |> E.O.default(t |> integral |> Continuous.lastY)
let integralEndYResult = (t: t) => t->integralEndY->Ok
let minX = shapeFn(XYShape.T.minX)
let maxX = shapeFn(XYShape.T.maxX)
let toDiscreteProbabilityMassFraction = _ => 1.0
@ -230,10 +224,9 @@ module T = Dist({
let klDivergence = (prediction: t, answer: t) => {
combinePointwise(
~combiner=XYShape.PointwiseCombination.combineAlongSupportOfSecondArgument0,
~fn=PointSetDist_Scoring.KLDivergence.integrand,
prediction,
answer,
) |> E.R2.bind(integralEndYResult)
)->E.R2.fmap(integralEndY)
}
})

View File

@ -302,9 +302,10 @@ module T = Dist({
}
let klDivergence = (prediction: t, answer: t) => {
combinePointwise(PointSetDist_Scoring.KLDivergence.integrand, prediction, answer) |> E.R.fmap(
integralEndY,
)
Error(Operation.NotYetImplemented)
// combinePointwise(PointSetDist_Scoring.KLDivergence.integrand, prediction, answer) |> E.R.fmap(
// integralEndY,
// )
}
})

View File

@ -199,6 +199,7 @@ module T = Dist({
let klDivergence = (t1: t, t2: t) =>
switch (t1, t2) {
| (Continuous(t1), Continuous(t2)) => Continuous.T.klDivergence(t1, t2)
| (Discrete(t1), Discrete(t2)) => Discrete.T.klDivergence(t1, t2)
| _ => Error(NotYetImplemented)
}
})

View File

@ -4,10 +4,11 @@ module KLDivergence = {
float,
Operation.Error.t,
> =>
// We decided that negative infinity, not an error at answerElement = 0.0, is a desirable value.
if answerElement == 0.0 {
Ok(0.0)
} else if predictionElement == 0.0 {
Error(Operation.NegativeInfinityError)
Ok(infinity)
} else {
let quot = predictionElement /. answerElement
quot < 0.0 ? Error(Operation.ComplexNumberError) : Ok(-.answerElement *. logFn(quot))