Merge pull request #386 from quantified-uncertainty/loss-testing

Loss testing
2022-04-27 12:04:48 -04:00 · 2022-04-27 12:04:48 -04:00 · 8c07b33102
commit 8c07b33102
parent ff9f8f5ef2 b5f88e5ec9
6 changed files with 230 additions and 5 deletions
--- a/packages/squiggle-lang/benchmark/conversion_tests.ts
+++ b/packages/squiggle-lang/benchmark/conversion_tests.ts
@ -0,0 +1,46 @@
+import { distributions, generateInt, generateFloatRange } from "./generators";
+import { test, expectEqual } from "./lib";
+
+let checkDistributionSame = (
+  distribution: string,
+  operation: (arg: string) => string
+): void => {
+  expectEqual(
+    operation(distribution),
+    operation(`toPointSet(${distribution})`)
+  );
+  expectEqual(
+    operation(distribution),
+    operation(`toSampleSet(${distribution})`)
+  );
+};
+
+Object.entries(distributions).map(([key, generator]) => {
+  let distribution = generator();
+  test(`mean is the same for ${key} distribution under all distribution types`, () =>
+    checkDistributionSame(distribution, (d: string) => `mean(${d})`));
+
+  test(`cdf is the same for ${key} distribution under all distribution types`, () => {
+    let cdf_value = generateInt();
+    checkDistributionSame(
+      distribution,
+      (d: string) => `cdf(${d}, ${cdf_value})`
+    );
+  });
+
+  test(`pdf is the same for ${key} distribution under all distribution types`, () => {
+    let pdf_value = generateInt();
+    checkDistributionSame(
+      distribution,
+      (d: string) => `pdf(${d}, ${pdf_value})`
+    );
+  });
+
+  test(`inv is the same for ${key} distribution under all distribution types`, () => {
+    let inv_value = generateFloatRange(0, 1);
+    checkDistributionSame(
+      distribution,
+      (d: string) => `inv(${d}, ${inv_value})`
+    );
+  });
+});
--- a/packages/squiggle-lang/benchmark/generators.ts
+++ b/packages/squiggle-lang/benchmark/generators.ts
@ -0,0 +1,48 @@
+export let generateFloatRange = (min: number, max: number): number =>
+  Math.random() * (max - min) + min;
+export let generateIntRange = (min: number, max: number): number =>
+  Math.floor(generateFloatRange(min, max));
+
+export let generateIntMin = (min: number): number => generateIntRange(min, 100);
+
+export let generateInt = (): number => generateIntMin(-100);
+
+let generatePositiveInt = (): number => generateIntMin(1);
+
+export let generateNormal = (): string =>
+  `normal(${generateInt()}, ${generatePositiveInt()})`;
+
+export let generateBeta = (): string =>
+  `beta(${generatePositiveInt()}, ${generatePositiveInt()})`;
+
+export let generateLognormal = (): string =>
+  `lognormal(${generateInt()}, ${generatePositiveInt()})`;
+
+export let generateExponential = (): string =>
+  `exponential(${generatePositiveInt()})`;
+
+export let generateUniform = (): string => {
+  let a = generateInt();
+  let b = generateIntMin(a + 1);
+  return `uniform(${a}, ${b})`;
+};
+export let generateCauchy = (): string => {
+  return `cauchy(${generateInt()}, ${generatePositiveInt()})`;
+};
+
+export let generateTriangular = (): string => {
+  let a = generateInt();
+  let b = generateIntMin(a + 1);
+  let c = generateIntMin(b + 1);
+  return `triangular(${a}, ${b}, ${c})`;
+};
+
+export let distributions: { [key: string]: () => string } = {
+  normal: generateNormal,
+  beta: generateBeta,
+  lognormal: generateLognormal,
+  exponential: generateExponential,
+  triangular: generateTriangular,
+  cauchy: generateCauchy,
+  uniform: generateUniform,
+};
--- a/packages/squiggle-lang/benchmark/lib.ts
+++ b/packages/squiggle-lang/benchmark/lib.ts
@ -0,0 +1,42 @@
+import { run, squiggleExpression, errorValueToString } from "../src/js/index";
+import * as chalk from "chalk";
+
+let testRun = (x: string): squiggleExpression => {
+  let result = run(x, { sampleCount: 100, xyPointLength: 100 });
+  if (result.tag === "Ok") {
+    return result.value;
+  } else {
+    throw Error(
+      "Expected squiggle expression to evaluate but got error: " +
+        errorValueToString(result.value)
+    );
+  }
+};
+
+export function test(name: string, fn: () => void) {
+  console.log(chalk.cyan.bold(name));
+  fn();
+}
+
+export function expectEqual(expression1: string, expression2: string) {
+  let result1 = testRun(expression1);
+  let result2 = testRun(expression2);
+  if (result1.tag === "number" && result2.tag === "number") {
+    let logloss = Math.log(Math.abs(result1.value - result2.value));
+    let isBadLogless = logloss > 1;
+    console.log(chalk.blue(`${expression1} = ${expression2}`));
+    console.log(`${result1.value} = ${result2.value}`);
+    console.log(
+      `logloss = ${
+        isBadLogless
+          ? chalk.red(logloss.toFixed(2))
+          : chalk.green(logloss.toFixed(2))
+      }`
+    );
+    console.log();
+  } else {
+    throw Error(
+      `Expected both to be number, but got ${result1.tag} and ${result2.tag}`
+    );
+  }
+}
--- a/packages/squiggle-lang/package.json
+++ b/packages/squiggle-lang/package.json
@ -9,6 +9,7 @@
    "start": "rescript build -w -with-deps",
    "clean": "rescript clean",
    "test:reducer": "jest __tests__/Reducer*/",
+    "benchmark": "ts-node benchmark/conversion_tests.ts",
    "test": "jest",
    "test:ts": "jest __tests__/TS/",
    "test:rescript": "jest --modulePathIgnorePatterns=__tests__/TS/*",
@ -43,15 +44,24 @@
    "@istanbuljs/nyc-config-typescript": "^1.0.2",
    "@types/jest": "^27.4.0",
    "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2",
+    "bisect_ppx": "^2.7.1",
+    "chalk": "^4.1.2",
    "codecov": "3.8.3",
    "fast-check": "2.25.0",
    "gentype": "^4.3.0",
    "jest": "^27.5.1",
+    "jstat": "^1.9.5",
+    "lodash": "4.17.21",
+    "mathjs": "10.5.0",
    "moduleserve": "0.9.1",
-    "reanalyze": "^2.19.0",
    "nyc": "^15.1.0",
+    "pdfast": "^0.2.0",
+    "rationale": "0.2.0",
+    "reanalyze": "^2.19.0",
+    "rescript": "^9.1.4",
    "ts-jest": "^27.1.4",
-    "ts-loader": "^9.2.9",
+    "ts-loader": "^9.2.8",
+    "ts-node": "^10.7.0",
    "typescript": "^4.6.3",
    "webpack": "^5.72.0",
    "webpack-cli": "^4.9.2"
--- a/packages/squiggle-lang/src/rescript/TypescriptInterface.res
+++ b/packages/squiggle-lang/src/rescript/TypescriptInterface.res
@ -31,6 +31,9 @@ let makeSampleSetDist = SampleSetDist.make
@genType
 let evaluate = Reducer.evaluate

+@genType
+let evaluateUsingExternalBindings = Reducer.evaluateUsingExternalBindings
+
@genType
 type expressionValue = ReducerInterface_ExpressionValue.expressionValue

--- a/yarn.lock
+++ b/yarn.lock
@ -1334,6 +1334,18 @@
  resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9"
  integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==

+"@cspotcode/source-map-consumer@0.8.0":
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"
+  integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg==
+
+"@cspotcode/source-map-support@0.7.0":
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.7.0.tgz#4789840aa859e46d2f3173727ab707c66bf344f5"
+  integrity sha512-X4xqRHqN8ACt2aHVe51OxeA2HjbcL4MqFqXkrmQszJ1NOUuUu5u6Vqx/0lZSVNku7velL5FC/s5uEAj1lsBMhA==
+  dependencies:
+    "@cspotcode/source-map-consumer" "0.8.0"
+
 "@csstools/normalize.css@*":
  version "12.0.0"
  resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-12.0.0.tgz#a9583a75c3f150667771f30b60d9f059473e62c4"
@ -3694,6 +3706,26 @@
  resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad"
  integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==

+"@tsconfig/node10@^1.0.7":
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.8.tgz#c1e4e80d6f964fbecb3359c43bd48b40f7cadad9"
+  integrity sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg==
+
+"@tsconfig/node12@^1.0.7":
+  version "1.0.9"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.9.tgz#62c1f6dee2ebd9aead80dc3afa56810e58e1a04c"
+  integrity sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw==
+
+"@tsconfig/node14@^1.0.0":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2"
+  integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==
+
+"@tsconfig/node16@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e"
+  integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==
+
 "@types/aria-query@^4.2.0":
  version "4.2.2"
  resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-4.2.2.tgz#ed4e0ad92306a704f9fb132a0cfcf77486dbe2bc"
@ -4677,7 +4709,7 @@ acorn-walk@^7.0.0, acorn-walk@^7.1.1, acorn-walk@^7.2.0:
  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
  integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==

-acorn-walk@^8.0.0:
+acorn-walk@^8.0.0, acorn-walk@^8.1.1:
  version "8.2.0"
  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1"
  integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==
@ -4995,6 +5027,11 @@ are-we-there-yet@^2.0.0:
    delegates "^1.0.0"
    readable-stream "^3.6.0"

+arg@^4.1.0:
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
+  integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
+
 arg@^5.0.0, arg@^5.0.1:
  version "5.0.1"
  resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.1.tgz#eb0c9a8f77786cad2af8ff2b862899842d7b6adb"
@ -6790,6 +6827,11 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
    safe-buffer "^5.0.1"
    sha.js "^2.4.8"

+create-require@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
+  integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==
+
 cross-env@^7.0.3:
  version "7.0.3"
  resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf"
@ -7531,6 +7573,11 @@ diff-sequences@^27.5.1:
  resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327"
  integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ==

+diff@^4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
+  integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
+
 diffie-hellman@^5.0.0:
  version "5.0.3"
  resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875"
@ -11519,7 +11566,7 @@ make-dir@^3.0.0, make-dir@^3.0.2, make-dir@^3.1.0:
  dependencies:
    semver "^6.0.0"

-make-error@1.x:
+make-error@1.x, make-error@^1.1.1:
  version "1.3.6"
  resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
  integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==
@ -16608,7 +16655,7 @@ ts-jest@^27.1.4:
    semver "7.x"
    yargs-parser "20.x"

-ts-loader@^9.2.9:
+ts-loader@^9.2.8, ts-loader@^9.2.9:
  version "9.2.9"
  resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.2.9.tgz#0653e07fa1b4f225d0ca57a84fddbfd43d930f9e"
  integrity sha512-b0+vUY2/enb0qYtDQuNlDnJ9900NTiPiJcDJ6sY7ax1CCCwXfYIqPOMm/BwW7jsF1km+Oz8W9s31HLuD+FLIMg==
@ -16618,6 +16665,25 @@ ts-loader@^9.2.9:
    micromatch "^4.0.0"
    semver "^7.3.4"

+ts-node@^10.7.0:
+  version "10.7.0"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.7.0.tgz#35d503d0fab3e2baa672a0e94f4b40653c2463f5"
+  integrity sha512-TbIGS4xgJoX2i3do417KSaep1uRAW/Lu+WAL2doDHC0D6ummjirVOXU5/7aiZotbQ5p1Zp9tP7U6cYhA0O7M8A==
+  dependencies:
+    "@cspotcode/source-map-support" "0.7.0"
+    "@tsconfig/node10" "^1.0.7"
+    "@tsconfig/node12" "^1.0.7"
+    "@tsconfig/node14" "^1.0.0"
+    "@tsconfig/node16" "^1.0.2"
+    acorn "^8.4.1"
+    acorn-walk "^8.1.1"
+    arg "^4.1.0"
+    create-require "^1.1.0"
+    diff "^4.0.1"
+    make-error "^1.1.1"
+    v8-compile-cache-lib "^3.0.0"
+    yn "3.1.1"
+
 ts-pnp@^1.1.6:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.2.0.tgz#a500ad084b0798f1c3071af391e65912c86bca92"
@ -17115,6 +17181,11 @@ uuid@^8.0.0, uuid@^8.3.2:
  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==

+v8-compile-cache-lib@^3.0.0:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
+  integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
+
 v8-compile-cache@^2.0.3:
  version "2.3.0"
  resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee"
@ -18353,6 +18424,11 @@ yargs@~17.2.1:
    y18n "^5.0.5"
    yargs-parser "^20.2.2"

+yn@3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"
+  integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==
+
 yocto-queue@^0.1.0:
  version "0.1.0"
  resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"