From 60f3445277b59d72a5a96a47c66ea20aa676b814 Mon Sep 17 00:00:00 2001 From: Sam Nolan Date: Tue, 26 Apr 2022 14:26:02 -0400 Subject: [PATCH] Initial loss testing proof of concept --- .../benchmark/conversion_tests.ts | 9 ++ .../squiggle-lang/benchmark/generators.ts | 12 +++ packages/squiggle-lang/benchmark/lib.ts | 31 +++++++ packages/squiggle-lang/package.json | 17 ++-- packages/squiggle-lang/tsconfig.json | 1 + yarn.lock | 91 ++++++++++++++++++- 6 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 packages/squiggle-lang/benchmark/conversion_tests.ts create mode 100644 packages/squiggle-lang/benchmark/generators.ts create mode 100644 packages/squiggle-lang/benchmark/lib.ts diff --git a/packages/squiggle-lang/benchmark/conversion_tests.ts b/packages/squiggle-lang/benchmark/conversion_tests.ts new file mode 100644 index 00000000..52804c0b --- /dev/null +++ b/packages/squiggle-lang/benchmark/conversion_tests.ts @@ -0,0 +1,9 @@ +import { generateNormal } from "./generators"; +import { test, expectEqual } from "./lib"; + +test("mean is the same under point set transform", () => { + let normal = generateNormal(); + let symbolicMean = `mean(${normal})`; + let pointSetMean = `mean(toPointSet(${normal}))`; + return expectEqual(symbolicMean, pointSetMean); +}); diff --git a/packages/squiggle-lang/benchmark/generators.ts b/packages/squiggle-lang/benchmark/generators.ts new file mode 100644 index 00000000..6d5d7854 --- /dev/null +++ b/packages/squiggle-lang/benchmark/generators.ts @@ -0,0 +1,12 @@ +let generateFloat = (): number => Math.random() * 200 - 100; + +let generatePositive = (): number => Math.random() * 100; + +export let generateNormal = (): string => + `normal(${generateFloat()}, ${generatePositive()})`; + +export let generateBeta = (): string => + `beta(${generatePositive()}, ${generatePositive()})`; + +export let generateLognormal = (): string => + `lognormal(${generateFloat()}, ${generatePositive()})`; diff --git a/packages/squiggle-lang/benchmark/lib.ts b/packages/squiggle-lang/benchmark/lib.ts new file mode 100644 index 00000000..382d2d54 --- /dev/null +++ b/packages/squiggle-lang/benchmark/lib.ts @@ -0,0 +1,31 @@ +import { run, squiggleExpression, errorValueToString } from "../src/js/index"; +import _ from "lodash"; + +let testRun = (x: string): squiggleExpression => { + let result = run(x, { sampleCount: 100, xyPointLength: 100 }); + if (result.tag === "Ok") { + return result.value; + } else { + throw Error( + "Expected squiggle expression to evaluate but got error: " + + errorValueToString(result.value) + ); + } +}; + +export function test(name: string, fn: () => void) { + console.log(name); + fn(); +} + +export function expectEqual(expression1: string, expression2: string) { + let result1 = testRun(expression1); + let result2 = testRun(expression2); + if (result1.tag === "number" && result2.tag === "number") { + let loss = getLoss(result1.value, result2.value); + console.log(`${result1.value} === ${result2.value}\nLoss: ${loss}`); + } +} + +let getLoss = (actual: number, expected: number): number => + Math.abs(expected - actual); diff --git a/packages/squiggle-lang/package.json b/packages/squiggle-lang/package.json index bf31a059..e9e77766 100644 --- a/packages/squiggle-lang/package.json +++ b/packages/squiggle-lang/package.json @@ -30,26 +30,27 @@ "author": "Quantified Uncertainty Research Institute", "license": "MIT", "devDependencies": { - "bisect_ppx": "^2.7.1", - "jstat": "^1.9.5", - "lodash": "4.17.21", - "mathjs": "10.5.0", - "pdfast": "^0.2.0", - "rationale": "0.2.0", - "rescript": "^9.1.4", "@glennsl/rescript-jest": "^0.9.0", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@types/jest": "^27.4.0", "babel-plugin-transform-es2015-modules-commonjs": "^6.26.2", + "bisect_ppx": "^2.7.1", "codecov": "3.8.3", "fast-check": "2.25.0", "gentype": "^4.3.0", "jest": "^27.5.1", + "jstat": "^1.9.5", + "lodash": "4.17.21", + "mathjs": "10.5.0", "moduleserve": "0.9.1", - "reanalyze": "^2.19.0", "nyc": "^15.1.0", + "pdfast": "^0.2.0", + "rationale": "0.2.0", + "reanalyze": "^2.19.0", + "rescript": "^9.1.4", "ts-jest": "^27.1.4", "ts-loader": "^9.2.8", + "ts-node": "^10.7.0", "typescript": "^4.6.3", "webpack": "^5.72.0", "webpack-cli": "^4.9.2" diff --git a/packages/squiggle-lang/tsconfig.json b/packages/squiggle-lang/tsconfig.json index 7a610d10..f1f9a00d 100644 --- a/packages/squiggle-lang/tsconfig.json +++ b/packages/squiggle-lang/tsconfig.json @@ -4,6 +4,7 @@ "jsx": "react", "allowJs": true, "noImplicitAny": true, + "esModuleInterop": true, "removeComments": true, "preserveConstEnums": true, "sourceMap": true, diff --git a/yarn.lock b/yarn.lock index 90b85189..b82019b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1334,6 +1334,18 @@ resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== +"@cspotcode/source-map-consumer@0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b" + integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg== + +"@cspotcode/source-map-support@0.7.0": + version "0.7.0" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.7.0.tgz#4789840aa859e46d2f3173727ab707c66bf344f5" + integrity sha512-X4xqRHqN8ACt2aHVe51OxeA2HjbcL4MqFqXkrmQszJ1NOUuUu5u6Vqx/0lZSVNku7velL5FC/s5uEAj1lsBMhA== + dependencies: + "@cspotcode/source-map-consumer" "0.8.0" + "@csstools/normalize.css@*": version "12.0.0" resolved "https://registry.yarnpkg.com/@csstools/normalize.css/-/normalize.css-12.0.0.tgz#a9583a75c3f150667771f30b60d9f059473e62c4" @@ -3694,6 +3706,26 @@ resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== +"@tsconfig/node10@^1.0.7": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.8.tgz#c1e4e80d6f964fbecb3359c43bd48b40f7cadad9" + integrity sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg== + +"@tsconfig/node12@^1.0.7": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.9.tgz#62c1f6dee2ebd9aead80dc3afa56810e58e1a04c" + integrity sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw== + +"@tsconfig/node14@^1.0.0": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2" + integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg== + +"@tsconfig/node16@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e" + integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA== + "@types/aria-query@^4.2.0": version "4.2.2" resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-4.2.2.tgz#ed4e0ad92306a704f9fb132a0cfcf77486dbe2bc" @@ -4088,7 +4120,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^16.9.19", "@types/react@^18.0.1", "@types/react@^18.0.3": +"@types/react@*", "@types/react@^18.0.3": version "18.0.7" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.0.7.tgz#8437a226763adf854969954dfe582529a406cbad" integrity sha512-CXSXHzTexlX9esf4ReIUJeaemKcmBEvYzxHDUk19c3BCcEGUvUjkeC3jkscPSfSaQ6SPDRNd/zMxi8oc/P1zxA== @@ -4097,6 +4129,15 @@ "@types/scheduler" "*" csstype "^3.0.2" +"@types/react@^16.9.19": + version "16.14.25" + resolved "https://registry.yarnpkg.com/@types/react/-/react-16.14.25.tgz#d003f712c7563fdef5a87327f1892825af375608" + integrity sha512-cXRVHd7vBT5v1is72mmvmsg9stZrbJO04DJqFeh3Yj2tVKO6vmxg5BI+ybI6Ls7ROXRG3aFbZj9x0WA3ZAoDQw== + dependencies: + "@types/prop-types" "*" + "@types/scheduler" "*" + csstype "^3.0.2" + "@types/resolve@1.17.1": version "1.17.1" resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.17.1.tgz#3afd6ad8967c77e4376c598a82ddd58f46ec45d6" @@ -4682,7 +4723,7 @@ acorn-walk@^7.0.0, acorn-walk@^7.1.1, acorn-walk@^7.2.0: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== -acorn-walk@^8.0.0: +acorn-walk@^8.0.0, acorn-walk@^8.1.1: version "8.2.0" resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== @@ -5000,6 +5041,11 @@ are-we-there-yet@^2.0.0: delegates "^1.0.0" readable-stream "^3.6.0" +arg@^4.1.0: + version "4.1.3" + resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" + integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== + arg@^5.0.0, arg@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.1.tgz#eb0c9a8f77786cad2af8ff2b862899842d7b6adb" @@ -6788,6 +6834,11 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: safe-buffer "^5.0.1" sha.js "^2.4.8" +create-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" + integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== + cross-env@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf" @@ -7529,6 +7580,11 @@ diff-sequences@^27.5.1: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327" integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ== +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + diffie-hellman@^5.0.0: version "5.0.3" resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875" @@ -11517,7 +11573,7 @@ make-dir@^3.0.0, make-dir@^3.0.2, make-dir@^3.1.0: dependencies: semver "^6.0.0" -make-error@1.x: +make-error@1.x, make-error@^1.1.1: version "1.3.6" resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== @@ -16602,6 +16658,25 @@ ts-loader@^9.2.8: micromatch "^4.0.0" semver "^7.3.4" +ts-node@^10.7.0: + version "10.7.0" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.7.0.tgz#35d503d0fab3e2baa672a0e94f4b40653c2463f5" + integrity sha512-TbIGS4xgJoX2i3do417KSaep1uRAW/Lu+WAL2doDHC0D6ummjirVOXU5/7aiZotbQ5p1Zp9tP7U6cYhA0O7M8A== + dependencies: + "@cspotcode/source-map-support" "0.7.0" + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" + arg "^4.1.0" + create-require "^1.1.0" + diff "^4.0.1" + make-error "^1.1.1" + v8-compile-cache-lib "^3.0.0" + yn "3.1.1" + ts-pnp@^1.1.6: version "1.2.0" resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.2.0.tgz#a500ad084b0798f1c3071af391e65912c86bca92" @@ -17099,6 +17174,11 @@ uuid@^8.0.0, uuid@^8.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +v8-compile-cache-lib@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" + integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== + v8-compile-cache@^2.0.3: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" @@ -18337,6 +18417,11 @@ yargs@~17.2.1: y18n "^5.0.5" yargs-parser "^20.2.2" +yn@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" + integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== + yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"