From 6463b52052e712af2e035bd980f8a52b6f7dfc3f Mon Sep 17 00:00:00 2001 From: Vyacheslav Matyukhin Date: Thu, 15 Sep 2022 15:17:19 +0400 Subject: [PATCH] benchmark and other scripts --- .../benchmark/Benchmark_Array.res | 76 +++++++++++++++++++ .../benchmark/Benchmark_Helpers.res | 11 +++ .../squiggle-lang/benchmark/Benchmark_Map.res | 65 ++++++++++++++++ packages/squiggle-lang/bsconfig.json | 12 ++- .../squiggle-lang/examples/integrate.squiggle | 13 ++++ packages/squiggle-lang/package.json | 1 + packages/squiggle-lang/scripts/README.md | 3 + .../{bench.js => scripts/bench-map-reduce.js} | 14 ++-- packages/squiggle-lang/scripts/bench-map.js | 27 +++++++ packages/squiggle-lang/scripts/run-file.js | 38 ++++++++++ packages/squiggle-lang/scripts/run.js | 18 +++++ yarn.lock | 22 ++++-- 12 files changed, 287 insertions(+), 13 deletions(-) create mode 100644 packages/squiggle-lang/benchmark/Benchmark_Array.res create mode 100644 packages/squiggle-lang/benchmark/Benchmark_Helpers.res create mode 100644 packages/squiggle-lang/benchmark/Benchmark_Map.res create mode 100644 packages/squiggle-lang/examples/integrate.squiggle create mode 100644 packages/squiggle-lang/scripts/README.md rename packages/squiggle-lang/{bench.js => scripts/bench-map-reduce.js} (61%) create mode 100755 packages/squiggle-lang/scripts/bench-map.js create mode 100755 packages/squiggle-lang/scripts/run-file.js create mode 100755 packages/squiggle-lang/scripts/run.js diff --git a/packages/squiggle-lang/benchmark/Benchmark_Array.res b/packages/squiggle-lang/benchmark/Benchmark_Array.res new file mode 100644 index 00000000..61562732 --- /dev/null +++ b/packages/squiggle-lang/benchmark/Benchmark_Array.res @@ -0,0 +1,76 @@ +module Map: Benchmark_Helpers.BenchmarkTopic = { + let arraySize = 1000 + let iterations = 300_000 + + let beltArray = () => { + let x = Belt.Array.make(arraySize, 0.) + Belt.Range.forEach(1, iterations, _ => { + let _ = x->Belt.Array.map(v => v) + }) + } + + let jsArray2 = () => { + let x = Belt.Array.make(arraySize, 0.) + Belt.Range.forEach(1, iterations, _ => { + let _ = x->Js.Array2.map(v => v) + }) + } + + let ocamlArray = () => { + let x = Belt.Array.make(arraySize, 0.) + Belt.Range.forEach(1, iterations, _ => { + let _ = x->Array.map(v => v, _) + }) + } + + let runAll = () => { + Js.log(`Mapping identity function over arrays of size ${arraySize->Js.Int.toString} (${iterations->Js.Int.toString} iterations)`) + Benchmark_Helpers.measure("Belt.Array.map", beltArray) + Benchmark_Helpers.measure("Js.Array2.map", jsArray2) + Benchmark_Helpers.measure("Array.map", ocamlArray) + } +} + +module Sort: Benchmark_Helpers.BenchmarkTopic = { + let arraySize = 1000 + let iterations = 30000 + + let jsArray2 = () => { + let x = Belt.Array.make(arraySize, 0.) + let compare = (a: float, b: float) => { + if a < b { + -1 + } else { + 1 + } + } + + Belt.Range.forEach(1, iterations, _ => { + let _ = x->Js.Array2.sortInPlaceWith(compare) + }) + } + + let jsArray2withOcamlCompare = () => { + let x = Belt.Array.make(arraySize, 0.) + Belt.Range.forEach(1, iterations, _ => { + let _ = x->Js.Array2.sortInPlaceWith(Pervasives.compare) + }) + } + + let ocamlArray = () => { + let x = Belt.Array.make(arraySize, 0.) + Belt.Range.forEach(1, iterations, _ => { + let _ = x->Array.fast_sort(compare, _) + }) + } + + let runAll = () => { + Js.log(`Sorting arrays of size ${arraySize->Js.Int.toString} (${iterations->Js.Int.toString} iterations)`) + Benchmark_Helpers.measure("Js.Array2.sort", jsArray2) + Benchmark_Helpers.measure("Js.Array2.sort with Ocaml compare", jsArray2withOcamlCompare) + Benchmark_Helpers.measure("Array.fast_sort", ocamlArray) + } +} + +Map.runAll() +Sort.runAll() diff --git a/packages/squiggle-lang/benchmark/Benchmark_Helpers.res b/packages/squiggle-lang/benchmark/Benchmark_Helpers.res new file mode 100644 index 00000000..0e889d15 --- /dev/null +++ b/packages/squiggle-lang/benchmark/Benchmark_Helpers.res @@ -0,0 +1,11 @@ +module type BenchmarkTopic = { + let runAll: () => unit +} + +let measure = (name: string, f: () => unit) => { + let start = Js.Date.make()->Js.Date.valueOf + f() + let end = Js.Date.make()->Js.Date.valueOf + let duration = (end -. start) /. 1000. + Js.log2(duration, name) +} diff --git a/packages/squiggle-lang/benchmark/Benchmark_Map.res b/packages/squiggle-lang/benchmark/Benchmark_Map.res new file mode 100644 index 00000000..d04d501d --- /dev/null +++ b/packages/squiggle-lang/benchmark/Benchmark_Map.res @@ -0,0 +1,65 @@ +module StringMap: Benchmark_Helpers.BenchmarkTopic = { + let size = 1000 + let iterations = 10_000 + + let kv = Belt.Array.range(1, size)->Belt.Array.map( + v => ("key" ++ v->Belt.Int.toString, v) + ) + + let beltMap = () => { + Belt.Range.forEach(1, iterations, _ => { + let m = Belt.Map.String.empty + let _ = Belt.Array.reduce(kv, m, (acc, (k, v)) => acc->Belt.Map.String.set(k, v)) + }) + } + + let beltMutableMap = () => { + Belt.Range.forEach(1, iterations, _ => { + let m = Belt.MutableMap.String.make() + let _ = Belt.Array.reduce(kv, m, (acc, (k, v)) => { + acc->Belt.MutableMap.String.set(k, v) + acc + }) + }) + } + + let beltHashMap = () => { + Belt.Range.forEach(1, iterations, _ => { + let m = Belt.HashMap.String.make(~hintSize=100) + let _ = Belt.Array.reduce(kv, m, (acc, (k, v)) => { + acc->Belt.HashMap.String.set(k, v) + acc + }) + }) + } + + let jsDict = () => { + Belt.Range.forEach(1, iterations, _ => { + let m = Js.Dict.empty() + let _ = Belt.Array.reduce(kv, m, (acc, (k, v)) => { + acc->Js.Dict.set(k, v) + acc + }) + }) + } + + let jsMap = () => { + Belt.Range.forEach(1, iterations, _ => { + let m = Js_map.make() + let _ = Belt.Array.reduce(kv, m, (acc, (k, v)) => + acc->Js_map.set(k, v) + ) + }) + } + + let runAll = () => { + Js.log(`Filling a map with ("key{i}" => "i") key-value pairs, size ${size->Js.Int.toString} (${iterations->Js.Int.toString} iterations)`) + Benchmark_Helpers.measure("Belt.Map.String", beltMap) + Benchmark_Helpers.measure("Belt.MutableMap.String", beltMutableMap) + Benchmark_Helpers.measure("Belt.HashMap.String", beltHashMap) + Benchmark_Helpers.measure("Js.Dict", jsDict) + Benchmark_Helpers.measure("Js.Map", jsMap) + } +} + +let runAll = StringMap.runAll() diff --git a/packages/squiggle-lang/bsconfig.json b/packages/squiggle-lang/bsconfig.json index 1fa663ac..a05df290 100644 --- a/packages/squiggle-lang/bsconfig.json +++ b/packages/squiggle-lang/bsconfig.json @@ -9,6 +9,11 @@ "dir": "__tests__", "type": "dev", "subdirs": true + }, + { + "dir": "benchmark", + "type": "dev", + "subdirs": true } ], "bsc-flags": ["-bs-super-errors", "-bs-no-version-header", "-bs-g"], @@ -21,7 +26,12 @@ "suffix": ".bs.js", "namespace": true, "bs-dependencies": ["bisect_ppx"], - "bs-dev-dependencies": ["@glennsl/rescript-jest", "rescript-fast-check"], + "bs-dev-dependencies": [ + "@glennsl/rescript-jest", + "rescript-fast-check", + "rescript-js-map", + "rescript-js-iterator" + ], "gentypeconfig": { "language": "typescript", "module": "commonjs", diff --git a/packages/squiggle-lang/examples/integrate.squiggle b/packages/squiggle-lang/examples/integrate.squiggle new file mode 100644 index 00000000..d41e18c7 --- /dev/null +++ b/packages/squiggle-lang/examples/integrate.squiggle @@ -0,0 +1,13 @@ +integrate(fun, min, max) = { + // assume that min and max are integers. + epsilon = 1 + l = max - min + meanF(t) = fun(t) + intervals = map(List.upTo(0, (l/epsilon)), ({|n| min + n*epsilon})) + values = map(intervals, ({ |x | meanF(x)})) + result = reduce(values, 0, ({|acc, x| acc + x})) * epsilon + result +} + +f(x) = x +integrate(f, 1, 100k) diff --git a/packages/squiggle-lang/package.json b/packages/squiggle-lang/package.json index 80c542f5..89ea4432 100644 --- a/packages/squiggle-lang/package.json +++ b/packages/squiggle-lang/package.json @@ -66,6 +66,7 @@ "reanalyze": "^2.23.0", "rescript": "^9.1.4", "rescript-fast-check": "^1.1.1", + "rescript-js-map": "^1.1.0", "ts-jest": "^27.1.4", "ts-loader": "^9.3.0", "ts-node": "^10.9.1", diff --git a/packages/squiggle-lang/scripts/README.md b/packages/squiggle-lang/scripts/README.md new file mode 100644 index 00000000..e8dea081 --- /dev/null +++ b/packages/squiggle-lang/scripts/README.md @@ -0,0 +1,3 @@ +Various scripts used for development, benchmarking and testing. + +None of these are bundled in NPM package yet. diff --git a/packages/squiggle-lang/bench.js b/packages/squiggle-lang/scripts/bench-map-reduce.js similarity index 61% rename from packages/squiggle-lang/bench.js rename to packages/squiggle-lang/scripts/bench-map-reduce.js index effccdfa..ffada066 100755 --- a/packages/squiggle-lang/bench.js +++ b/packages/squiggle-lang/scripts/bench-map-reduce.js @@ -1,5 +1,5 @@ #!/usr/bin/env node -const s = require("./dist/src/js"); +const s = require("@quri/squiggle-lang"); const measure = (cb, times = 1) => { const t1 = new Date(); @@ -11,17 +11,17 @@ const measure = (cb, times = 1) => { return (t2 - t1) / 1000; }; -const maxP = 7; +const maxP = 5; for (let p = 0; p <= maxP; p++) { const size = Math.pow(10, p); const prj = s.SqProject.create(); - prj.setSource("list", `l = List.upTo(1,${size})`); - prj.run("list"); - prj.setSource("map", "l |> map({|x| x})"); - prj.setContinues("map", ["list"]); + prj.setSource( + "main", + `List.upTo(1, ${size}) |> map({|x| List.upTo(1, 100) |> reduce(0, {|a,b|a+b})})` + ); const t = measure(() => { - prj.run("map"); + prj.run("main"); }); console.log(`1e${p}`, "\t", t); } diff --git a/packages/squiggle-lang/scripts/bench-map.js b/packages/squiggle-lang/scripts/bench-map.js new file mode 100755 index 00000000..befa1a7b --- /dev/null +++ b/packages/squiggle-lang/scripts/bench-map.js @@ -0,0 +1,27 @@ +#!/usr/bin/env node +const s = require("@quri/squiggle-lang"); + +const measure = (cb, times = 1) => { + const t1 = new Date(); + + for (let i = 1; i <= times; i++) { + cb(); + } + const t2 = new Date(); + return (t2 - t1) / 1000; +}; + +const maxP = 7; + +for (let p = 0; p <= maxP; p++) { + const size = Math.pow(10, p); + const project = s.SqProject.create(); + project.setSource("list", `l = List.upTo(1,${size})`); + project.run("list"); + project.setSource("map", "l |> map({|x| x})"); + project.setContinues("map", ["list"]); + const time = measure(() => { + project.run("map"); + }); + console.log(`1e${p}`, "\t", time); +} diff --git a/packages/squiggle-lang/scripts/run-file.js b/packages/squiggle-lang/scripts/run-file.js new file mode 100755 index 00000000..fb09aa23 --- /dev/null +++ b/packages/squiggle-lang/scripts/run-file.js @@ -0,0 +1,38 @@ +#!/usr/bin/env node +const s = require("@quri/squiggle-lang"); +const fs = require("fs"); + +const measure = (cb, times = 1) => { + const t1 = new Date(); + + for (let i = 1; i <= times; i++) { + cb(); + } + const t2 = new Date(); + return (t2 - t1) / 1000; +}; + +const project = s.SqProject.create(); +const sampleCount = process.env.SAMPLE_COUNT; +if (sampleCount) { + project.setEnvironment({ + sampleCount, + xyPointLength: sampleCount, + }); +} + +const src = fs.readFileSync(process.argv[2], "utf-8"); +if (!src) { + throw new Error("Expected src"); +} +console.log(`Running ${src}`); +project.setSource("a", src); + +const t = measure(() => project.run("a")); +console.log(`Time: ${t}`); + +const result = project.getResult("a"); +console.log("Result:", result.tag, result.value.toString()); + +const bindings = project.getBindings("a"); +console.log("Bindings:", bindings.asValue().toString()); diff --git a/packages/squiggle-lang/scripts/run.js b/packages/squiggle-lang/scripts/run.js new file mode 100755 index 00000000..e77b953b --- /dev/null +++ b/packages/squiggle-lang/scripts/run.js @@ -0,0 +1,18 @@ +#!/usr/bin/env node +const s = require("@quri/squiggle-lang"); + +const p = s.SqProject.create(); + +const src = process.argv[2]; +if (!src) { + throw new Error("Expected src"); +} +console.log(`Running ${src}`); +p.setSource("a", src); +p.run("a"); + +const result = p.getResult("a"); +console.log(result.tag, result.value.toString()); + +const bindings = p.getBindings("a"); +console.log(bindings.asValue().toString()); diff --git a/yarn.lock b/yarn.lock index 9a156c50..f4ed4bd8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4797,10 +4797,10 @@ "@types/history" "^4.7.11" "@types/react" "*" -"@types/react@*", "@types/react@^18.0.1", "@types/react@^18.0.18": - version "18.0.19" - resolved "https://registry.yarnpkg.com/@types/react/-/react-18.0.19.tgz#269a5f35b9a73c69dfb0c7189017013ab02acbaa" - integrity sha512-BDc3Q+4Q3zsn7k9xZrKfjWyJsSlEDMs38gD1qp2eDazLCdcPqAT+vq1ND+Z8AGel/UiwzNUk8ptpywgNQcJ1MQ== +"@types/react@*", "@types/react@^18.0.18": + version "18.0.20" + resolved "https://registry.yarnpkg.com/@types/react/-/react-18.0.20.tgz#e4c36be3a55eb5b456ecf501bd4a00fd4fd0c9ab" + integrity sha512-MWul1teSPxujEHVwZl4a5HxQ9vVNsjTchVA+xRqv/VYGCuKGAU6UhfrTdF5aBefwD1BHUD8i/zq+O/vyCm/FrA== dependencies: "@types/prop-types" "*" "@types/scheduler" "*" @@ -15152,7 +15152,7 @@ react-vega@^7.6.0: prop-types "^15.8.1" vega-embed "^6.5.1" -react@^18.0.0, react@^18.1.0: +react@^18.1.0: version "18.2.0" resolved "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5" integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ== @@ -15578,6 +15578,18 @@ rescript-fast-check@^1.1.1: dependencies: fast-check "^2.17.0" +rescript-js-iterator@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/rescript-js-iterator/-/rescript-js-iterator-1.1.0.tgz#7fc1e6097cec5f3b847189e112deed0bca331f4d" + integrity sha512-9VSGfAUXdmOl0OoersJZknpGjCi4gsIuejcHQwIYKwoPPlfswFvrKjwOLTy4SoqM2037RxiwxsMPRlWgfDY22A== + +rescript-js-map@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/rescript-js-map/-/rescript-js-map-1.1.0.tgz#5e4e57d9147733bd2d14307d65f0f8febb5850fc" + integrity sha512-kJB43pIRv/Cd9OY17Fpof+xq6WTCqagdW+zrmBxFIj8SpK48rG4nF8jXSsYK3ksdwIJwh7J3sx3pwA27YkeM+A== + dependencies: + rescript-js-iterator "^1.1.0" + rescript@^9.1.4: version "9.1.4" resolved "https://registry.yarnpkg.com/rescript/-/rescript-9.1.4.tgz#1eb126f98d6c16942c0bf0df67c050198e580515"