From f489d0bcfee15a146ecce168ca61d9cb6347cd80 Mon Sep 17 00:00:00 2001 From: Sam Nolan Date: Wed, 28 Sep 2022 14:14:35 +1000 Subject: [PATCH] Add project serialization and run project --- .../ReducerProject/ReducerProject_test.res | 39 +++++++++++++++++++ ...leLibrary_FunctionRegistryLibrary_test.res | 5 ++- packages/squiggle-lang/scripts/lib.mjs | 23 +++++++++++ .../squiggle-lang/scripts/run-project.mjs | 28 +++++++++++++ packages/squiggle-lang/src/js/SqProject.ts | 10 +++++ .../squiggle-lang/src/rescript/FR/FR_Dict.res | 18 +++++---- .../squiggle-lang/src/rescript/FR/FR_Dist.res | 10 ++--- .../squiggle-lang/src/rescript/FR/FR_List.res | 4 +- .../src/rescript/FR/FR_Pointset.res | 32 +++++++-------- .../src/rescript/FR/FR_Sampleset.res | 5 +-- .../src/rescript/FR/FR_Scoring.res | 16 ++++---- .../rescript/ForTS/ForTS_ReducerProject.res | 7 ++++ .../FunctionRegistry_Core.res | 20 +++++----- .../FunctionRegistry_Helpers.res | 24 ++++++------ .../ReducerProject/ReducerProject.res | 22 +++++++++++ .../ReducerProject_ProjectItem.res | 1 - .../ReducerProject/ReducerProject_T.res | 11 ++++++ 17 files changed, 211 insertions(+), 64 deletions(-) create mode 100755 packages/squiggle-lang/scripts/run-project.mjs diff --git a/packages/squiggle-lang/__tests__/ReducerProject/ReducerProject_test.res b/packages/squiggle-lang/__tests__/ReducerProject/ReducerProject_test.res index aee917a3..41e4e10e 100644 --- a/packages/squiggle-lang/__tests__/ReducerProject/ReducerProject_test.res +++ b/packages/squiggle-lang/__tests__/ReducerProject/ReducerProject_test.res @@ -180,3 +180,42 @@ describe("project with independent sources", () => { expect(Project.getRunOrderFor(project, "second")) == ["second"] }) }) + +describe("project serialization", () => { + let project = Project.createProject() + Project.setSource(project, "first", "x = 1") + Project.setSource(project, "second", "y = 1") + Project.setSource(project, "main", "x + y") + Project.setContinues(project, "main", ["first", "second"]) + let expectedJson: Project.reducerProjectJson = { + environment: { + sampleCount: 10000, + xyPointLength: 1000, + }, + items: [ + { + continues: [], + id: "first", + source: "x = 1", + }, + { + continues: ["first", "second"], + id: "main", + source: "x + y", + }, + { + continues: [], + id: "second", + source: "y = 1", + }, + ], + } + + test("serializes correctly", () => { + expect(Project.toJson(project)) == expectedJson + }) + + test("serializes reflexive", () => { + expect(Project.toJson(Project.fromJson(Project.toJson(project)))) == Project.toJson(project) + }) +}) diff --git a/packages/squiggle-lang/__tests__/SquiggleLibrary/SquiggleLibrary_FunctionRegistryLibrary_test.res b/packages/squiggle-lang/__tests__/SquiggleLibrary/SquiggleLibrary_FunctionRegistryLibrary_test.res index fd1ec78e..af2c3d7a 100644 --- a/packages/squiggle-lang/__tests__/SquiggleLibrary/SquiggleLibrary_FunctionRegistryLibrary_test.res +++ b/packages/squiggle-lang/__tests__/SquiggleLibrary/SquiggleLibrary_FunctionRegistryLibrary_test.res @@ -82,7 +82,10 @@ describe("FunctionRegistry Library", () => { ) testEvalToBe("Dict.merge({a: 1, b: 2}, {b: 3, c: 4, d: 5})", "Ok({a: 1,b: 3,c: 4,d: 5})") - testEvalToBe("Dict.mergeMany([{a: 1, b: 2}, {c: 3, d: 4}, {c: 5, e: 6}])", "Ok({a: 1,b: 2,c: 5,d: 4,e: 6})") + testEvalToBe( + "Dict.mergeMany([{a: 1, b: 2}, {c: 3, d: 4}, {c: 5, e: 6}])", + "Ok({a: 1,b: 2,c: 5,d: 4,e: 6})", + ) testEvalToBe("Dict.keys({a: 1, b: 2})", "Ok(['a','b'])") testEvalToBe("Dict.values({a: 1, b: 2})", "Ok([1,2])") testEvalToBe("Dict.toList({a: 1, b: 2})", "Ok([['a',1],['b',2]])") diff --git a/packages/squiggle-lang/scripts/lib.mjs b/packages/squiggle-lang/scripts/lib.mjs index 6f778769..606f58ec 100644 --- a/packages/squiggle-lang/scripts/lib.mjs +++ b/packages/squiggle-lang/scripts/lib.mjs @@ -39,3 +39,26 @@ export const run = (src, { output, sampleCount } = {}) => { result.tag === "Error" ? result.value.toString() : "" ); }; + +export const runProject = (project, output) => { + const time = measure(() => project.runAll()); + console.log("Time: ", time); + + const ids = project.getSourceIds(); + + ids.forEach((id) => { + const result = project.getResult(id); + const bindings = project.getBindings("main"); + + console.log(id + ":"); + if (output) { + console.log("Result:", result.tag, result.value.toString()); + console.log("Bindings:", bindings.toString()); + } + + console.log( + result.tag === "Error" ? red(result.tag) : green(result.tag), + result.tag === "Error" ? result.value.toString() : "" + ); + }); +}; diff --git a/packages/squiggle-lang/scripts/run-project.mjs b/packages/squiggle-lang/scripts/run-project.mjs new file mode 100755 index 00000000..b6f841ac --- /dev/null +++ b/packages/squiggle-lang/scripts/run-project.mjs @@ -0,0 +1,28 @@ +#!/usr/bin/env node +import fs from "fs"; + +import { Command } from "commander"; + +import { SqProject } from "@quri/squiggle-lang"; +import { runProject } from "./lib.mjs"; + +const program = new Command(); + +program.option("-o, --output"); +program.arguments(""); + +const options = program.parse(process.argv); + +const sampleCount = process.env.SAMPLE_COUNT ?? 10000; +console.log(sampleCount); + +const src = fs.readFileSync(program.args[0], "utf-8"); +if (!src) { + throw new Error("Expected src"); +} + +const projectJson = JSON.parse(src); + +const project = SqProject.fromJson(projectJson); + +runProject(project, options.output); diff --git a/packages/squiggle-lang/src/js/SqProject.ts b/packages/squiggle-lang/src/js/SqProject.ts index 9e79b564..aee5396e 100644 --- a/packages/squiggle-lang/src/js/SqProject.ts +++ b/packages/squiggle-lang/src/js/SqProject.ts @@ -115,4 +115,14 @@ export class SqProject { getEnvironment(): environment { return RSProject.getEnvironment(this._value); } + + toJSON(): RSProject.reducerProjectJson { + return RSProject.toJson(this._value); + } + + static fromJson( + projectJson: RSProject.reducerProjectJson + ): RSProject.reducerProject { + return RSProject.fromJson(projectJson); + } } diff --git a/packages/squiggle-lang/src/rescript/FR/FR_Dict.res b/packages/squiggle-lang/src/rescript/FR/FR_Dict.res index 13bcfe33..d135717c 100644 --- a/packages/squiggle-lang/src/rescript/FR/FR_Dict.res +++ b/packages/squiggle-lang/src/rescript/FR/FR_Dict.res @@ -59,7 +59,6 @@ let library = [ ], (), ), - Function.make( ~name="mergeMany", ~nameSpace, @@ -72,13 +71,16 @@ let library = [ ~inputs=[FRTypeArray(FRTypeDict(FRTypeAny))], ~run=(inputs, _, _) => { switch inputs { - | [IEvArray(dicts)] => { - dicts->Belt.Array.map(dictValue => switch dictValue { - | IEvRecord(dict) => dict - | _ => impossibleError->Reducer_ErrorValue.toException - })->Internals.mergeMany->Ok - } - | _ => impossibleError->Error + | [IEvArray(dicts)] => dicts + ->Belt.Array.map(dictValue => + switch dictValue { + | IEvRecord(dict) => dict + | _ => impossibleError->Reducer_ErrorValue.toException + } + ) + ->Internals.mergeMany + ->Ok + | _ => impossibleError->Error } }, (), diff --git a/packages/squiggle-lang/src/rescript/FR/FR_Dist.res b/packages/squiggle-lang/src/rescript/FR/FR_Dist.res index 3a7504f2..97befcd4 100644 --- a/packages/squiggle-lang/src/rescript/FR/FR_Dist.res +++ b/packages/squiggle-lang/src/rescript/FR/FR_Dist.res @@ -22,8 +22,7 @@ module DistributionCreation = { FnDefinition.make( ~name, ~inputs=[FRTypeDistOrNumber, FRTypeDistOrNumber], - ~run=(inputs, env, _) => - inputs->Prepare.ToValueTuple.twoDistOrNumber->process(~fn, ~env), + ~run=(inputs, env, _) => inputs->Prepare.ToValueTuple.twoDistOrNumber->process(~fn, ~env), (), ) } @@ -43,7 +42,9 @@ module DistributionCreation = { ~name, ~inputs=[FRTypeRecord([("mean", FRTypeDistOrNumber), ("stdev", FRTypeDistOrNumber)])], ~run=(inputs, env, _) => - inputs->Prepare.ToValueTuple.Record.twoDistOrNumber(("mean", "stdev"))->process(~fn, ~env), + inputs + ->Prepare.ToValueTuple.Record.twoDistOrNumber(("mean", "stdev")) + ->process(~fn, ~env), (), ) } @@ -60,8 +61,7 @@ module DistributionCreation = { FnDefinition.make( ~name, ~inputs=[FRTypeDistOrNumber], - ~run=(inputs, env, _) => - inputs->Prepare.ToValueTuple.oneDistOrNumber->process(~fn, ~env), + ~run=(inputs, env, _) => inputs->Prepare.ToValueTuple.oneDistOrNumber->process(~fn, ~env), (), ) } diff --git a/packages/squiggle-lang/src/rescript/FR/FR_List.res b/packages/squiggle-lang/src/rescript/FR/FR_List.res index fb675417..21106978 100644 --- a/packages/squiggle-lang/src/rescript/FR/FR_List.res +++ b/packages/squiggle-lang/src/rescript/FR/FR_List.res @@ -109,8 +109,8 @@ let library = [ ~inputs=[FRTypeNumber, FRTypeNumber], ~run=(inputs, _, _) => switch inputs { - | [IEvNumber(low), IEvNumber(high)] => Internals.upTo(low, high)->Ok - | _ => impossibleError->Error + | [IEvNumber(low), IEvNumber(high)] => Internals.upTo(low, high)->Ok + | _ => impossibleError->Error }, (), ), diff --git a/packages/squiggle-lang/src/rescript/FR/FR_Pointset.res b/packages/squiggle-lang/src/rescript/FR/FR_Pointset.res index 4babe231..ee4f7ca8 100644 --- a/packages/squiggle-lang/src/rescript/FR/FR_Pointset.res +++ b/packages/squiggle-lang/src/rescript/FR/FR_Pointset.res @@ -7,24 +7,24 @@ let requiresNamespace = true let inputsToDist = (inputs: array, xyShapeToPointSetDist) => { // TODO - rewrite in more functional/functor-based style switch inputs { - | [IEvArray(items)] => { - items->Belt.Array.map( - item => - switch item { - | IEvRecord(map) => { - let xValue = map->Belt.Map.String.get("x") - let yValue = map->Belt.Map.String.get("y") - switch (xValue, yValue) { - | (Some(IEvNumber(x)), Some(IEvNumber(y))) => (x, y) - | _ => impossibleError->Reducer_ErrorValue.toException - } - } + | [IEvArray(items)] => items + ->Belt.Array.map(item => + switch item { + | IEvRecord(map) => { + let xValue = map->Belt.Map.String.get("x") + let yValue = map->Belt.Map.String.get("y") + switch (xValue, yValue) { + | (Some(IEvNumber(x)), Some(IEvNumber(y))) => (x, y) | _ => impossibleError->Reducer_ErrorValue.toException + } } - )->Ok->E.R.bind(r => r->XYShape.T.makeFromZipped->E.R2.errMap(XYShape.Error.toString)) - ->E.R2.fmap(r => Reducer_T.IEvDistribution(PointSet(r->xyShapeToPointSetDist))) - } - | _ => impossibleError->Reducer_ErrorValue.toException + | _ => impossibleError->Reducer_ErrorValue.toException + } + ) + ->Ok + ->E.R.bind(r => r->XYShape.T.makeFromZipped->E.R2.errMap(XYShape.Error.toString)) + ->E.R2.fmap(r => Reducer_T.IEvDistribution(PointSet(r->xyShapeToPointSetDist))) + | _ => impossibleError->Reducer_ErrorValue.toException } } diff --git a/packages/squiggle-lang/src/rescript/FR/FR_Sampleset.res b/packages/squiggle-lang/src/rescript/FR/FR_Sampleset.res index 5bbfb506..9263bf29 100644 --- a/packages/squiggle-lang/src/rescript/FR/FR_Sampleset.res +++ b/packages/squiggle-lang/src/rescript/FR/FR_Sampleset.res @@ -115,9 +115,8 @@ let libaryBase = [ ~inputs=[FRTypeArray(FRTypeNumber)], ~run=(inputs, _, _) => { let sampleSet = - inputs->Prepare.ToTypedArray.numbers |> E.R2.bind(r => - SampleSetDist.make(r)->E.R2.errMap(_ => "AM I HERE? WHYERE AMI??") - ) + inputs->Prepare.ToTypedArray.numbers + |> E.R2.bind(r => SampleSetDist.make(r)->E.R2.errMap(_ => "AM I HERE? WHYERE AMI??")) sampleSet ->E.R2.fmap(Wrappers.sampleSet) ->E.R2.fmap(Wrappers.evDistribution) diff --git a/packages/squiggle-lang/src/rescript/FR/FR_Scoring.res b/packages/squiggle-lang/src/rescript/FR/FR_Scoring.res index 63589e83..5b780da5 100644 --- a/packages/squiggle-lang/src/rescript/FR/FR_Scoring.res +++ b/packages/squiggle-lang/src/rescript/FR/FR_Scoring.res @@ -31,14 +31,13 @@ let library = [ ]), ], ~run=(inputs, environment, _) => { - switch FunctionRegistry_Helpers.Prepare.ToValueArray.Record.threeArgs(inputs, ("estimate", "answer", "prior")) { + switch FunctionRegistry_Helpers.Prepare.ToValueArray.Record.threeArgs( + inputs, + ("estimate", "answer", "prior"), + ) { | Ok([IEvDistribution(estimate), IEvDistribution(d), IEvDistribution(prior)]) => runScoring(estimate, Score_Dist(d), Some(prior), environment) - | Ok([ - IEvDistribution(estimate), - IEvNumber(d), - IEvDistribution(prior), - ]) => + | Ok([IEvDistribution(estimate), IEvNumber(d), IEvDistribution(prior)]) => runScoring(estimate, Score_Scalar(d), Some(prior), environment) | Error(e) => Error(e->FunctionRegistry_Helpers.wrapError) | _ => Error(FunctionRegistry_Helpers.impossibleError) @@ -50,7 +49,10 @@ let library = [ ~name="logScore", ~inputs=[FRTypeRecord([("estimate", FRTypeDist), ("answer", FRTypeDistOrNumber)])], ~run=(inputs, environment, _) => { - switch FunctionRegistry_Helpers.Prepare.ToValueArray.Record.twoArgs(inputs, ("estimate", "answer")) { + switch FunctionRegistry_Helpers.Prepare.ToValueArray.Record.twoArgs( + inputs, + ("estimate", "answer"), + ) { | Ok([IEvDistribution(estimate), IEvDistribution(d)]) => runScoring(estimate, Score_Dist(d), None, environment) | Ok([IEvDistribution(estimate), IEvNumber(d)]) => diff --git a/packages/squiggle-lang/src/rescript/ForTS/ForTS_ReducerProject.res b/packages/squiggle-lang/src/rescript/ForTS/ForTS_ReducerProject.res index ac9c167c..f4b8af44 100644 --- a/packages/squiggle-lang/src/rescript/ForTS/ForTS_ReducerProject.res +++ b/packages/squiggle-lang/src/rescript/ForTS/ForTS_ReducerProject.res @@ -1,4 +1,5 @@ @genType type reducerProject = ReducerProject_T.project //re-export +@genType type reducerProjectJson = ReducerProject_T.projectJson //re-export type reducerErrorValue = ForTS_Reducer_ErrorValue.reducerErrorValue //use @@ -214,6 +215,12 @@ let setEnvironment = (project: reducerProject, environment: environment): unit = @genType let getEnvironment = (project: reducerProject): environment => project->Private.getEnvironment +@genType +let toJson = (project: reducerProject): reducerProjectJson => project->Private.toJson + +@genType +let fromJson = (project: reducerProjectJson): reducerProject => project->Private.fromJson + /* Foreign function interface is intentionally demolished. There is another way to do that: Umur. diff --git a/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Core.res b/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Core.res index e6929761..5d606bb8 100644 --- a/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Core.res +++ b/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Core.res @@ -83,19 +83,20 @@ module FRType = { | (FRTypeNumeric, IEvNumber(_)) => true | (FRTypeNumeric, IEvDistribution(Symbolic(#Float(_)))) => true | (FRTypeLambda, IEvLambda(_)) => true - | (FRTypeArray(intendedType), IEvArray(elements)) => { - elements->Belt.Array.every(v => matchWithValue(intendedType, v)) - } + | (FRTypeArray(intendedType), IEvArray(elements)) => elements->Belt.Array.every(v => + matchWithValue(intendedType, v) + ) | (FRTypeDict(r), IEvRecord(map)) => map->Belt.Map.String.valuesToArray->Belt.Array.every(v => matchWithValue(r, v)) - | (FRTypeRecord(recordParams), IEvRecord(map)) => { - recordParams->Belt.Array.every(((name, input)) => { + | (FRTypeRecord(recordParams), IEvRecord(map)) => recordParams->Belt.Array.every((( + name, + input, + )) => { switch map->Belt.Map.String.get(name) { - | Some(v) => matchWithValue(input, v) - | None => false + | Some(v) => matchWithValue(input, v) + | None => false } }) - } | _ => false } @@ -104,8 +105,7 @@ module FRType = { if !isSameLength { false } else { - E.A.zip(inputs, args) - ->Belt.Array.every(((input, arg)) => matchWithValue(input, arg)) + E.A.zip(inputs, args)->Belt.Array.every(((input, arg)) => matchWithValue(input, arg)) } } } diff --git a/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Helpers.res b/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Helpers.res index a5a0f523..9a7f04db 100644 --- a/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Helpers.res +++ b/packages/squiggle-lang/src/rescript/FunctionRegistry/FunctionRegistry_Helpers.res @@ -30,21 +30,21 @@ module Prepare = { let twoArgs = (inputs: ts, (arg1: string, arg2: string)): result => switch inputs { | [IEvRecord(map)] => { - let n1 = map->Belt.Map.String.getExn(arg1) - let n2 = map->Belt.Map.String.getExn(arg2) - Ok([n1, n2]) - } + let n1 = map->Belt.Map.String.getExn(arg1) + let n2 = map->Belt.Map.String.getExn(arg2) + Ok([n1, n2]) + } | _ => Error(impossibleErrorString) } let threeArgs = (inputs: ts, (arg1: string, arg2: string, arg3: string)): result => switch inputs { | [IEvRecord(map)] => { - let n1 = map->Belt.Map.String.getExn(arg1) - let n2 = map->Belt.Map.String.getExn(arg2) - let n3 = map->Belt.Map.String.getExn(arg3) - Ok([n1, n2, n3]) - } + let n1 = map->Belt.Map.String.getExn(arg1) + let n2 = map->Belt.Map.String.getExn(arg2) + let n3 = map->Belt.Map.String.getExn(arg3) + Ok([n1, n2, n3]) + } | _ => Error(impossibleErrorString) } } @@ -108,8 +108,10 @@ module Prepare = { } module Record = { - let twoDistOrNumber = (values: ts, labels: (string, string)): result<(frValueDistOrNumber, frValueDistOrNumber), err> => - values->ToValueArray.Record.twoArgs(labels)->E.R.bind(twoDistOrNumber) + let twoDistOrNumber = (values: ts, labels: (string, string)): result< + (frValueDistOrNumber, frValueDistOrNumber), + err, + > => values->ToValueArray.Record.twoArgs(labels)->E.R.bind(twoDistOrNumber) let twoDist = (values: ts, labels: (string, string)): result< (DistributionTypes.genericDist, DistributionTypes.genericDist), diff --git a/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject.res b/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject.res index 1942a136..4efa535e 100644 --- a/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject.res +++ b/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject.res @@ -147,6 +147,28 @@ let setEnvironment = (project: t, value: Reducer_T.environment): unit => { project.environment = value } +let toJson = (project: t): T.projectJson => { + items: Belt.Array.map(Belt.MutableMap.String.toArray(project.items), ((id, projectItem)) => { + let projectItem: T.projectItemJson = { + id: id, + source: ProjectItem.getSource(projectItem), + continues: ProjectItem.getContinues(projectItem), + } + projectItem + }), + environment: getEnvironment(project), +} + +let fromJson = (json: T.projectJson): t => { + let project = createProject() + setEnvironment(project, json.environment) + Belt.Array.forEach(json.items, item => { + setSource(project, item.id, item.source) + setContinues(project, item.id, item.continues) + }) + project +} + let getBindings = (project: t, sourceId: string): Reducer_T.namespace => { project->getItem(sourceId)->ProjectItem.getContinuation } diff --git a/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_ProjectItem.res b/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_ProjectItem.res index 93d434a3..c4ec1208 100644 --- a/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_ProjectItem.res +++ b/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_ProjectItem.res @@ -182,7 +182,6 @@ let doRun = (this: t, context: Reducer_T.context): t => ->setContinuation(contextAfterEvaluation.bindings->Reducer_Bindings.locals) } catch { | Reducer_ErrorValue.ErrorException(e) => this->failRun(e) - | _ => this->failRun(RETodo("unhandled rescript exception")) } | Error(e) => this->failRun(e) } diff --git a/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_T.res b/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_T.res index 0eac623a..58abd2e7 100644 --- a/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_T.res +++ b/packages/squiggle-lang/src/rescript/ReducerProject/ReducerProject_T.res @@ -9,6 +9,17 @@ type project = { } type t = project +type projectItemJson = { + id: string, + source: string, + continues: array, +} + +type projectJson = { + items: array, + environment: Reducer_T.environment, +} + // these functions are used in ReducerProject_Topology, so they are defined here to avoid circular dependencies let getSourceIds = (project: t): array => Belt.MutableMap.String.keysToArray(project.items)