Merge pull request #1233 from quantified-uncertainty/turborepo

Turborepo

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Squiggle packages check
+name: Squiggle packages checks
 
 on:
   push:
@@ -9,217 +9,38 @@ on:
     branches:
       - master
       - develop
-      - reducer-dev
+
-      - epic-reducer-project
+env:
-      - epic-0.5.0
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: quantified-uncertainty
 
 jobs:
-  pre_check:
+  build-test-lint:
-    name: Precheck for skipping redundant jobs
+    name: Build, test, lint
     runs-on: ubuntu-latest
-    outputs:
-      should_skip_lang: ${{ steps.skip_lang_check.outputs.should_skip }}
-      should_skip_components: ${{ steps.skip_components_check.outputs.should_skip }}
-      should_skip_website: ${{ steps.skip_website_check.outputs.should_skip }}
-      should_skip_vscodeext: ${{ steps.skip_vscodeext_check.outputs.should_skip }}
-      should_skip_cli: ${{ steps.skip_cli_check.outputs.should_skip }}
     steps:
-      - id: skip_lang_check
+      - uses: actions/checkout@v3
-        name: Check if the changes are about squiggle-lang src files
+      - name: Setup Node.js environment
-        uses: fkirc/skip-duplicate-actions@v5.2.0
+        uses: actions/setup-node@v3
         with:
-          paths: '["packages/squiggle-lang/**"]'
+          node-version: 16
-      - id: skip_components_check
+          cache: 'yarn'
-        name: Check if the changes are about components src files
+      - name: Install dependencies
-        uses: fkirc/skip-duplicate-actions@v5.2.0
+        run: yarn --frozen-lockfile
-        with:
+      - name: Turbo run
-          paths: '["packages/components/**"]'
+        run: npx turbo run build test lint bundle
-      - id: skip_website_check
-        name: Check if the changes are about website src files
-        uses: fkirc/skip-duplicate-actions@v5.2.0
-        with:
-          paths: '["packages/website/**"]'
-      - id: skip_vscodeext_check
-        name: Check if the changes are about vscode extension src files
-        uses: fkirc/skip-duplicate-actions@v5.2.0
-        with:
-          paths: '["packages/vscode-ext/**"]'
-      - id: skip_cli_check
-        name: Check if the changes are about cli src files
-        uses: fkirc/skip-duplicate-actions@v5.2.0
-        with:
-          paths: '["packages/cli/**"]'
 
-  lang-lint:
+  coverage:
-    name: Language lint
+    name: Coverage
     runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ needs.pre_check.outputs.should_skip_lang != 'true' }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/squiggle-lang
     steps:
       - uses: actions/checkout@v3
-      - name: Install Dependencies
+      - name: Setup Node.js environment
-        run: cd ../../ && yarn
+        uses: actions/setup-node@v2
-      - name: Check rescript lint
-        run: yarn lint:rescript
-      - name: Check javascript, typescript, and markdown lint
-        uses: creyD/prettier_action@v4.2
         with:
-          dry: true
+          node-version: 16
-          prettier_options: --check packages/squiggle-lang
+          cache: 'yarn'
-
+      - name: Install dependencies
-  lang-build-test-bundle:
+        run: yarn
-    name: Language build, test, and bundle
+      - name: Coverage
-    runs-on: ubuntu-latest
+        run: npx turbo run coverage
-    needs: pre_check
-    if: ${{ needs.pre_check.outputs.should_skip_lang != 'true' }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/squiggle-lang
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      - name: Install dependencies from monorepo level
-        run: cd ../../ && yarn
-      - name: Build rescript codebase
-        run: yarn build
-      - name: Run rescript tests
-        run: yarn test:rescript
-      - name: Run typescript tests
-        run: yarn test:ts
-      - name: Run webpack
-        run: yarn bundle
-      - name: Upload rescript coverage report
-        run: yarn coverage:rescript:ci
-      - name: Upload typescript coverage report
-        run: yarn coverage:ts:ci
-
-  components-lint:
-    name: Components lint
-    runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ needs.pre_check.outputs.should_skip_components != 'true' }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/components
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check javascript, typescript, and markdown lint
-        uses: creyD/prettier_action@v4.2
-        with:
-          dry: true
-          prettier_options: --check packages/components --ignore-path packages/components/.prettierignore
-
-  components-bundle-build-test:
-    name: Components bundle, build and test
-    runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ (needs.pre_check.outputs.should_skip_components != 'true') || (needs.pre_check.outputs.should_skip_lang != 'true') }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/components
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install dependencies from monorepo level
-        run: cd ../../ && yarn
-      - name: Build rescript codebase in squiggle-lang
-        run: cd ../squiggle-lang && yarn build
-      - name: Run webpack
-        run: yarn bundle
-      - name: Build storybook
-        run: yarn build
-      - name: Test components 
-        run: yarn test
-
-  website-lint:
-    name: Website lint
-    runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ needs.pre_check.outputs.should_skip_website != 'true' }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/website
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check javascript, typescript, and markdown lint
-        uses: creyD/prettier_action@v4.2
-        with:
-          dry: true
-          prettier_options: --check packages/website
-
-  website-build:
-    name: Website build
-    runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ (needs.pre_check.outputs.should_skip_website != 'true') || (needs.pre_check.outputs.should_skip_lang != 'true')  || (needs.pre_check.outputs.should_skip_components != 'true') }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/website
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install dependencies from monorepo level
-        run: cd ../../ && yarn
-      - name: Build rescript in squiggle-lang
-        run: cd ../squiggle-lang && yarn build
-      - name: Build components
-        run: cd ../components && yarn build
-      - name: Build website assets
-        run: yarn build
-
-  vscode-ext-lint:
-    name: VS Code extension lint
-    runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ needs.pre_check.outputs.should_skip_vscodeext != 'true' }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/vscode-ext
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check javascript, typescript, and markdown lint
-        uses: creyD/prettier_action@v4.2
-        with:
-          dry: true
-          prettier_options: --check packages/vscode-ext
-
-  vscode-ext-build:
-    name: VS Code extension build
-    runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ (needs.pre_check.outputs.should_skip_components != 'true') || (needs.pre_check.outputs.should_skip_lang != 'true') }} || (needs.pre_check.outputs.should_skip_vscodeext != 'true') }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/vscode-ext
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install dependencies from monorepo level
-        run: cd ../../ && yarn
-      - name: Build
-        run: yarn compile
-  cli-lint:
-    name: CLI lint
-    runs-on: ubuntu-latest
-    needs: pre_check
-    if: ${{ needs.pre_check.outputs.should_skip_cli != 'true' }}
-    defaults:
-      run:
-        shell: bash
-        working-directory: packages/cli
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check javascript, typescript, and markdown lint
-        uses: creyD/prettier_action@v4.2
-        with:
-          dry: true
-          prettier_options: --check packages/cli

.gitignore

@@ -12,3 +12,4 @@ yarn-error.log
 todo.txt
 result
 shell.nix
+.turbo

README.md

@@ -51,7 +51,19 @@ For any project in the repo, begin by running `yarn` in the top level
 yarn
 ```
 
-See `packages/*/README.md` to work with whatever project you're interested in.
+Then use `turbo` to build the specific packages or the entire monorepo:
+
+```sh
+turbo run build
+```
+
+Or:
+
+```sh
+turbo run build --filter=@quri/squiggle-components
+```
+
+You can also run specific npm scripts for the package you're working on. See `packages/*/README.md` for the details.
 
 # Contributing
 

package.json

@@ -2,12 +2,11 @@
   "private": true,
   "name": "squiggle",
   "scripts": {
-    "nodeclean": "rm -r node_modules && rm -r packages/*/node_modules",
+    "nodeclean": "rm -r node_modules && rm -r packages/*/node_modules"
-    "format:all": "prettier --write . && cd packages/squiggle-lang && yarn format",
-    "lint:all": "prettier --check . && cd packages/squiggle-lang && yarn lint:rescript"
   },
   "devDependencies": {
-    "prettier": "^2.7.1"
+    "prettier": "^2.7.1",
+    "turbo": "^1.5.5"
   },
   "workspaces": [
     "packages/*"

packages/cli/package.json

@@ -7,7 +7,9 @@
   "bin": "index.js",
   "type": "module",
   "scripts": {
-    "start": "node ."
+    "start": "node .",
+    "lint": "prettier --check .",
+    "format": "prettier --write ."
   },
   "license": "MIT",
   "dependencies": {

packages/components/vercel.json

@@ -1,5 +1,4 @@
 {
-  "buildCommand": "cd ../squiggle-lang && yarn build && cd ../components && yarn build",
+  "buildCommand": "cd ../.. && npx turbo run build --filter=@quri/squiggle-components",
-  "outputDirectory": "storybook-static",
+  "outputDirectory": "storybook-static"
-  "installCommand": "cd ../../ && yarn"
 }

packages/squiggle-lang/.prettierignore

@@ -6,3 +6,5 @@ lib
 _coverage/
 .cache/
 Reducer_Peggy_GeneratedParser.js
+ReducerProject_IncludeParser.js
+src/rescript/Reducer/Reducer_Peggy/helpers.js

packages/squiggle-lang/package.json

@@ -22,10 +22,12 @@
     "test:rescript": "jest --modulePathIgnorePatterns=__tests__/TS/*",
     "test:watch": "jest --watchAll",
     "test:fnRegistry": "jest __tests__/SquiggleLibrary/SquiggleLibrary_FunctionRegistryLibrary_test.bs.js",
-    "coverage:rescript": "rm -f *.coverage && yarn clean && BISECT_ENABLE=yes yarn build && yarn test:rescript && bisect-ppx-report html",
+    "coverage:rescript:local": "rm -f *.coverage && yarn clean && BISECT_ENABLE=yes yarn build && yarn test:rescript && bisect-ppx-report html",
-    "coverage:ts": "yarn clean && yarn build && nyc --reporter=lcov yarn test:ts",
+    "coverage:ts:local": "yarn clean && yarn build && nyc --reporter=lcov yarn test:ts",
-    "coverage:rescript:ci": "yarn clean && BISECT_ENABLE=yes yarn build:rescript && yarn test:rescript && bisect-ppx-report send-to Codecov",
+    "coverage:rescript": "yarn clean && BISECT_ENABLE=yes yarn build:rescript && yarn test:rescript && bisect-ppx-report send-to Codecov",
-    "coverage:ts:ci": "yarn coverage:ts && codecov",
+    "coverage:ts": "yarn coverage:ts:local && codecov",
+    "coverage": "yarn coverage:ts && yarn coverage:rescript",
+    "coverage:local": "yarn coverage:ts:local && yarn coverage:rescript:local",
     "lint:rescript": "./lint.sh",
     "lint:prettier": "prettier --check .",
     "lint": "yarn lint:rescript && yarn lint:prettier",

packages/vscode-ext/package.json

@@ -114,13 +114,13 @@
     }
   },
   "scripts": {
-    "vscode:prepublish": "yarn run lint && yarn run compile",
+    "vscode:prepublish": "yarn run lint && yarn run build",
-    "compile:grammar": "js-yaml syntaxes/squiggle.tmLanguage.yaml >syntaxes/squiggle.tmLanguage.json",
+    "build:grammar": "js-yaml syntaxes/squiggle.tmLanguage.yaml >syntaxes/squiggle.tmLanguage.json",
-    "compile:vendor": "(cd ../squiggle-lang && yarn run build) && (cd ../components && yarn run bundle && yarn run build:css) && mkdir -p media/vendor && cp ../components/dist/bundle.js media/vendor/components.js && cp ../components/dist/main.css media/vendor/components.css && cp ../../node_modules/react/umd/react.production.min.js media/vendor/react.js && cp ../../node_modules/react-dom/umd/react-dom.production.min.js media/vendor/react-dom.js && cp ../website/static/img/squiggle-logo.png media/vendor/icon.png",
+    "build:vendor": "mkdir -p media/vendor && cp ../components/dist/bundle.js media/vendor/components.js && cp ../components/dist/main.css media/vendor/components.css && cp ../../node_modules/react/umd/react.production.min.js media/vendor/react.js && cp ../../node_modules/react-dom/umd/react-dom.production.min.js media/vendor/react-dom.js && cp ../website/static/img/squiggle-logo.png media/vendor/icon.png",
-    "compile:bundle": "esbuild ./src/client/extension.ts ./src/server/server.ts --format=cjs --platform=node --sourcemap --minify --bundle --external:vscode --outdir=./dist",
+    "build:bundle": "esbuild ./src/client/extension.ts ./src/server/server.ts --format=cjs --platform=node --sourcemap --minify --bundle --external:vscode --outdir=./dist",
-    "compile": "yarn run compile:vendor && yarn run compile:grammar && yarn run compile:bundle",
+    "build": "yarn run build:vendor && yarn run build:grammar && yarn run build:bundle",
     "watch": "yarn run compile:bundle --watch",
-    "pretest": "yarn run compile && yarn run lint",
+    "pretest": "yarn run build && yarn run lint",
     "lint": "tsc -noEmit && prettier --check .",
     "format": "prettier --write .",
     "package": "vsce package --yarn"
@@ -139,6 +139,7 @@
     "vscode-languageclient": "^8.0.2",
     "vscode-languageserver": "^8.0.2",
     "vscode-languageserver-textdocument": "^1.0.7",
-    "@quri/squiggle-lang": "^0.2.11"
+    "@quri/squiggle-lang": "^0.2.11",
+    "@quri/squiggle-components": "*"
   }
 }

packages/website/package.json

@@ -15,7 +15,7 @@
     "@docusaurus/core": "2.1.0",
     "@docusaurus/preset-classic": "2.1.0",
     "@heroicons/react": "^1.0.6",
-    "@quri/squiggle-components": "^0.5.0",
+    "@quri/squiggle-components": "*",
     "base64-js": "^1.5.1",
     "clsx": "^1.2.1",
     "hast-util-is-element": "2.1.2",

packages/website/vercel.json

@@ -1,5 +1,4 @@
 {
-  "buildCommand": "cd ../squiggle-lang && yarn build && cd ../components && yarn build && cd ../website && yarn build",
+  "buildCommand": "cd ../.. && npx turbo run build --filter=squiggle-website",
-  "framework": "docusaurus-2",
+  "framework": "docusaurus-2"
-  "installCommand": "cd ../../ && yarn"
 }

turbo.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://turborepo.org/schema.json",
+  "pipeline": {
+    "lint": {
+      "outputs": []
+    },
+    "format": {},
+    "build": {
+      "dependsOn": ["^build"]
+    },
+    "vscode-squiggle#build": {
+      "dependsOn": ["^build", "@quri/squiggle-components#bundle"]
+    },
+    "@quri/squiggle-components#build": {
+      "dependsOn": ["^build"],
+      "outputs": ["dist/**", "storybook-static/**"]
+    },
+    "@quri/squiggle-lang#build": {
+      "dependsOn": ["^build"],
+      "outputs": [
+        "__tests__/**/*.bs.js",
+        "benchmark/**/*.bs.js",
+        "src/rescript/**/*.js",
+        "src/rescript/**/*.gen.tsx",
+        "../../node_modules/bisect_ppx/**/*.bs.js",
+        "dist/**"
+      ]
+    },
+    "test": {
+      "dependsOn": ["build"],
+      "outputs": []
+    },
+    "bundle": {
+      "dependsOn": ["^build", "build"]
+    },
+    "coverage": {
+      "cache": false
+    }
+  }
+}

yarn.lock

@@ -18491,6 +18491,48 @@ tunnel@0.0.6:
   resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
   integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==
 
+turbo-darwin-64@1.5.5:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/turbo-darwin-64/-/turbo-darwin-64-1.5.5.tgz#710d4e7999066bd4f500456f7cd1c30f6e6205ed"
+  integrity sha512-HvEn6P2B+NXDekq9LRpRgUjcT9/oygLTcK47U0qsAJZXRBSq/2hvD7lx4nAwgY/4W3rhYJeWtHTzbhoN6BXqGQ==
+
+turbo-darwin-arm64@1.5.5:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/turbo-darwin-arm64/-/turbo-darwin-arm64-1.5.5.tgz#f7955a800e945ab110e8a6e23f60a2b9795296ab"
+  integrity sha512-Dmxr09IUy6M0nc7/xWod9galIO2DD500B75sJSkHeT+CCdJOWnlinux0ZPF8CSygNqymwYO8AO2l15/6yxcycg==
+
+turbo-linux-64@1.5.5:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/turbo-linux-64/-/turbo-linux-64-1.5.5.tgz#f31eb117a9b605f5731048c50473bff903850047"
+  integrity sha512-wd07TZ4zXXWjzZE00FcFMLmkybQQK/NV9ff66vvAV0vdiuacSMBCNLrD6Mm4ncfrUPW/rwFW5kU/7hyuEqqtDw==
+
+turbo-linux-arm64@1.5.5:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/turbo-linux-arm64/-/turbo-linux-arm64-1.5.5.tgz#b9ce6912ae6477e829355d6f012500bfef58669d"
+  integrity sha512-q3q33tuo74R7gicnfvFbnZZvqmlq7Vakcvx0eshifnJw4PR+oMnTCb4w8ElVFx070zsb8DVTibq99y8NJH8T1Q==
+
+turbo-windows-64@1.5.5:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/turbo-windows-64/-/turbo-windows-64-1.5.5.tgz#609098de3bc6178f733615d21b06d5c1602637eb"
+  integrity sha512-lPp9kHonNFfqgovbaW+UAPO5cLmoAN+m3G3FzqcrRPnlzt97vXYsDhDd/4Zy3oAKoAcprtP4CGy0ddisqsKTVw==
+
+turbo-windows-arm64@1.5.5:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/turbo-windows-arm64/-/turbo-windows-arm64-1.5.5.tgz#60522e1e347a54c64bdddb68089fc322ee19c3d7"
+  integrity sha512-3AfGULKNZiZVrEzsIE+W79ZRW1+f5r4nM4wLlJ1PTBHyRxBZdD6KTH1tijGfy/uTlcV5acYnKHEkDc6Q9PAXGQ==
+
+turbo@^1.5.5:
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/turbo/-/turbo-1.5.5.tgz#9fc3a917c914ffa113c260a4eadb4bc632eee227"
+  integrity sha512-PVQSDl0STC9WXIyHcYUWs9gXsf8JjQig/FuHfuB8N6+XlgCGB3mPbfMEE6zrChGz2hufH4/guKRX1XJuNL6XTA==
+  optionalDependencies:
+    turbo-darwin-64 "1.5.5"
+    turbo-darwin-arm64 "1.5.5"
+    turbo-linux-64 "1.5.5"
+    turbo-linux-arm64 "1.5.5"
+    turbo-windows-64 "1.5.5"
+    turbo-windows-arm64 "1.5.5"
+
 type-check@^0.4.0, type-check@~0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"