flatten vendor dirs and simplify build-vendor

+ replace the inconvenient unicode symbol with ASCII `->` and flatten dirs by default to simplify writing the rules and improve their readability
+ rename and sort functions in the order they run
+ use `node-fetch` instead of the gargantuan `make-fetch-happen`
+ use `glob` which is already installed by other deps

manage/manage.js

@@ -144,7 +144,7 @@ async function toggleInjectionOrder(state) {
   const shown = $('.injection-order');
   if (state && !shown) {
     await require([
-      '/vendor/@eight04/draggable-list/dist/draggable-list.iife.min.js',
+      '/vendor/draggable-list/draggable-list.iife.min.js',
       '/injection-order/injection-order.css',
       '/injection-order/injection-order', /* global InjectionOrder */
     ]);

package.json

@@ -22,12 +22,11 @@
   },
   "devDependencies": {
     "archiver": "^4.0.1",
-    "endent": "^1.4.0",
     "eslint": "^7.20.0",
     "fs-extra": "^9.0.0",
-    "make-fetch-happen": "^8.0.7",
+    "glob": "^7.2.0",
+    "node-fetch": "^2.6.6",
     "sync-version": "^1.0.1",
-    "tiny-glob": "^0.2.6",
     "web-ext": "^6.5.0"
   },
   "scripts": {

tools/build-vendor.js

@@ -1,14 +1,16 @@
-/* eslint-env node */
 'use strict';
+
+const fetch = require('node-fetch');
+const fs = require('fs');
+const fse = require('fs-extra');
+const glob = require('glob').sync;
 const path = require('path');
 
-const endent = require('endent');
-const fetch = require('make-fetch-happen');
-const fse = require('fs-extra');
-const glob = require('tiny-glob');
+const KEEP_DIRECTORIES = null;
 
 const files = {
   'codemirror': [
+    KEEP_DIRECTORIES,
     'addon/comment/comment.js',
     'addon/dialog',
     'addon/edit/closebrackets.js',
@@ -35,30 +37,30 @@ const files = {
     'theme/*',
   ],
   'jsonlint': [
-    'lib/jsonlint.js → jsonlint.js',
-    'README.md → LICENSE',
+    'lib/jsonlint.js',
+    'README.md -> LICENSE',
   ],
   'less-bundle': [
-    'dist/less.min.js → less.min.js',
+    'dist/less.min.js',
   ],
   'lz-string-unsafe': [
     'lz-string-unsafe.min.js',
   ],
   'stylelint-bundle': [
-    'dist/stylelint-bundle.min.js → stylelint-bundle.min.js',
-    'https://github.com/stylelint/stylelint/raw/{VERSION}/LICENSE → LICENSE',
+    'dist/stylelint-bundle.min.js',
+    'https://github.com/stylelint/stylelint/raw/{VERSION}/LICENSE',
   ],
   'stylus-lang-bundle': [
-    'dist/stylus-renderer.min.js → stylus-renderer.min.js',
+    'dist/stylus-renderer.min.js',
   ],
   'usercss-meta': [
-    'dist/usercss-meta.min.js → usercss-meta.min.js',
+    'dist/usercss-meta.min.js',
   ],
   'db-to-cloud': [
-    'dist/db-to-cloud.min.js → db-to-cloud.min.js',
+    'dist/db-to-cloud.min.js',
   ],
   'webext-launch-web-auth-flow': [
-    'dist/webext-launch-web-auth-flow.min.js → webext-launch-web-auth-flow.min.js',
+    'dist/webext-launch-web-auth-flow.min.js',
   ],
   '@eight04/draggable-list': [
     'dist/draggable-list.iife.min.js',
@@ -69,96 +71,97 @@ main().catch(console.error);
 
 async function main() {
   fse.emptyDirSync('vendor');
-  for (const pkg in files) {
-    console.log('\x1b[32m%s\x1b[0m', `Building ${pkg}...`);
-    // other files
-    const [fetched, copied] = await buildFiles(pkg, files[pkg]);
-    // README
-    await fse.outputFile(`vendor/${pkg}/README.md`, generateReadme(pkg, fetched, copied));
-    // LICENSE
-    await copyLicense(pkg);
-  }
-  console.log('\x1b[32m%s\x1b[0m', 'updating codemirror themes list...');
-  await fse.outputFile('edit/codemirror-themes.js', await generateThemeList());
+  await Promise.all(Object.keys(files).map(async pkg => {
+    console.log(`Building ${pkg}...`);
+    const pkgName = getFileName(pkg);
+    const flatPkg = pkg === pkgName || files[pkgName]
+      ? pkg.replace(/\//g, '-')
+      : pkgName;
+    const res = await buildFiles(pkg, flatPkg, files[pkg]);
+    buildLicense(pkg, flatPkg);
+    buildReadme(pkg, flatPkg, res);
+  }));
+  console.log('Building CodeMirror theme list...');
+  buildThemeList();
 }
 
-async function generateThemeList() {
-  const themes = (await fse.readdir('vendor/codemirror/theme'))
-    .filter(name => name.endsWith('.css'))
-    .map(name => name.replace('.css', ''))
-    .sort();
-  return endent`
+async function buildFiles(pkg, flatPkg, patterns) {
+  const keepDirs = patterns.includes(KEEP_DIRECTORIES);
+  let fetched = '';
+  let copied = '';
+  for (let pattern of patterns) {
+    if (pattern === KEEP_DIRECTORIES) continue;
+    pattern = pattern.replace('{VERSION}', require(`${pkg}/package.json`).version);
+    const [src, dest = !keepDirs && getFileName(src)] = pattern.split(/\s*->\s*/);
+    if (/^https?:/.test(src)) {
+      fse.outputFileSync(`vendor/${flatPkg}/${dest}`, await (await fetch(src)).text());
+      fetched += `* ${dest}: ${src}\n`;
+    } else {
+      const files = glob(`node_modules/${pkg}/${src}`);
+      if (!files.length) {
+        throw new Error(`Pattern ${src} matches no files`);
+      }
+      for (const file of files) {
+        fse.copySync(file, dest
+          ? `vendor/${flatPkg}/${dest}`
+          : `vendor/${path.relative('node_modules', file).replace(pkg + '/', flatPkg + '/')}`);
+        const relSrc = path.relative(`node_modules/${pkg}`, file).replace(/\\/g, '/');
+        copied += dest && dest !== relSrc
+          ? `* ${dest}: ${
+            keepDirs || getFileName(dest) !== getFileName(relSrc)
+              ? relSrc
+              : relSrc.replace(/[^/]+$/, '*')
+          }\n`
+          : `* ${relSrc}\n`;
+      }
+    }
+  }
+  return {fetched, copied};
+}
+
+function buildLicense(pkg, flatPkg) {
+  const LICENSE = `vendor/${flatPkg}/LICENSE`;
+  if (!fs.existsSync(LICENSE)) {
+    const [src] = glob(`node_modules/${pkg}/LICEN[SC]E*`);
+    if (!src) throw new Error(`Cannot find license file for ${pkg}`);
+    fse.copySync(src, LICENSE);
+  }
+}
+
+function buildReadme(pkg, flatPkg, {fetched, copied}) {
+  const {name, version} = require(`${pkg}/package.json`);
+  fse.outputFileSync(`vendor/${flatPkg}/README.md`, [
+    `## ${name} v${version}`,
+    fetched && `Files downloaded from URL:\n${fetched}`,
+    copied && `Files copied from NPM (node_modules):\n${copied}`,
+  ].filter(Boolean).join('\n\n'));
+}
+
+function buildThemeList() {
+  fse.outputFileSync('edit/codemirror-themes.js', deindent(`\
     /* Do not edit. This file is auto-generated by build-vendor.js */
     'use strict';
 
     /* exported CODEMIRROR_THEMES */
     const CODEMIRROR_THEMES = [
     ${
-      themes.map(t => `  '${t.replace(/'/g, '\\$&')}',\n`).join('')
-    }];
-  ` + '\n';
-}
-
-async function copyLicense(pkg) {
-  try {
-    await fse.access(`vendor/${pkg}/LICENSE`);
-    return;
-  } catch (err) {
-    // pass
-  }
-  for (const file of await glob(`node_modules/${pkg}/LICEN[SC]E*`)) {
-    await fse.copy(file, `vendor/${pkg}/LICENSE`);
-    return;
-  }
-  throw new Error(`cannot find license file for ${pkg}`);
-}
-
-async function buildFiles(pkg, patterns) {
-  const fetchedFiles = [];
-  const copiedFiles = [];
-  for (let pattern of patterns) {
-    pattern = pattern.replace('{VERSION}', require(`${pkg}/package.json`).version);
-    const [src, dest] = pattern.split(/\s*→\s*/);
-    if (src.startsWith('http')) {
-      const content = await (await fetch(src)).text();
-      await fse.outputFile(`vendor/${pkg}/${dest}`, content);
-      fetchedFiles.push([src, dest]);
-    } else {
-      let dirty = false;
-      for (const file of await glob(`node_modules/${pkg}/${src}`)) {
-        if (dest) {
-          await fse.copy(file, `vendor/${pkg}/${dest}`);
-        } else {
-          await fse.copy(file, path.join('vendor', path.relative('node_modules', file)));
-        }
-        copiedFiles.push([path.relative(`node_modules/${pkg}`, file), dest]);
-        dirty = true;
-      }
-      if (!dirty) {
-        throw new Error(`Pattern ${src} matches no files`);
-      }
+    fs.readdirSync('vendor/codemirror/theme')
+      .filter(name => name.endsWith('.css'))
+      .map(name => name.replace('.css', ''))
+      .sort()
+      .map(t => `  '${t.replace(/'/g, '\\$&')}',`).join('\n')
     }
-  }
-  return [fetchedFiles, copiedFiles];
+    ];
+    `));
 }
 
-function generateReadme(lib, fetched, copied) {
-  const pkg = require(`${lib}/package.json`);
-  let txt = `## ${pkg.name} v${pkg.version}\n\n`;
-  if (fetched.length) {
-    txt += `Following files are downloaded from HTTP:\n\n${generateList(fetched)}\n\n`;
-  }
-  if (copied.length) {
-    txt += `Following files are copied from npm (node_modules):\n\n${generateList(copied)}\n`;
-  }
-  return txt;
+function deindent(str) {
+  const indent = str.match(/^\s*/)[0];
+  return indent
+    ? str.replace(new RegExp('^' + indent, 'gm'), '')
+    : str;
 }
 
-function generateList(list) {
-  return list.map(([src, dest]) => {
-    if (dest) {
-      return `* ${dest}: ${src}`;
-    }
-    return `* ${src}`;
-  }).join('\n');
+function getFileName(path) {
+  return path.split('/').pop();
 }

vendor/@eight04/draggable-list/README.md

@@ -1,5 +0,0 @@
-## @eight04/draggable-list v0.3.0
-
-Following files are copied from npm (node_modules):
-
-* dist\draggable-list.iife.min.js

vendor/codemirror/README.md

@@ -1,99 +1,98 @@
 ## codemirror v5.63.3
 
-Following files are copied from npm (node_modules):
-
-* addon\comment\comment.js
-* addon\dialog
-* addon\edit\closebrackets.js
-* addon\edit\matchbrackets.js
-* addon\fold\brace-fold.js
-* addon\fold\comment-fold.js
-* addon\fold\foldcode.js
-* addon\fold\foldgutter.css
-* addon\fold\foldgutter.js
-* addon\fold\indent-fold.js
-* addon\hint\css-hint.js
-* addon\hint\show-hint.css
-* addon\hint\show-hint.js
-* addon\lint\css-lint.js
-* addon\lint\json-lint.js
-* addon\lint\lint.css
-* addon\lint\lint.js
-* addon\scroll\annotatescrollbar.js
-* addon\search\matchesonscrollbar.css
-* addon\search\matchesonscrollbar.js
-* addon\search\searchcursor.js
-* addon\selection\active-line.js
-* keymap\emacs.js
-* keymap\sublime.js
-* keymap\vim.js
-* lib\codemirror.css
-* lib\codemirror.js
-* mode\css
-* mode\javascript
-* mode\stylus
-* theme\3024-day.css
-* theme\3024-night.css
-* theme\abbott.css
-* theme\abcdef.css
-* theme\ambiance-mobile.css
-* theme\ambiance.css
-* theme\ayu-dark.css
-* theme\ayu-mirage.css
-* theme\base16-dark.css
-* theme\base16-light.css
-* theme\bespin.css
-* theme\blackboard.css
-* theme\cobalt.css
-* theme\colorforth.css
-* theme\darcula.css
-* theme\dracula.css
-* theme\duotone-dark.css
-* theme\duotone-light.css
-* theme\eclipse.css
-* theme\elegant.css
-* theme\erlang-dark.css
-* theme\gruvbox-dark.css
-* theme\hopscotch.css
-* theme\icecoder.css
-* theme\idea.css
-* theme\isotope.css
-* theme\juejin.css
-* theme\lesser-dark.css
-* theme\liquibyte.css
-* theme\lucario.css
-* theme\material-darker.css
-* theme\material-ocean.css
-* theme\material-palenight.css
-* theme\material.css
-* theme\mbo.css
-* theme\mdn-like.css
-* theme\midnight.css
-* theme\monokai.css
-* theme\moxer.css
-* theme\neat.css
-* theme\neo.css
-* theme\night.css
-* theme\nord.css
-* theme\oceanic-next.css
-* theme\panda-syntax.css
-* theme\paraiso-dark.css
-* theme\paraiso-light.css
-* theme\pastel-on-dark.css
-* theme\railscasts.css
-* theme\rubyblue.css
-* theme\seti.css
-* theme\shadowfox.css
-* theme\solarized.css
-* theme\ssms.css
-* theme\the-matrix.css
-* theme\tomorrow-night-bright.css
-* theme\tomorrow-night-eighties.css
-* theme\ttcn.css
-* theme\twilight.css
-* theme\vibrant-ink.css
-* theme\xq-dark.css
-* theme\xq-light.css
-* theme\yeti.css
-* theme\yonce.css
-* theme\zenburn.css
+Files copied from NPM (node_modules):
+* addon/comment/comment.js
+* addon/dialog
+* addon/edit/closebrackets.js
+* addon/edit/matchbrackets.js
+* addon/fold/brace-fold.js
+* addon/fold/comment-fold.js
+* addon/fold/foldcode.js
+* addon/fold/foldgutter.css
+* addon/fold/foldgutter.js
+* addon/fold/indent-fold.js
+* addon/hint/css-hint.js
+* addon/hint/show-hint.css
+* addon/hint/show-hint.js
+* addon/lint/css-lint.js
+* addon/lint/json-lint.js
+* addon/lint/lint.css
+* addon/lint/lint.js
+* addon/scroll/annotatescrollbar.js
+* addon/search/matchesonscrollbar.css
+* addon/search/matchesonscrollbar.js
+* addon/search/searchcursor.js
+* addon/selection/active-line.js
+* keymap/emacs.js
+* keymap/sublime.js
+* keymap/vim.js
+* lib/codemirror.css
+* lib/codemirror.js
+* mode/css
+* mode/javascript
+* mode/stylus
+* theme/3024-day.css
+* theme/3024-night.css
+* theme/abbott.css
+* theme/abcdef.css
+* theme/ambiance-mobile.css
+* theme/ambiance.css
+* theme/ayu-dark.css
+* theme/ayu-mirage.css
+* theme/base16-dark.css
+* theme/base16-light.css
+* theme/bespin.css
+* theme/blackboard.css
+* theme/cobalt.css
+* theme/colorforth.css
+* theme/darcula.css
+* theme/dracula.css
+* theme/duotone-dark.css
+* theme/duotone-light.css
+* theme/eclipse.css
+* theme/elegant.css
+* theme/erlang-dark.css
+* theme/gruvbox-dark.css
+* theme/hopscotch.css
+* theme/icecoder.css
+* theme/idea.css
+* theme/isotope.css
+* theme/juejin.css
+* theme/lesser-dark.css
+* theme/liquibyte.css
+* theme/lucario.css
+* theme/material-darker.css
+* theme/material-ocean.css
+* theme/material-palenight.css
+* theme/material.css
+* theme/mbo.css
+* theme/mdn-like.css
+* theme/midnight.css
+* theme/monokai.css
+* theme/moxer.css
+* theme/neat.css
+* theme/neo.css
+* theme/night.css
+* theme/nord.css
+* theme/oceanic-next.css
+* theme/panda-syntax.css
+* theme/paraiso-dark.css
+* theme/paraiso-light.css
+* theme/pastel-on-dark.css
+* theme/railscasts.css
+* theme/rubyblue.css
+* theme/seti.css
+* theme/shadowfox.css
+* theme/solarized.css
+* theme/ssms.css
+* theme/the-matrix.css
+* theme/tomorrow-night-bright.css
+* theme/tomorrow-night-eighties.css
+* theme/ttcn.css
+* theme/twilight.css
+* theme/vibrant-ink.css
+* theme/xq-dark.css
+* theme/xq-light.css
+* theme/yeti.css
+* theme/yonce.css
+* theme/zenburn.css

vendor/db-to-cloud/README.md

@@ -1,5 +1,4 @@
 ## db-to-cloud v0.7.0
 
-Following files are copied from npm (node_modules):
-
-* db-to-cloud.min.js: dist\db-to-cloud.min.js
+Files copied from NPM (node_modules):
+* db-to-cloud.min.js: dist/*

vendor/draggable-list/README.md

@@ -0,0 +1,4 @@
+## @eight04/draggable-list v0.3.0
+
+Files copied from NPM (node_modules):
+* draggable-list.iife.min.js: dist/*

vendor/jsonlint/README.md

@@ -1,6 +1,5 @@
 ## jsonlint v1.6.3
 
-Following files are copied from npm (node_modules):
-
-* jsonlint.js: lib\jsonlint.js
+Files copied from NPM (node_modules):
+* jsonlint.js: lib/*
 * LICENSE: README.md

vendor/less-bundle/README.md

@@ -1,5 +1,4 @@
 ## less-bundle v0.1.0
 
-Following files are copied from npm (node_modules):
-
-* less.min.js: dist\less.min.js
+Files copied from NPM (node_modules):
+* less.min.js: dist/*

vendor/lz-string-unsafe/README.md

@@ -1,5 +1,4 @@
 ## lz-string-unsafe v1.4.4-fork-1
 
-Following files are copied from npm (node_modules):
-
+Files copied from NPM (node_modules):
 * lz-string-unsafe.min.js

vendor/stylelint-bundle/README.md

@@ -1,9 +1,8 @@
 ## stylelint-bundle v13.8.0
 
-Following files are downloaded from HTTP:
-
+Files downloaded from URL:
 * LICENSE: https://github.com/stylelint/stylelint/raw/13.8.0/LICENSE
 
-Following files are copied from npm (node_modules):
 
-* stylelint-bundle.min.js: dist\stylelint-bundle.min.js
+Files copied from NPM (node_modules):
+* stylelint-bundle.min.js: dist/*

vendor/stylus-lang-bundle/README.md

@@ -1,5 +1,4 @@
 ## stylus-lang-bundle v0.54.7
 
-Following files are copied from npm (node_modules):
-
-* stylus-renderer.min.js: dist\stylus-renderer.min.js
+Files copied from NPM (node_modules):
+* stylus-renderer.min.js: dist/*

vendor/usercss-meta/README.md

@@ -1,5 +1,4 @@
 ## usercss-meta v0.12.0
 
-Following files are copied from npm (node_modules):
-
-* usercss-meta.min.js: dist\usercss-meta.min.js
+Files copied from NPM (node_modules):
+* usercss-meta.min.js: dist/*

vendor/webext-launch-web-auth-flow/README.md

@@ -1,5 +1,4 @@
 ## webext-launch-web-auth-flow v0.1.1
 
-Following files are copied from npm (node_modules):
-
-* webext-launch-web-auth-flow.min.js: dist\webext-launch-web-auth-flow.min.js
+Files copied from NPM (node_modules):
+* webext-launch-web-auth-flow.min.js: dist/*