find @-moz-doc sections faster in the editor (#786)

* find @-moz-doc sections faster in the editor

* only recreate widgets if section data is changed

* CodeMirror speedup: reuse the old folding marks

* add a reminder to remove the CodeMirror hack in the future

* use precise getTokenAt

* check doc type for string/comment to be more mode-agnostic

* fix setGutterMarker hack

* fix skipSpace: EOL is a space too

* move deepEqual next to deepCopy

* fix getTokenTypeAt check for some cases

* remove the unnecessary \s*

edit/applies-to-line-widget.js

@@ -1,10 +1,11 @@
 /* global regExpTester debounce messageBox CodeMirror template colorMimicry msg
-  $ $create t prefs tryCatch */
+  $ $create t prefs tryCatch deepEqual */
 /* exported createAppliesToLineWidget */
 'use strict';
 
 function createAppliesToLineWidget(cm) {
   const THROTTLE_DELAY = 400;
+  const RX_SPACE = /(?:\s+|\/\*)+/y;
   let TPL, EVENTS, CLICK_ROUTE;
   let widgets = [];
   let fromLine, toLine, actualStyle;
@@ -294,21 +295,15 @@ function createAppliesToLineWidget(cm) {
     const toPos = {line: widgets[j] ? widgets[j].line.lineNo() : toLine + 1, ch: 0};
 
     // calc index->pos lookup table
-    let line = 0;
     let index = 0;
-    let fromIndex, toIndex;
+    const lineIndexes = [0];
-    const lineIndexes = [index];
+    cm.doc.iter(0, toPos.line + 1, ({text}) => {
-    cm.doc.iter(({text}) => {
-      fromIndex = line === fromPos.line ? index : fromIndex;
       lineIndexes.push((index += text.length + 1));
-      line++;
-      toIndex = line >= toPos.line ? index : toIndex;
-      return toIndex;
     });
 
     // splice
     i = Math.max(0, i);
-    widgets.splice(i, 0, ...createWidgets(fromIndex, toIndex, widgets.splice(i, j - i), lineIndexes));
+    widgets.splice(i, 0, ...createWidgets(fromPos, toPos, widgets.splice(i, j - i), lineIndexes));
 
     fromLine = null;
     toLine = null;
@@ -317,12 +312,17 @@ function createAppliesToLineWidget(cm) {
   function *createWidgets(start, end, removed, lineIndexes) {
     let i = 0;
     let itemHeight;
-    for (const section of findAppliesTo(start, end)) {
+    for (const section of findAppliesTo(start, end, lineIndexes)) {
       let removedWidget = removed[i];
       while (removedWidget && removedWidget.line.lineNo() < section.pos.line) {
         clearWidget(removed[i]);
         removedWidget = removed[++i];
       }
+      if (removedWidget && deepEqual(removedWidget.node.__applies, section.applies, ['mark'])) {
+        yield removedWidget;
+        i++;
+        continue;
+      }
       for (const a of section.applies) {
         setupApplyMarkers(a, lineIndexes);
       }
@@ -488,40 +488,84 @@ function createAppliesToLineWidget(cm) {
     };
   }
 
-  function *findAppliesTo(posStart, posEnd) {
+  function *findAppliesTo(posStart, posEnd, lineIndexes) {
-    const text = cm.getValue();
+    const funcRe = /^(url|url-prefix|domain|regexp)$/i;
-    const re = /^[\t ]*@-moz-document[\s\n]+/gm;
+    let pos;
-    const applyRe = new RegExp([
+    const eatToken = sticky => {
-      /(?:\/\*[\s\S]*?(?:\*\/\s*|$))*/,
+      if (!sticky) skipSpace(pos, posEnd);
-      /(url|url-prefix|domain|regexp)/,
+      pos.ch++;
-      /\(((['"])(?:\\\\|\\\n|\\\3|[^\n])*?\3|[^)\n]*)\)\s*(,\s*)?/,
+      const token = cm.getTokenAt(pos, true);
-    ].map(rx => rx.source).join(''), 'giy');
+      pos.ch = token.end;
-    let match;
+      return CodeMirror.cmpPos(pos, posEnd) <= 0 ? token : {};
-    re.lastIndex = posStart;
+    };
-    while ((match = re.exec(text))) {
+    const docCur = cm.getSearchCursor('@-moz-document', posStart);
-      if (match.index >= posEnd) {
+    while (docCur.findNext() &&
-        return;
+           CodeMirror.cmpPos(docCur.pos.to, posEnd) <= 0) {
-      }
+      // CM can be nitpicky at token boundary so we'll check the next character
+      const safePos = {line: docCur.pos.from.line, ch: docCur.pos.from.ch + 1};
+      if (/\b(string|comment)\b/.test(cm.getTokenTypeAt(safePos))) continue;
       const applies = [];
-      let m;
+      pos = docCur.pos.to;
-      applyRe.lastIndex = re.lastIndex;
+      do {
-      while ((m = applyRe.exec(text))) {
+        skipSpace(pos, posEnd);
+        const funcIndex = lineIndexes[pos.line] + pos.ch;
+        const func = eatToken().string;
+        // no space allowed before the opening parenthesis
+        if (!funcRe.test(func) || eatToken(true).string !== '(') break;
+        const url = eatToken();
+        if (url.type !== 'string' || eatToken().string !== ')') break;
+        const unquotedUrl = unquote(url.string);
         const apply = createApply(
-          m.index,
+          funcIndex,
-          m[1],
+          func,
-          unquote(m[2]),
+          unquotedUrl,
-          unquote(m[2]) !== m[2]
+          unquotedUrl !== url.string
         );
         applies.push(apply);
-        re.lastIndex = applyRe.lastIndex;
+      } while (eatToken().string === ',');
-      }
       yield {
-        pos: cm.posFromIndex(match.index),
+        pos: docCur.pos.from,
         applies
       };
     }
   }
 
+  function skipSpace(pos, posEnd) {
+    let {ch, line} = pos;
+    let lookForEnd;
+    line--;
+    cm.doc.iter(pos.line, posEnd.line + 1, ({text}) => {
+      line++;
+      while (true) {
+        if (lookForEnd) {
+          ch = text.indexOf('*/', ch) + 1;
+          if (!ch) {
+            return;
+          }
+          ch++;
+          lookForEnd = false;
+        }
+        // EOL is a whitespace so we'll check the next line
+        if (ch >= text.length) {
+          ch = 0;
+          return;
+        }
+        RX_SPACE.lastIndex = ch;
+        const m = RX_SPACE.exec(text);
+        if (!m) {
+          return true;
+        }
+        ch += m[0].length;
+        lookForEnd = m[0].includes('/*');
+        if (ch < text.length && !lookForEnd) {
+          return true;
+        }
+      }
+    });
+    pos.line = line;
+    pos.ch = ch;
+  }
+
   function unquote(s) {
     const first = s.charAt(0);
     return (first === '"' || first === "'") && s.endsWith(first) ? s.slice(1, -1) : s;

edit/codemirror-default.js

@@ -242,6 +242,23 @@
     CodeMirror.commands[name] = (...args) => editor[name](...args);
   }
 
+  // speedup: reuse the old folding marks
+  // TODO: remove when https://github.com/codemirror/CodeMirror/pull/6010 is shipped in /vendor
+  const {setGutterMarker} = CodeMirror.prototype;
+  CodeMirror.prototype.setGutterMarker = function (line, gutterID, value) {
+    const o = this.state.foldGutter.options;
+    if (typeof o.indicatorOpen === 'string' ||
+        typeof o.indicatorFolded === 'string') {
+      const old = line.gutterMarkers && line.gutterMarkers[gutterID];
+      // old className can contain other names set by CodeMirror so we'll use classList
+      if (old && value && old.classList.contains(value.className) ||
+          !old && !value) {
+        return line;
+      }
+    }
+    return setGutterMarker.apply(this, arguments);
+  };
+
   // CodeMirror convenience commands
   Object.assign(CodeMirror.commands, {
     toggleEditorFocus,

js/messaging.js

@@ -1,5 +1,5 @@
 /* exported getActiveTab onTabReady stringAsRegExp getTabRealURL openURL
-  getStyleWithNoCode tryRegExp sessionStorageHash download
+  getStyleWithNoCode tryRegExp sessionStorageHash download deepEqual
   closeCurrentTab */
 'use strict';
 
@@ -360,6 +360,31 @@ function deepCopy(obj) {
 }
 
 
+function deepEqual(a, b, ignoredKeys) {
+  if (!a || !b) return a === b;
+  const type = typeof a;
+  if (type !== typeof b) return false;
+  if (type !== 'object') return a === b;
+  if (Array.isArray(a)) {
+    return Array.isArray(b) &&
+           a.length === b.length &&
+           a.every((v, i) => deepEqual(v, b[i], ignoredKeys));
+  }
+  for (const key in a) {
+    if (!Object.hasOwnProperty.call(a, key) ||
+        ignoredKeys && ignoredKeys.includes(key)) continue;
+    if (!Object.hasOwnProperty.call(b, key)) return false;
+    if (!deepEqual(a[key], b[key], ignoredKeys)) return false;
+  }
+  for (const key in b) {
+    if (!Object.hasOwnProperty.call(b, key) ||
+        ignoredKeys && ignoredKeys.includes(key)) continue;
+    if (!Object.hasOwnProperty.call(a, key)) return false;
+  }
+  return true;
+}
+
+
 function sessionStorageHash(name) {
   return {
     name,