convert USO styles to USO-archive on update

background/style-manager.js

@@ -47,7 +47,7 @@ const styleMan = (() => {
     _id: () => uuidv4(),
     _rev: () => Date.now(),
   };
-  const DELETE_IF_NULL = ['id', 'customName'];
+  const DELETE_IF_NULL = ['id', 'customName', 'md5Url', 'originalMd5'];
   /** @type {Promise|boolean} will be `true` to avoid wasting a microtask tick on each `await` */
   let ready = init();
 

background/style-search-db.js

@@ -1,5 +1,5 @@
 /* global API */// msg.js
-/* global URLS debounce stringAsRegExp tryRegExp */// toolbox.js
+/* global RX_META debounce stringAsRegExp tryRegExp */// toolbox.js
 /* global addAPI */// common.js
 'use strict';
 
@@ -10,12 +10,12 @@
 
   const extractMeta = style =>
     style.usercssData
-      ? (style.sourceCode.match(URLS.rxMETA) || [''])[0]
+      ? (style.sourceCode.match(RX_META) || [''])[0]
       : null;
 
   const stripMeta = style =>
     style.usercssData
-      ? style.sourceCode.replace(URLS.rxMETA, '')
+      ? style.sourceCode.replace(RX_META, '')
       : null;
 
   const MODES = Object.assign(Object.create(null), {

background/update-manager.js

@@ -1,7 +1,8 @@
 /* global API */// msg.js
-/* global URLS debounce download ignoreChromeError */// toolbox.js
+/* global RX_META URLS debounce download ignoreChromeError */// toolbox.js
 /* global calcStyleDigest styleJSONseemsValid styleSectionsEqual */ // sections-util.js
 /* global chromeLocal */// storage-util.js
+/* global db */
 /* global prefs */
 'use strict';
 
@@ -21,7 +22,14 @@ const updateMan = (() => {
     ERROR_JSON:    'error: JSON is invalid',
     ERROR_VERSION: 'error: version is older than installed style',
   };
-
+  const RH_ETAG = {responseHeaders: ['etag']}; // a hashsum of file contents
+  const RX_DATE2VER = new RegExp([
+    /^(\d{4})/,
+    /(1(?:0|[12](?=\d\d))?|[2-9])/, // in ambiguous cases like yyyy123 the month will be 1
+    /([1-2][0-9]?|3[0-1]?|[4-9])/,
+    /\.(0|1[0-9]?|2[0-3]?|[3-9])/,
+    /\.(0|[1-5][0-9]?|[6-9])$/,
+  ].map(rx => rx.source).join(''));
   const ALARM_NAME = 'scheduledUpdate';
   const MIN_INTERVAL_MS = 60e3;
   const RETRY_ERRORS = [
@@ -96,13 +104,14 @@ const updateMan = (() => {
    'ignoreDigest' option is set on the second manual individual update check on the manage page.
    */
   async function checkStyle(opts) {
+    let {id} = opts;
     const {
-      id,
       style = await API.styles.get(id),
       ignoreDigest,
       port,
       save,
     } = opts;
+    if (!id) id = style.id;
     const ucd = style.usercssData;
     let res, state;
     try {
@@ -119,7 +128,7 @@ const updateMan = (() => {
       res = {error, style, STATES};
       state = `${STATES.SKIPPED} (${error})`;
     }
-    log(`${state} #${style.id} ${style.customName || style.name}`);
+    log(`${state} #${id} ${style.customName || style.name}`);
     if (port) port.postMessage(res);
     return res;
 
@@ -132,6 +141,11 @@ const updateMan = (() => {
     }
 
     async function updateUSO() {
+      const url = URLS.makeUsoArchiveCodeUrl(style.md5Url.match(/\d+/)[0]);
+      const req = await tryDownload(url, RH_ETAG).catch(() => null);
+      if (req) {
+        return updateToUSOArchive(url, req);
+      }
       const md5 = await tryDownload(style.md5Url);
       if (!md5 || md5.length !== 32) {
         return Promise.reject(STATES.ERROR_MD5);
@@ -148,33 +162,82 @@ const updateMan = (() => {
       return json;
     }
 
-    async function updateUsercss() {
+    async function updateToUSOArchive(url, req) {
-      // TODO: when sourceCode is > 100kB use http range request(s) for version check
+      // UserCSS metadata may be embedded in the original USO style so let's use its updateURL
-      const url = style.updateUrl;
+      const [meta2] = req.response.replace(RX_META, '').match(RX_META) || [];
-      const metaUrl = URLS.extractGreasyForkInstallUrl(url) &&
+      if (meta2 && meta2.includes('@updateURL')) {
-        url.replace(/\.user\.css$/, '.meta.css');
+        const {updateUrl} = await API.usercss.buildMeta({sourceCode: meta2}).catch(() => ({}));
-      const text = await tryDownload(metaUrl || url);
+        if (updateUrl) {
-      const json = await API.usercss.buildMeta({sourceCode: text});
+          url = updateUrl;
-      await require(['/vendor/semver-bundle/semver']); /* global semverCompare */
+          req = await tryDownload(url, RH_ETAG);
-      const delta = semverCompare(json.usercssData.version, ucd.version);
+        }
-      if (!delta && !ignoreDigest) {
-        // re-install is invalid in a soft upgrade
-        const sameCode = !metaUrl && text === style.sourceCode;
-        return Promise.reject(sameCode ? STATES.SAME_CODE : STATES.SAME_VERSION);
       }
-      if (delta < 0) {
+      const json = await API.usercss.buildMeta({
-        // downgrade is always invalid
+        id,
-        return Promise.reject(STATES.ERROR_VERSION);
+        etag: req.headers.etag,
-      }
+        md5Url: null,
-      if (metaUrl) {
+        originalMd5: null,
-        json.sourceCode = await tryDownload(url);
+        sourceCode: req.response,
+        updateUrl: url,
+        url: URLS.extractUsoArchiveInstallUrl(url),
+      });
+      const varUrlValues = style.updateUrl.split('?')[1];
+      const varData = json.usercssData.vars;
+      if (varUrlValues && varData) {
+        const IK = 'ik-';
+        const IK_LEN = IK.length;
+        for (let [key, val] of new URLSearchParams(varUrlValues)) {
+          if (!key.startsWith(IK)) continue;
+          key = key.slice(IK_LEN);
+          const varDef = varData[key];
+          if (!varDef) continue;
+          if (varDef.options) {
+            let sel = val.startsWith(IK) && getVarOptByName(varDef, val.slice(IK_LEN));
+            if (!sel) {
+              key += '-custom';
+              sel = getVarOptByName(varDef, key + '-dropdown');
+              if (sel) varData[key].value = val;
+            }
+            if (sel) varDef.value = sel.name;
+          } else {
+            varDef.value = val;
+          }
+        }
       }
       return API.usercss.buildCode(json);
     }
 
+    async function updateUsercss() {
+      if (style.etag && style.etag === await downloadEtag()) {
+        return Promise.reject(STATES.SAME_CODE);
+      }
+      // TODO: when sourceCode is > 100kB use http range request(s) for version check
+      const {headers: {etag}, response} = await tryDownload(style.updateUrl, RH_ETAG);
+      const json = await API.usercss.buildMeta({sourceCode: response, etag});
+      await require(['/vendor/semver-bundle/semver']); /* global semverCompare */
+      const delta = semverCompare(json.usercssData.version, ucd.version);
+      let err;
+      if (!delta && !ignoreDigest) {
+        // re-install is invalid in a soft upgrade
+        err = response === style.sourceCode ? STATES.SAME_CODE : STATES.SAME_VERSION;
+      }
+      if (delta < 0) {
+        // downgrade is always invalid
+        err = STATES.ERROR_VERSION;
+      }
+      if (err && etag && !style.etag) {
+        // first check of ETAG, gonna write it directly to DB as it's too trivial to sync or announce
+        style.etag = etag;
+        await db.exec('put', style);
+      }
+      return err
+        ? Promise.reject(err)
+        : API.usercss.buildCode(json);
+    }
+
     async function maybeSave(json) {
-      json.id = style.id;
+      json.id = id;
-      json.updateDate = Date.now();
+      json.updateDate = getDateFromVer(json) || Date.now();
       // keep current state
       delete json.customName;
       delete json.enabled;
@@ -206,6 +269,25 @@ const updateMan = (() => {
         await new Promise(resolve => setTimeout(resolve, retryDelay));
       }
     }
+
+    async function downloadEtag() {
+      const opts = Object.assign({method: 'head'}, RH_ETAG);
+      const req = await tryDownload(style.updateUrl, opts);
+      return req.headers.etag;
+    }
+
+    function getDateFromVer(style) {
+      const m = style.updateUrl.startsWith(URLS.usoArchiveRaw) &&
+        style.usercssData.version.match(RX_DATE2VER);
+      if (m) {
+        m[2]--; // month is 0-based in `Date` constructor
+        return new Date(...m.slice(1)).getTime();
+      }
+    }
+
+    function getVarOptByName(varDef, name) {
+      return varDef.options.find(o => o.name === name);
+    }
   }
 
   function schedule() {

background/usercss-install-helper.js

@@ -1,4 +1,4 @@
-/* global URLS download openURL */// toolbox.js
+/* global RX_META URLS download openURL */// toolbox.js
 /* global addAPI bgReady */// common.js
 /* global tabMan */// msg.js
 'use strict';
@@ -85,7 +85,7 @@ bgReady.all.then(() => {
         !oldUrl.startsWith(URLS.installUsercss)) {
       const inTab = url.startsWith('file:') && !chrome.app;
       const code = await (inTab ? loadFromFile : loadFromUrl)(tabId, url);
-      if (!/^\s*</.test(code) && URLS.rxMETA.test(code)) {
+      if (!/^\s*</.test(code) && RX_META.test(code)) {
         openInstallerPage(tabId, url, {code, inTab});
       }
     }

background/usercss-manager.js

@@ -1,5 +1,5 @@
 /* global API */// msg.js
-/* global URLS deepCopy download */// toolbox.js
+/* global RX_META deepCopy download */// toolbox.js
 'use strict';
 
 const usercssMan = {
@@ -15,7 +15,7 @@ const usercssMan = {
   async assignVars(style, oldStyle) {
     const meta = style.usercssData;
     const vars = meta.vars;
-    const oldVars = oldStyle.usercssData.vars;
+    const oldVars = (oldStyle.usercssData || {}).vars;
     if (vars && oldVars) {
       // The type of var might be changed during the update. Set value to null if the value is invalid.
       for (const [key, v] of Object.entries(vars)) {
@@ -51,7 +51,7 @@ const usercssMan = {
 
   async buildCode(style) {
     const {sourceCode: code, usercssData: {vars, preprocessor}} = style;
-    const match = code.match(URLS.rxMETA);
+    const match = code.match(RX_META);
     const i = match.index;
     const j = i + match[0].length;
     const codeNoMeta = code.slice(0, i) + blankOut(code, i, j) + code.slice(j);
@@ -74,7 +74,7 @@ const usercssMan = {
       enabled: true,
       sections: [],
     }, style);
-    const match = code.match(URLS.rxMETA);
+    const match = code.match(RX_META);
     if (!match) {
       return Promise.reject(new Error('Could not find metadata.'));
     }

edit/sections-editor.js

@@ -1,7 +1,7 @@
 /* global $ $$ $create $remove messageBoxProxy */// dom.js
 /* global API */// msg.js
 /* global CodeMirror */
-/* global FIREFOX URLS debounce ignoreChromeError sessionStore */// toolbox.js
+/* global FIREFOX RX_META debounce ignoreChromeError sessionStore */// toolbox.js
 /* global MozDocMapper clipString helpPopup rerouteHotkeys showCodeMirrorPopup */// util.js
 /* global createSection */// sections-editor-section.js
 /* global editor */
@@ -360,7 +360,7 @@ function SectionsEditor() {
       lockPageUI(true);
       try {
         const code = popup.codebox.getValue().trim();
-        if (!URLS.rxMETA.test(code) ||
+        if (!RX_META.test(code) ||
             !await getPreprocessor(code) ||
             await messageBoxProxy.confirm(
               t('importPreprocessor'), 'pre-line',

edit/source-editor.js

@@ -4,7 +4,7 @@
 /* global MozDocMapper */// util.js
 /* global MozSectionFinder */
 /* global MozSectionWidget */
-/* global URLS debounce sessionStore */// toolbox.js
+/* global RX_META debounce sessionStore */// toolbox.js
 /* global chromeSync */// storage-util.js
 /* global cmFactory */
 /* global editor */
@@ -307,7 +307,7 @@ function SourceEditor() {
       if (_cm !== cm) {
         return;
       }
-      const match = text.match(URLS.rxMETA);
+      const match = text.match(RX_META);
       if (!match) {
         return [];
       }

js/toolbox.js

@@ -2,6 +2,7 @@
 
 /* exported
   CHROME_POPUP_BORDER_BUG
+  RX_META
   capitalize
   closeCurrentTab
   deepEqual
@@ -71,8 +72,6 @@ const URLS = {
   // TODO: remove when "minimum_chrome_version": "61" or higher
   chromeProtectsNTP: CHROME >= 61,
 
-  rxMETA: /\/\*!?\s*==userstyle==[\s\S]*?==\/userstyle==\s*\*\//i,
-
   uso: 'https://userstyles.org/',
   usoJson: 'https://userstyles.org/styles/chrome/',
 
@@ -86,6 +85,7 @@ const URLS = {
     const id = URLS.extractUsoArchiveId(url);
     return id ? `${URLS.usoArchive}?style=${id}` : '';
   },
+  makeUsoArchiveCodeUrl: id => `${URLS.usoArchiveRaw}usercss/${id}.user.css`,
 
   extractGreasyForkInstallUrl: url =>
     /^(https:\/\/(?:greasy|sleazy)fork\.org\/scripts\/\d+)[^/]*\/code\/[^/]*\.user\.css$|$/.exec(url)[1],
@@ -99,6 +99,8 @@ const URLS = {
   ),
 };
 
+const RX_META = /\/\*!?\s*==userstyle==[\s\S]*?==\/userstyle==\s*\*\//i;
+
 if (FIREFOX || OPERA || VIVALDI) {
   document.documentElement.classList.add(
     FIREFOX && 'firefox' ||
@@ -358,10 +360,11 @@ const sessionStore = new Proxy({}, {
  * @param {Object} params
  * @param {String} [params.method]
  * @param {String|Object} [params.body]
- * @param {String} [params.responseType] arraybuffer, blob, document, json, text
+ * @param {'arraybuffer'|'blob'|'document'|'json'|'text'} [params.responseType]
  * @param {Number} [params.requiredStatusCode] resolved when matches, otherwise rejected
  * @param {Number} [params.timeout] ms
  * @param {Object} [params.headers] {name: value}
+ * @param {string[]} [params.responseHeaders]
  * @returns {Promise}
  */
 function download(url, {
@@ -372,6 +375,7 @@ function download(url, {
   timeout = 60e3, // connection timeout, USO is that bad
   loadTimeout = 2 * 60e3, // data transfer timeout (counted from the first remote response)
   headers,
+  responseHeaders,
 } = {}) {
   /* USO can't handle POST requests for style json and XHR/fetch can't handle super long URL
    * so we need to collapse all long variables and expand them in the response */
@@ -404,10 +408,20 @@ function download(url, {
         timer = loadTimeout && setTimeout(onTimeout, loadTimeout);
       }
     };
-    xhr.onload = () =>
+    xhr.onload = () => {
-      xhr.status === requiredStatusCode || !requiredStatusCode || u.protocol === 'file:'
+      if (xhr.status === requiredStatusCode || !requiredStatusCode || u.protocol === 'file:') {
-        ? resolve(expandUsoVars(xhr.response))
+        const response = expandUsoVars(xhr.response);
-        : reject(xhr.status);
+        if (responseHeaders) {
+          const headers = {};
+          for (const h of responseHeaders) headers[h] = xhr.getResponseHeader(h);
+          resolve({headers, response});
+        } else {
+          resolve(response);
+        }
+      } else {
+        reject(xhr.status);
+      }
+    };
     xhr.onerror = () => reject(xhr.status);
     xhr.onloadend = () => clearTimeout(timer);
     xhr.responseType = responseType;

js/usercss-compiler.js

@@ -140,7 +140,12 @@ function simplifyUsercssVars(vars) {
       case 'dropdown':
       case 'image':
         // TODO: handle customized image
-        value = va.options.find(o => o.name === value).value;
+        for (const opt of va.options) {
+          if (opt.name === value) {
+            value = opt.value;
+            break;
+          }
+        }
         break;
       case 'number':
       case 'range':

manage/import-export.js

@@ -1,5 +1,5 @@
 /* global API */// msg.js
-/* global URLS deepEqual isEmptyObj tryJSONparse */// toolbox.js
+/* global RX_META deepEqual isEmptyObj tryJSONparse */// toolbox.js
 /* global changeQueue */// manage.js
 /* global chromeSync */// storage-util.js
 /* global prefs */
@@ -83,7 +83,7 @@ function importFromFile({fileTypeFilter, file} = {}) {
         fReader.onloadend = event => {
           fileInput.remove();
           const text = event.target.result;
-          const maybeUsercss = !/^\s*\[/.test(text) && URLS.rxMETA.test(text);
+          const maybeUsercss = !/^\s*\[/.test(text) && RX_META.test(text);
           if (maybeUsercss) {
             messageBoxProxy.alert(t('dragDropUsercssTabstrip'));
           } else {

popup/search.js

@@ -409,7 +409,7 @@
     result.pingbackTimer = setTimeout(download, PINGBACK_DELAY,
       `${URLS.uso}styles/install/${id}?source=stylish-ch`);
 
-    const updateUrl = `${URLS.usoArchiveRaw}usercss/${id}.user.css`;
+    const updateUrl = URLS.makeUsoArchiveCodeUrl(id);
     try {
       const sourceCode = await download(updateUrl);
       const style = await API.usercss.install({sourceCode, updateUrl});