From ab588d8c646c4d7d486153f2a47ab657bff80825 Mon Sep 17 00:00:00 2001
From: tophf <tophf@gmx.com>
Date: Tue, 26 Jan 2021 16:33:17 +0300
Subject: [PATCH] convert USO styles to USO-archive on update
---
background/style-manager.js | 2 +-
background/style-search-db.js | 6 +-
background/update-manager.js | 132 ++++++++++++++++++++++-----
background/usercss-install-helper.js | 4 +-
background/usercss-manager.js | 8 +-
edit/sections-editor.js | 4 +-
edit/source-editor.js | 4 +-
js/toolbox.js | 28 ++++--
js/usercss-compiler.js | 7 +-
manage/import-export.js | 4 +-
popup/search.js | 2 +-
11 files changed, 151 insertions(+), 50 deletions(-)
diff --git a/background/style-manager.js b/background/style-manager.js
index 167328fc..26406feb 100644
--- a/background/style-manager.js
+++ b/background/style-manager.js
@@ -47,7 +47,7 @@ const styleMan = (() => {
_id: () => uuidv4(),
_rev: () => Date.now(),
};
- const DELETE_IF_NULL = ['id', 'customName'];
+ const DELETE_IF_NULL = ['id', 'customName', 'md5Url', 'originalMd5'];
/** @type {Promise|boolean} will be `true` to avoid wasting a microtask tick on each `await` */
let ready = init();
diff --git a/background/style-search-db.js b/background/style-search-db.js
index ca8e6e06..23e18f76 100644
--- a/background/style-search-db.js
+++ b/background/style-search-db.js
@@ -1,5 +1,5 @@
/* global API */// msg.js
-/* global URLS debounce stringAsRegExp tryRegExp */// toolbox.js
+/* global RX_META debounce stringAsRegExp tryRegExp */// toolbox.js
/* global addAPI */// common.js
'use strict';
@@ -10,12 +10,12 @@
const extractMeta = style =>
style.usercssData
- ? (style.sourceCode.match(URLS.rxMETA) || [''])[0]
+ ? (style.sourceCode.match(RX_META) || [''])[0]
: null;
const stripMeta = style =>
style.usercssData
- ? style.sourceCode.replace(URLS.rxMETA, '')
+ ? style.sourceCode.replace(RX_META, '')
: null;
const MODES = Object.assign(Object.create(null), {
diff --git a/background/update-manager.js b/background/update-manager.js
index 72951e59..c8578987 100644
--- a/background/update-manager.js
+++ b/background/update-manager.js
@@ -1,7 +1,8 @@
/* global API */// msg.js
-/* global URLS debounce download ignoreChromeError */// toolbox.js
+/* global RX_META URLS debounce download ignoreChromeError */// toolbox.js
/* global calcStyleDigest styleJSONseemsValid styleSectionsEqual */ // sections-util.js
/* global chromeLocal */// storage-util.js
+/* global db */
/* global prefs */
'use strict';
@@ -21,7 +22,14 @@ const updateMan = (() => {
ERROR_JSON: 'error: JSON is invalid',
ERROR_VERSION: 'error: version is older than installed style',
};
-
+ const RH_ETAG = {responseHeaders: ['etag']}; // a hashsum of file contents
+ const RX_DATE2VER = new RegExp([
+ /^(\d{4})/,
+ /(1(?:0|[12](?=\d\d))?|[2-9])/, // in ambiguous cases like yyyy123 the month will be 1
+ /([1-2][0-9]?|3[0-1]?|[4-9])/,
+ /\.(0|1[0-9]?|2[0-3]?|[3-9])/,
+ /\.(0|[1-5][0-9]?|[6-9])$/,
+ ].map(rx => rx.source).join(''));
const ALARM_NAME = 'scheduledUpdate';
const MIN_INTERVAL_MS = 60e3;
const RETRY_ERRORS = [
@@ -96,13 +104,14 @@ const updateMan = (() => {
'ignoreDigest' option is set on the second manual individual update check on the manage page.
*/
async function checkStyle(opts) {
+ let {id} = opts;
const {
- id,
style = await API.styles.get(id),
ignoreDigest,
port,
save,
} = opts;
+ if (!id) id = style.id;
const ucd = style.usercssData;
let res, state;
try {
@@ -119,7 +128,7 @@ const updateMan = (() => {
res = {error, style, STATES};
state = `${STATES.SKIPPED} (${error})`;
}
- log(`${state} #${style.id} ${style.customName || style.name}`);
+ log(`${state} #${id} ${style.customName || style.name}`);
if (port) port.postMessage(res);
return res;
@@ -132,6 +141,11 @@ const updateMan = (() => {
}
async function updateUSO() {
+ const url = URLS.makeUsoArchiveCodeUrl(style.md5Url.match(/\d+/)[0]);
+ const req = await tryDownload(url, RH_ETAG).catch(() => null);
+ if (req) {
+ return updateToUSOArchive(url, req);
+ }
const md5 = await tryDownload(style.md5Url);
if (!md5 || md5.length !== 32) {
return Promise.reject(STATES.ERROR_MD5);
@@ -148,33 +162,82 @@ const updateMan = (() => {
return json;
}
- async function updateUsercss() {
- // TODO: when sourceCode is > 100kB use http range request(s) for version check
- const url = style.updateUrl;
- const metaUrl = URLS.extractGreasyForkInstallUrl(url) &&
- url.replace(/\.user\.css$/, '.meta.css');
- const text = await tryDownload(metaUrl || url);
- const json = await API.usercss.buildMeta({sourceCode: text});
- await require(['/vendor/semver-bundle/semver']); /* global semverCompare */
- const delta = semverCompare(json.usercssData.version, ucd.version);
- if (!delta && !ignoreDigest) {
- // re-install is invalid in a soft upgrade
- const sameCode = !metaUrl && text === style.sourceCode;
- return Promise.reject(sameCode ? STATES.SAME_CODE : STATES.SAME_VERSION);
+ async function updateToUSOArchive(url, req) {
+ // UserCSS metadata may be embedded in the original USO style so let's use its updateURL
+ const [meta2] = req.response.replace(RX_META, '').match(RX_META) || [];
+ if (meta2 && meta2.includes('@updateURL')) {
+ const {updateUrl} = await API.usercss.buildMeta({sourceCode: meta2}).catch(() => ({}));
+ if (updateUrl) {
+ url = updateUrl;
+ req = await tryDownload(url, RH_ETAG);
+ }
}
- if (delta < 0) {
- // downgrade is always invalid
- return Promise.reject(STATES.ERROR_VERSION);
- }
- if (metaUrl) {
- json.sourceCode = await tryDownload(url);
+ const json = await API.usercss.buildMeta({
+ id,
+ etag: req.headers.etag,
+ md5Url: null,
+ originalMd5: null,
+ sourceCode: req.response,
+ updateUrl: url,
+ url: URLS.extractUsoArchiveInstallUrl(url),
+ });
+ const varUrlValues = style.updateUrl.split('?')[1];
+ const varData = json.usercssData.vars;
+ if (varUrlValues && varData) {
+ const IK = 'ik-';
+ const IK_LEN = IK.length;
+ for (let [key, val] of new URLSearchParams(varUrlValues)) {
+ if (!key.startsWith(IK)) continue;
+ key = key.slice(IK_LEN);
+ const varDef = varData[key];
+ if (!varDef) continue;
+ if (varDef.options) {
+ let sel = val.startsWith(IK) && getVarOptByName(varDef, val.slice(IK_LEN));
+ if (!sel) {
+ key += '-custom';
+ sel = getVarOptByName(varDef, key + '-dropdown');
+ if (sel) varData[key].value = val;
+ }
+ if (sel) varDef.value = sel.name;
+ } else {
+ varDef.value = val;
+ }
+ }
}
return API.usercss.buildCode(json);
}
+ async function updateUsercss() {
+ if (style.etag && style.etag === await downloadEtag()) {
+ return Promise.reject(STATES.SAME_CODE);
+ }
+ // TODO: when sourceCode is > 100kB use http range request(s) for version check
+ const {headers: {etag}, response} = await tryDownload(style.updateUrl, RH_ETAG);
+ const json = await API.usercss.buildMeta({sourceCode: response, etag});
+ await require(['/vendor/semver-bundle/semver']); /* global semverCompare */
+ const delta = semverCompare(json.usercssData.version, ucd.version);
+ let err;
+ if (!delta && !ignoreDigest) {
+ // re-install is invalid in a soft upgrade
+ err = response === style.sourceCode ? STATES.SAME_CODE : STATES.SAME_VERSION;
+ }
+ if (delta < 0) {
+ // downgrade is always invalid
+ err = STATES.ERROR_VERSION;
+ }
+ if (err && etag && !style.etag) {
+ // first check of ETAG, gonna write it directly to DB as it's too trivial to sync or announce
+ style.etag = etag;
+ await db.exec('put', style);
+ }
+ return err
+ ? Promise.reject(err)
+ : API.usercss.buildCode(json);
+ }
+
async function maybeSave(json) {
- json.id = style.id;
- json.updateDate = Date.now();
+ json.id = id;
+ json.updateDate = getDateFromVer(json) || Date.now();
// keep current state
delete json.customName;
delete json.enabled;
@@ -206,6 +269,25 @@ const updateMan = (() => {
await new Promise(resolve => setTimeout(resolve, retryDelay));
}
}
+
+ async function downloadEtag() {
+ const opts = Object.assign({method: 'head'}, RH_ETAG);
+ const req = await tryDownload(style.updateUrl, opts);
+ return req.headers.etag;
+ }
+
+ function getDateFromVer(style) {
+ const m = style.updateUrl.startsWith(URLS.usoArchiveRaw) &&
+ style.usercssData.version.match(RX_DATE2VER);
+ if (m) {
+ m[2]--; // month is 0-based in `Date` constructor
+ return new Date(...m.slice(1)).getTime();
+ }
+ }
+
+ function getVarOptByName(varDef, name) {
+ return varDef.options.find(o => o.name === name);
+ }
}
function schedule() {
diff --git a/background/usercss-install-helper.js b/background/usercss-install-helper.js
index b33052a1..6656ac7b 100644
--- a/background/usercss-install-helper.js
+++ b/background/usercss-install-helper.js
@@ -1,4 +1,4 @@
-/* global URLS download openURL */// toolbox.js
+/* global RX_META URLS download openURL */// toolbox.js
/* global addAPI bgReady */// common.js
/* global tabMan */// msg.js
'use strict';
@@ -85,7 +85,7 @@ bgReady.all.then(() => {
!oldUrl.startsWith(URLS.installUsercss)) {
const inTab = url.startsWith('file:') && !chrome.app;
const code = await (inTab ? loadFromFile : loadFromUrl)(tabId, url);
- if (!/^\s*</.test(code) && URLS.rxMETA.test(code)) {
+ if (!/^\s*</.test(code) && RX_META.test(code)) {
openInstallerPage(tabId, url, {code, inTab});
}
}
diff --git a/background/usercss-manager.js b/background/usercss-manager.js
index 88993c1f..a1ae635c 100644
--- a/background/usercss-manager.js
+++ b/background/usercss-manager.js
@@ -1,5 +1,5 @@
/* global API */// msg.js
-/* global URLS deepCopy download */// toolbox.js
+/* global RX_META deepCopy download */// toolbox.js
'use strict';
const usercssMan = {
@@ -15,7 +15,7 @@ const usercssMan = {
async assignVars(style, oldStyle) {
const meta = style.usercssData;
const vars = meta.vars;
- const oldVars = oldStyle.usercssData.vars;
+ const oldVars = (oldStyle.usercssData || {}).vars;
if (vars && oldVars) {
// The type of var might be changed during the update. Set value to null if the value is invalid.
for (const [key, v] of Object.entries(vars)) {
@@ -51,7 +51,7 @@ const usercssMan = {
async buildCode(style) {
const {sourceCode: code, usercssData: {vars, preprocessor}} = style;
- const match = code.match(URLS.rxMETA);
+ const match = code.match(RX_META);
const i = match.index;
const j = i + match[0].length;
const codeNoMeta = code.slice(0, i) + blankOut(code, i, j) + code.slice(j);
@@ -74,7 +74,7 @@ const usercssMan = {
enabled: true,
sections: [],
}, style);
- const match = code.match(URLS.rxMETA);
+ const match = code.match(RX_META);
if (!match) {
return Promise.reject(new Error('Could not find metadata.'));
}
diff --git a/edit/sections-editor.js b/edit/sections-editor.js
index af539415..6bec4b21 100644
--- a/edit/sections-editor.js
+++ b/edit/sections-editor.js
@@ -1,7 +1,7 @@
/* global $ $$ $create $remove messageBoxProxy */// dom.js
/* global API */// msg.js
/* global CodeMirror */
-/* global FIREFOX URLS debounce ignoreChromeError sessionStore */// toolbox.js
+/* global FIREFOX RX_META debounce ignoreChromeError sessionStore */// toolbox.js
/* global MozDocMapper clipString helpPopup rerouteHotkeys showCodeMirrorPopup */// util.js
/* global createSection */// sections-editor-section.js
/* global editor */
@@ -358,7 +358,7 @@ function SectionsEditor() {
lockPageUI(true);
try {
const code = popup.codebox.getValue().trim();
- if (!URLS.rxMETA.test(code) ||
+ if (!RX_META.test(code) ||
!await getPreprocessor(code) ||
await messageBoxProxy.confirm(
t('importPreprocessor'), 'pre-line',
diff --git a/edit/source-editor.js b/edit/source-editor.js
index 6f826580..bc8a9f30 100644
--- a/edit/source-editor.js
+++ b/edit/source-editor.js
@@ -4,7 +4,7 @@
/* global MozDocMapper */// util.js
/* global MozSectionFinder */
/* global MozSectionWidget */
-/* global URLS debounce sessionStore */// toolbox.js
+/* global RX_META debounce sessionStore */// toolbox.js
/* global chromeSync */// storage-util.js
/* global cmFactory */
/* global editor */
@@ -307,7 +307,7 @@ function SourceEditor() {
if (_cm !== cm) {
return;
}
- const match = text.match(URLS.rxMETA);
+ const match = text.match(RX_META);
if (!match) {
return [];
}
diff --git a/js/toolbox.js b/js/toolbox.js
index e104a559..7c4eb442 100644
--- a/js/toolbox.js
+++ b/js/toolbox.js
@@ -2,6 +2,7 @@
/* exported
CHROME_POPUP_BORDER_BUG
+ RX_META
capitalize
closeCurrentTab
deepEqual
@@ -71,8 +72,6 @@ const URLS = {
// TODO: remove when "minimum_chrome_version": "61" or higher
chromeProtectsNTP: CHROME >= 61,
- rxMETA: /\/\*!?\s*==userstyle==[\s\S]*?==\/userstyle==\s*\*\//i,
-
uso: 'https://userstyles.org/',
usoJson: 'https://userstyles.org/styles/chrome/',
@@ -86,6 +85,7 @@ const URLS = {
const id = URLS.extractUsoArchiveId(url);
return id ? `${URLS.usoArchive}?style=${id}` : '';
},
+ makeUsoArchiveCodeUrl: id => `${URLS.usoArchiveRaw}usercss/${id}.user.css`,
extractGreasyForkInstallUrl: url =>
/^(https:\/\/(?:greasy|sleazy)fork\.org\/scripts\/\d+)[^/]*\/code\/[^/]*\.user\.css$|$/.exec(url)[1],
@@ -99,6 +99,8 @@ const URLS = {
),
};
+const RX_META = /\/\*!?\s*==userstyle==[\s\S]*?==\/userstyle==\s*\*\//i;
+
if (FIREFOX || OPERA || VIVALDI) {
document.documentElement.classList.add(
FIREFOX && 'firefox' ||
@@ -358,10 +360,11 @@ const sessionStore = new Proxy({}, {
* @param {Object} params
* @param {String} [params.method]
* @param {String|Object} [params.body]
- * @param {String} [params.responseType] arraybuffer, blob, document, json, text
+ * @param {'arraybuffer'|'blob'|'document'|'json'|'text'} [params.responseType]
* @param {Number} [params.requiredStatusCode] resolved when matches, otherwise rejected
* @param {Number} [params.timeout] ms
* @param {Object} [params.headers] {name: value}
+ * @param {string[]} [params.responseHeaders]
* @returns {Promise}
*/
function download(url, {
@@ -372,6 +375,7 @@ function download(url, {
timeout = 60e3, // connection timeout, USO is that bad
loadTimeout = 2 * 60e3, // data transfer timeout (counted from the first remote response)
headers,
+ responseHeaders,
} = {}) {
/* USO can't handle POST requests for style json and XHR/fetch can't handle super long URL
* so we need to collapse all long variables and expand them in the response */
@@ -404,10 +408,20 @@ function download(url, {
timer = loadTimeout && setTimeout(onTimeout, loadTimeout);
}
};
- xhr.onload = () =>
- xhr.status === requiredStatusCode || !requiredStatusCode || u.protocol === 'file:'
- ? resolve(expandUsoVars(xhr.response))
- : reject(xhr.status);
+ xhr.onload = () => {
+ if (xhr.status === requiredStatusCode || !requiredStatusCode || u.protocol === 'file:') {
+ const response = expandUsoVars(xhr.response);
+ if (responseHeaders) {
+ const headers = {};
+ for (const h of responseHeaders) headers[h] = xhr.getResponseHeader(h);
+ resolve({headers, response});
+ } else {
+ resolve(response);
+ }
+ } else {
+ reject(xhr.status);
+ }
+ };
xhr.onerror = () => reject(xhr.status);
xhr.onloadend = () => clearTimeout(timer);
xhr.responseType = responseType;
diff --git a/js/usercss-compiler.js b/js/usercss-compiler.js
index 8ff87b15..ca03e83a 100644
--- a/js/usercss-compiler.js
+++ b/js/usercss-compiler.js
@@ -140,7 +140,12 @@ function simplifyUsercssVars(vars) {
case 'dropdown':
case 'image':
// TODO: handle customized image
- value = va.options.find(o => o.name === value).value;
+ for (const opt of va.options) {
+ if (opt.name === value) {
+ value = opt.value;
+ break;
+ }
+ }
break;
case 'number':
case 'range':
diff --git a/manage/import-export.js b/manage/import-export.js
index 7a4b00f5..146b82a3 100644
--- a/manage/import-export.js
+++ b/manage/import-export.js
@@ -1,5 +1,5 @@
/* global API */// msg.js
-/* global URLS deepEqual isEmptyObj tryJSONparse */// toolbox.js
+/* global RX_META deepEqual isEmptyObj tryJSONparse */// toolbox.js
/* global changeQueue */// manage.js
/* global chromeSync */// storage-util.js
/* global prefs */
@@ -83,7 +83,7 @@ function importFromFile({fileTypeFilter, file} = {}) {
fReader.onloadend = event => {
fileInput.remove();
const text = event.target.result;
- const maybeUsercss = !/^\s*\[/.test(text) && URLS.rxMETA.test(text);
+ const maybeUsercss = !/^\s*\[/.test(text) && RX_META.test(text);
if (maybeUsercss) {
messageBoxProxy.alert(t('dragDropUsercssTabstrip'));
} else {
diff --git a/popup/search.js b/popup/search.js
index 1f649627..820084f5 100644
--- a/popup/search.js
+++ b/popup/search.js
@@ -409,7 +409,7 @@
result.pingbackTimer = setTimeout(download, PINGBACK_DELAY,
`${URLS.uso}styles/install/${id}?source=stylish-ch`);
- const updateUrl = `${URLS.usoArchiveRaw}usercss/${id}.user.css`;
+ const updateUrl = URLS.makeUsoArchiveCodeUrl(id);
try {
const sourceCode = await download(updateUrl);
const style = await API.usercss.install({sourceCode, updateUrl});