From c12d3fc5e38522823aa31942601ac4f395cb9804 Mon Sep 17 00:00:00 2001 From: tophf Date: Tue, 26 Jan 2021 16:33:17 +0300 Subject: [PATCH] convert USO styles to USO-archive on update --- background/style-manager.js | 2 +- background/style-search-db.js | 6 +- background/update-manager.js | 132 ++++++++++++++++++++++----- background/usercss-install-helper.js | 4 +- background/usercss-manager.js | 8 +- edit/sections-editor.js | 4 +- edit/source-editor.js | 4 +- js/toolbox.js | 28 ++++-- js/usercss-compiler.js | 7 +- manage/import-export.js | 4 +- popup/search.js | 2 +- 11 files changed, 151 insertions(+), 50 deletions(-) diff --git a/background/style-manager.js b/background/style-manager.js index 167328fc..26406feb 100644 --- a/background/style-manager.js +++ b/background/style-manager.js @@ -47,7 +47,7 @@ const styleMan = (() => { _id: () => uuidv4(), _rev: () => Date.now(), }; - const DELETE_IF_NULL = ['id', 'customName']; + const DELETE_IF_NULL = ['id', 'customName', 'md5Url', 'originalMd5']; /** @type {Promise|boolean} will be `true` to avoid wasting a microtask tick on each `await` */ let ready = init(); diff --git a/background/style-search-db.js b/background/style-search-db.js index ca8e6e06..23e18f76 100644 --- a/background/style-search-db.js +++ b/background/style-search-db.js @@ -1,5 +1,5 @@ /* global API */// msg.js -/* global URLS debounce stringAsRegExp tryRegExp */// toolbox.js +/* global RX_META debounce stringAsRegExp tryRegExp */// toolbox.js /* global addAPI */// common.js 'use strict'; @@ -10,12 +10,12 @@ const extractMeta = style => style.usercssData - ? (style.sourceCode.match(URLS.rxMETA) || [''])[0] + ? (style.sourceCode.match(RX_META) || [''])[0] : null; const stripMeta = style => style.usercssData - ? style.sourceCode.replace(URLS.rxMETA, '') + ? style.sourceCode.replace(RX_META, '') : null; const MODES = Object.assign(Object.create(null), { diff --git a/background/update-manager.js b/background/update-manager.js index 72951e59..c8578987 100644 --- a/background/update-manager.js +++ b/background/update-manager.js @@ -1,7 +1,8 @@ /* global API */// msg.js -/* global URLS debounce download ignoreChromeError */// toolbox.js +/* global RX_META URLS debounce download ignoreChromeError */// toolbox.js /* global calcStyleDigest styleJSONseemsValid styleSectionsEqual */ // sections-util.js /* global chromeLocal */// storage-util.js +/* global db */ /* global prefs */ 'use strict'; @@ -21,7 +22,14 @@ const updateMan = (() => { ERROR_JSON: 'error: JSON is invalid', ERROR_VERSION: 'error: version is older than installed style', }; - + const RH_ETAG = {responseHeaders: ['etag']}; // a hashsum of file contents + const RX_DATE2VER = new RegExp([ + /^(\d{4})/, + /(1(?:0|[12](?=\d\d))?|[2-9])/, // in ambiguous cases like yyyy123 the month will be 1 + /([1-2][0-9]?|3[0-1]?|[4-9])/, + /\.(0|1[0-9]?|2[0-3]?|[3-9])/, + /\.(0|[1-5][0-9]?|[6-9])$/, + ].map(rx => rx.source).join('')); const ALARM_NAME = 'scheduledUpdate'; const MIN_INTERVAL_MS = 60e3; const RETRY_ERRORS = [ @@ -96,13 +104,14 @@ const updateMan = (() => { 'ignoreDigest' option is set on the second manual individual update check on the manage page. */ async function checkStyle(opts) { + let {id} = opts; const { - id, style = await API.styles.get(id), ignoreDigest, port, save, } = opts; + if (!id) id = style.id; const ucd = style.usercssData; let res, state; try { @@ -119,7 +128,7 @@ const updateMan = (() => { res = {error, style, STATES}; state = `${STATES.SKIPPED} (${error})`; } - log(`${state} #${style.id} ${style.customName || style.name}`); + log(`${state} #${id} ${style.customName || style.name}`); if (port) port.postMessage(res); return res; @@ -132,6 +141,11 @@ const updateMan = (() => { } async function updateUSO() { + const url = URLS.makeUsoArchiveCodeUrl(style.md5Url.match(/\d+/)[0]); + const req = await tryDownload(url, RH_ETAG).catch(() => null); + if (req) { + return updateToUSOArchive(url, req); + } const md5 = await tryDownload(style.md5Url); if (!md5 || md5.length !== 32) { return Promise.reject(STATES.ERROR_MD5); @@ -148,33 +162,82 @@ const updateMan = (() => { return json; } - async function updateUsercss() { - // TODO: when sourceCode is > 100kB use http range request(s) for version check - const url = style.updateUrl; - const metaUrl = URLS.extractGreasyForkInstallUrl(url) && - url.replace(/\.user\.css$/, '.meta.css'); - const text = await tryDownload(metaUrl || url); - const json = await API.usercss.buildMeta({sourceCode: text}); - await require(['/vendor/semver-bundle/semver']); /* global semverCompare */ - const delta = semverCompare(json.usercssData.version, ucd.version); - if (!delta && !ignoreDigest) { - // re-install is invalid in a soft upgrade - const sameCode = !metaUrl && text === style.sourceCode; - return Promise.reject(sameCode ? STATES.SAME_CODE : STATES.SAME_VERSION); + async function updateToUSOArchive(url, req) { + // UserCSS metadata may be embedded in the original USO style so let's use its updateURL + const [meta2] = req.response.replace(RX_META, '').match(RX_META) || []; + if (meta2 && meta2.includes('@updateURL')) { + const {updateUrl} = await API.usercss.buildMeta({sourceCode: meta2}).catch(() => ({})); + if (updateUrl) { + url = updateUrl; + req = await tryDownload(url, RH_ETAG); + } } - if (delta < 0) { - // downgrade is always invalid - return Promise.reject(STATES.ERROR_VERSION); - } - if (metaUrl) { - json.sourceCode = await tryDownload(url); + const json = await API.usercss.buildMeta({ + id, + etag: req.headers.etag, + md5Url: null, + originalMd5: null, + sourceCode: req.response, + updateUrl: url, + url: URLS.extractUsoArchiveInstallUrl(url), + }); + const varUrlValues = style.updateUrl.split('?')[1]; + const varData = json.usercssData.vars; + if (varUrlValues && varData) { + const IK = 'ik-'; + const IK_LEN = IK.length; + for (let [key, val] of new URLSearchParams(varUrlValues)) { + if (!key.startsWith(IK)) continue; + key = key.slice(IK_LEN); + const varDef = varData[key]; + if (!varDef) continue; + if (varDef.options) { + let sel = val.startsWith(IK) && getVarOptByName(varDef, val.slice(IK_LEN)); + if (!sel) { + key += '-custom'; + sel = getVarOptByName(varDef, key + '-dropdown'); + if (sel) varData[key].value = val; + } + if (sel) varDef.value = sel.name; + } else { + varDef.value = val; + } + } } return API.usercss.buildCode(json); } + async function updateUsercss() { + if (style.etag && style.etag === await downloadEtag()) { + return Promise.reject(STATES.SAME_CODE); + } + // TODO: when sourceCode is > 100kB use http range request(s) for version check + const {headers: {etag}, response} = await tryDownload(style.updateUrl, RH_ETAG); + const json = await API.usercss.buildMeta({sourceCode: response, etag}); + await require(['/vendor/semver-bundle/semver']); /* global semverCompare */ + const delta = semverCompare(json.usercssData.version, ucd.version); + let err; + if (!delta && !ignoreDigest) { + // re-install is invalid in a soft upgrade + err = response === style.sourceCode ? STATES.SAME_CODE : STATES.SAME_VERSION; + } + if (delta < 0) { + // downgrade is always invalid + err = STATES.ERROR_VERSION; + } + if (err && etag && !style.etag) { + // first check of ETAG, gonna write it directly to DB as it's too trivial to sync or announce + style.etag = etag; + await db.exec('put', style); + } + return err + ? Promise.reject(err) + : API.usercss.buildCode(json); + } + async function maybeSave(json) { - json.id = style.id; - json.updateDate = Date.now(); + json.id = id; + json.updateDate = getDateFromVer(json) || Date.now(); // keep current state delete json.customName; delete json.enabled; @@ -206,6 +269,25 @@ const updateMan = (() => { await new Promise(resolve => setTimeout(resolve, retryDelay)); } } + + async function downloadEtag() { + const opts = Object.assign({method: 'head'}, RH_ETAG); + const req = await tryDownload(style.updateUrl, opts); + return req.headers.etag; + } + + function getDateFromVer(style) { + const m = style.updateUrl.startsWith(URLS.usoArchiveRaw) && + style.usercssData.version.match(RX_DATE2VER); + if (m) { + m[2]--; // month is 0-based in `Date` constructor + return new Date(...m.slice(1)).getTime(); + } + } + + function getVarOptByName(varDef, name) { + return varDef.options.find(o => o.name === name); + } } function schedule() { diff --git a/background/usercss-install-helper.js b/background/usercss-install-helper.js index b33052a1..6656ac7b 100644 --- a/background/usercss-install-helper.js +++ b/background/usercss-install-helper.js @@ -1,4 +1,4 @@ -/* global URLS download openURL */// toolbox.js +/* global RX_META URLS download openURL */// toolbox.js /* global addAPI bgReady */// common.js /* global tabMan */// msg.js 'use strict'; @@ -85,7 +85,7 @@ bgReady.all.then(() => { !oldUrl.startsWith(URLS.installUsercss)) { const inTab = url.startsWith('file:') && !chrome.app; const code = await (inTab ? loadFromFile : loadFromUrl)(tabId, url); - if (!/^\s*= 61, - rxMETA: /\/\*!?\s*==userstyle==[\s\S]*?==\/userstyle==\s*\*\//i, - uso: 'https://userstyles.org/', usoJson: 'https://userstyles.org/styles/chrome/', @@ -86,6 +85,7 @@ const URLS = { const id = URLS.extractUsoArchiveId(url); return id ? `${URLS.usoArchive}?style=${id}` : ''; }, + makeUsoArchiveCodeUrl: id => `${URLS.usoArchiveRaw}usercss/${id}.user.css`, extractGreasyForkInstallUrl: url => /^(https:\/\/(?:greasy|sleazy)fork\.org\/scripts\/\d+)[^/]*\/code\/[^/]*\.user\.css$|$/.exec(url)[1], @@ -99,6 +99,8 @@ const URLS = { ), }; +const RX_META = /\/\*!?\s*==userstyle==[\s\S]*?==\/userstyle==\s*\*\//i; + if (FIREFOX || OPERA || VIVALDI) { document.documentElement.classList.add( FIREFOX && 'firefox' || @@ -358,10 +360,11 @@ const sessionStore = new Proxy({}, { * @param {Object} params * @param {String} [params.method] * @param {String|Object} [params.body] - * @param {String} [params.responseType] arraybuffer, blob, document, json, text + * @param {'arraybuffer'|'blob'|'document'|'json'|'text'} [params.responseType] * @param {Number} [params.requiredStatusCode] resolved when matches, otherwise rejected * @param {Number} [params.timeout] ms * @param {Object} [params.headers] {name: value} + * @param {string[]} [params.responseHeaders] * @returns {Promise} */ function download(url, { @@ -372,6 +375,7 @@ function download(url, { timeout = 60e3, // connection timeout, USO is that bad loadTimeout = 2 * 60e3, // data transfer timeout (counted from the first remote response) headers, + responseHeaders, } = {}) { /* USO can't handle POST requests for style json and XHR/fetch can't handle super long URL * so we need to collapse all long variables and expand them in the response */ @@ -404,10 +408,20 @@ function download(url, { timer = loadTimeout && setTimeout(onTimeout, loadTimeout); } }; - xhr.onload = () => - xhr.status === requiredStatusCode || !requiredStatusCode || u.protocol === 'file:' - ? resolve(expandUsoVars(xhr.response)) - : reject(xhr.status); + xhr.onload = () => { + if (xhr.status === requiredStatusCode || !requiredStatusCode || u.protocol === 'file:') { + const response = expandUsoVars(xhr.response); + if (responseHeaders) { + const headers = {}; + for (const h of responseHeaders) headers[h] = xhr.getResponseHeader(h); + resolve({headers, response}); + } else { + resolve(response); + } + } else { + reject(xhr.status); + } + }; xhr.onerror = () => reject(xhr.status); xhr.onloadend = () => clearTimeout(timer); xhr.responseType = responseType; diff --git a/js/usercss-compiler.js b/js/usercss-compiler.js index 8ff87b15..ca03e83a 100644 --- a/js/usercss-compiler.js +++ b/js/usercss-compiler.js @@ -140,7 +140,12 @@ function simplifyUsercssVars(vars) { case 'dropdown': case 'image': // TODO: handle customized image - value = va.options.find(o => o.name === value).value; + for (const opt of va.options) { + if (opt.name === value) { + value = opt.value; + break; + } + } break; case 'number': case 'range': diff --git a/manage/import-export.js b/manage/import-export.js index 7a4b00f5..146b82a3 100644 --- a/manage/import-export.js +++ b/manage/import-export.js @@ -1,5 +1,5 @@ /* global API */// msg.js -/* global URLS deepEqual isEmptyObj tryJSONparse */// toolbox.js +/* global RX_META deepEqual isEmptyObj tryJSONparse */// toolbox.js /* global changeQueue */// manage.js /* global chromeSync */// storage-util.js /* global prefs */ @@ -83,7 +83,7 @@ function importFromFile({fileTypeFilter, file} = {}) { fReader.onloadend = event => { fileInput.remove(); const text = event.target.result; - const maybeUsercss = !/^\s*\[/.test(text) && URLS.rxMETA.test(text); + const maybeUsercss = !/^\s*\[/.test(text) && RX_META.test(text); if (maybeUsercss) { messageBoxProxy.alert(t('dragDropUsercssTabstrip')); } else { diff --git a/popup/search.js b/popup/search.js index 1f649627..820084f5 100644 --- a/popup/search.js +++ b/popup/search.js @@ -409,7 +409,7 @@ result.pingbackTimer = setTimeout(download, PINGBACK_DELAY, `${URLS.uso}styles/install/${id}?source=stylish-ch`); - const updateUrl = `${URLS.usoArchiveRaw}usercss/${id}.user.css`; + const updateUrl = URLS.makeUsoArchiveCodeUrl(id); try { const sourceCode = await download(updateUrl); const style = await API.usercss.install({sourceCode, updateUrl});