From 63b557cf8cde83f3b5156b53ae0769f3e65134c2 Mon Sep 17 00:00:00 2001 From: Kevin R Date: Fri, 25 Mar 2022 18:01:59 +0100 Subject: [PATCH] Version 1.24.0 --- CHANGELOG.md | 13 ++++++ clearurls.js | 80 ++++++++++++++++++------------------ core_js/eTagFilter.js | 4 +- core_js/tools.js | 96 ++++++++++++++----------------------------- core_js/watchdog.js | 9 ++-- manifest.json | 2 +- 6 files changed, 90 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f37b27..78269b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Require Firefox >= 55 - Require Chrome >= 37 +## [1.24.0] - 2022-03-25 + +### Compatibility note +- Require Firefox >= 55 +- Require Chrome >= 37 + +### Changed +- Replaced self-written URL parser through `URL` and `URLSearchParams` from the Web API + +### Fixed +- Fixed [185](https://github.com/ClearURLs/Addon/issues/185) +- Fixed [186](https://github.com/ClearURLs/Addon/issues/186) + ## [1.23.1] - 2022-03-23 ### Compatibility note diff --git a/clearurls.js b/clearurls.js index 42f4da6..7764fb0 100644 --- a/clearurls.js +++ b/clearurls.js @@ -46,8 +46,9 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { let changes = false; let cancel = false; let rawRules = provider.getRawRules(); + let urlObject = new URL(url); - if (storage.localHostsSkipping && checkLocalURL(pureUrl)) { + if (storage.localHostsSkipping && checkLocalURL(urlObject)) { return { "changes": false, "url": url, @@ -73,12 +74,7 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { } }); - if (existsFragments(url)) { - domain = url.replace(new RegExp("#.*", "i"), ""); - } - if (existsFields(url)) { - domain = url.replace(new RegExp("\\?.*", "i"), ""); - } + urlObject = new URL(url); /* * Expand the url by provider redirections. So no tracking on @@ -101,47 +97,55 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { } } - if (existsFields(url)) { - fields = "?" + extractFileds(url).rmEmpty().join("&"); - } - - if (existsFragments(url)) { - fragments = "#" + extractFragments(url).rmEmpty().join("&"); - } + fields = urlObject.searchParams; + fragments = extractFragments(urlObject); + domain = urlWithoutParamsAndHash(urlObject).toString(); /** * Only test for matches, if there are fields or fragments that can be cleaned. */ - if (fields !== "" || fragments !== "") { - rules.forEach(function (rule) { - let beforeReplace = fields; - let beforeReplaceFragments = fragments; - fields = fields.replace(new RegExp(rule, "gi"), ""); - fragments = fragments.replace(new RegExp(rule, "gi"), ""); + if (fields.toString() !== "" || fragments.toString() !== "") { + rules.forEach(rule => { + const beforeFields = fields.toString(); + const beforeFragments = fragments.toString(); + let localChange = false; - if (beforeReplace !== fields || beforeReplaceFragments !== fragments) { - //Log the action - if (storage.loggingStatus) { - let tempURL = domain; - let tempBeforeURL = domain; - - if (fields !== "") tempURL += "?" + fields.replace("?&", "?").replace("?", ""); - if (fragments !== "") tempURL += "#" + fragments.replace("#&", "#").replace("#", ""); - if (beforeReplace !== "") tempBeforeURL += "?" + beforeReplace.replace("?&", "?").replace("?", ""); - if (beforeReplaceFragments !== "") tempBeforeURL += "#" + beforeReplaceFragments.replace("#&", "#").replace("#", ""); - - if (!quiet) pushToLog(tempBeforeURL, tempURL, rule); + for (const field of fields.keys()) { + if (new RegExp(rule, "gi").test(field)) { + fields.delete(field); + changes = true; + localChange = true; } + } + + for (const fragment of fragments.keys()) { + if (new RegExp(rule, "gi").test(fragment)) { + fragments.delete(fragment); + changes = true; + localChange = true; + } + } + + //Log the action + if (localChange && storage.loggingStatus) { + let tempURL = domain; + let tempBeforeURL = domain; + + if (fields.toString() !== "") tempURL += "?" + fields.toString(); + if (fragments.toString() !== "") tempURL += "#" + fragments.toString(); + if (beforeFields.toString() !== "") tempBeforeURL += "?" + beforeFields.toString(); + if (beforeFragments.toString() !== "") tempBeforeURL += "#" + beforeFragments.toString(); + + if (!quiet) pushToLog(tempBeforeURL, tempURL, rule); increaseBadged(quiet, request); - changes = true; } }); let finalURL = domain; - if (fields !== "") finalURL += "?" + fields.replace("?", ""); - if (fragments !== "") finalURL += "#" + fragments.replace("#", ""); + if (fields.toString() !== "") finalURL += "?" + fields.toString(); + if (fragments.toString() !== "") finalURL += "#" + fragments.toString(); url = finalURL.replace(new RegExp("\\?&"), "?").replace(new RegExp("#&"), "#"); } @@ -240,7 +244,7 @@ function start() { * Deactivates ClearURLs, if no rules can be downloaded and also no old rules in storage */ function deactivateOnFailure() { - if(storage.ClearURLsData.length === 0) { + if (storage.ClearURLsData.length === 0) { storage.globalStatus = false; storage.dataHash = ""; changeIcon(); @@ -430,8 +434,6 @@ function start() { * @param {boolean} isActive Is this rule active? */ this.addRule = function (rule, isActive = true) { - rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^&]*)"; - this.applyRule(enabled_rules, disabled_rules, rule, isActive); }; @@ -476,8 +478,6 @@ function start() { * @param {boolean} isActive Is this rule active? */ this.addReferralMarketing = function (rule, isActive = true) { - rule = "([\\/\\?#]|(&|&))+(" + rule + "=[^&]*)"; - this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive); }; diff --git a/core_js/eTagFilter.js b/core_js/eTagFilter.js index cb8213c..254562d 100644 --- a/core_js/eTagFilter.js +++ b/core_js/eTagFilter.js @@ -23,7 +23,7 @@ */ function eTagFilter(requestDetails) { if(!requestDetails.responseHeaders || !storage.eTagFiltering - || storage.localHostsSkipping && checkLocalURL(requestDetails.url)) return {}; + || storage.localHostsSkipping && checkLocalURL(new URL(requestDetails.url))) return {}; const responseHeaders = requestDetails.responseHeaders; const filteredHeaders = responseHeaders.filter(header => { @@ -43,4 +43,4 @@ browser.webRequest.onHeadersReceived.addListener( eTagFilter, {urls: [""]}, ["blocking", "responseHeaders"] -); \ No newline at end of file +); diff --git a/core_js/tools.js b/core_js/tools.js index df5f384..8adb9f5 100644 --- a/core_js/tools.js +++ b/core_js/tools.js @@ -56,8 +56,7 @@ function isEmpty(obj) { * @param {string} string Name of the attribute used for localization * @param {string[]} placeholders Array of placeholders */ -function translate(string, ...placeholders) -{ +function translate(string, ...placeholders) { return browser.i18n.getMessage(string, placeholders); } @@ -84,24 +83,22 @@ async function checkOSAndroid() { /** * Extract the host without port from an url. - * @param {String} url URL as String + * @param {URL} url URL as String * @return {String} host as string */ function extractHost(url) { - let parsed_url = new URL(url); - - return parsed_url.hostname; + return url.hostname; } /** * Returns true if the url has a local host. - * @param {String} url URL as String + * @param {URL} url URL as object * @return {boolean} */ function checkLocalURL(url) { let host = extractHost(url); - if(!host.match(/^\d/) && host !== 'localhost') { + if (!host.match(/^\d/) && host !== 'localhost') { return false; } @@ -117,72 +114,39 @@ function checkLocalURL(url) { * @return {int} Number of Parameters */ function countFields(url) { - return extractFileds(url).length; -} - -/** - * Returns true if fields exists. - * @param {String} url URL as String - * @return {boolean} - */ -function existsFields(url) { - let matches = (url.match(/\?.+/i) || []); - let count = matches.length; - - return (count > 0); -} - -/** - * Extract the fields from an url. - * @param {String} url URL as String - * @return {Array} Fields as array - */ -function extractFileds(url) { - if (existsFields(url)) { - let fields = url.replace(new RegExp(".*?\\?", "i"), ""); - if (existsFragments(url)) { - fields = fields.replace(new RegExp("#.*", "i"), ""); - } - - return (fields.match(/[^\/|\?|&]+=?[^&]*/gi) || []); - } - - return []; -} - -/** - * Return the number of fragments query strings. - * @param {String} url URL as String - * @return {int} Number of fragments - */ -function countFragments(url) { - return extractFragments(url).length; + return new URL(url).searchParams.entries.length; } /** * Extract the fragments from an url. - * @param {String} url URL as String - * @return {Array} fragments as array + * @param {URL} url URL as object + * @return {URLSearchParams} fragments as URLSearchParams object */ function extractFragments(url) { - if (existsFragments(url)) { - let fragments = url.replace(new RegExp(".*?#", "i"), ""); - return (fragments.match(/[^&]+=?[^&]*/gi) || []); + if (url.hash) { + return new URLSearchParams(url.hash.slice(1)); + } else { + return new URLSearchParams(); } - - return []; } /** - * Returns true if fragments exists. - * @param {String} url URL as String - * @return {boolean} + * Returns the given URL without searchParams and hash. + * @param {URL} url the URL as object + * @return {URL} the url without searchParams and hash */ -function existsFragments(url) { - let matches = (url.match(/\#.+/i) || []); - let count = matches.length; +function urlWithoutParamsAndHash(url) { + let newURL = url.toString(); - return (count > 0); + if (url.search) { + newURL = newURL.replace(url.search, ""); + } + + if (url.hash) { + newURL = newURL.replace(url.hash, ""); + } + + return new URL(newURL); } /** @@ -283,13 +247,13 @@ function getBrowser() { function decodeURL(url) { let rtn = decodeURIComponent(url); - while(isEncodedURI(rtn)) { + while (isEncodedURI(rtn)) { rtn = decodeURIComponent(rtn); } // Required (e.g., to fix https://github.com/ClearURLs/Addon/issues/71) - if(!rtn.startsWith('http')) { - rtn = 'http://'+rtn + if (!rtn.startsWith('http')) { + rtn = 'http://' + rtn } return rtn; @@ -367,4 +331,4 @@ async function sha256(message) { const hashArray = Array.from(new Uint8Array(hashBuffer)); return hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); -} \ No newline at end of file +} diff --git a/core_js/watchdog.js b/core_js/watchdog.js index bfd37e4..c966308 100644 --- a/core_js/watchdog.js +++ b/core_js/watchdog.js @@ -25,13 +25,12 @@ * This watchdog restarts the whole Add-on, when the check fails. */ const CHECK_INTERVAL = 60000; +const __dirtyURL = "https://clearurls.roebert.eu?utm_source=addon"; +const __cleanURL = new URL("https://clearurls.roebert.eu").toString(); setInterval(function() { if(isStorageAvailable() && storage.globalStatus) { - const dirtyURL = "https://clearurls.roebert.eu?utm_source=addon"; - const cleanURL = "https://clearurls.roebert.eu"; - - if(pureCleaning(dirtyURL, true) !== cleanURL) { + if(new URL(pureCleaning(__dirtyURL, true)).toString() !== __cleanURL) { storage.watchDogErrorCount += 1; console.log(translate('watchdog', storage.watchDogErrorCount)); saveOnExit(); @@ -41,4 +40,4 @@ setInterval(function() { saveOnExit(); } } -}, CHECK_INTERVAL); \ No newline at end of file +}, CHECK_INTERVAL); diff --git a/manifest.json b/manifest.json index 362f27f..106185a 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "ClearURLs", - "version": "1.23.1", + "version": "1.24.0", "author": "Kevin Roebert", "description": "__MSG_extension_description__", "homepage_url": "https://docs.clearurls.xyz",