Merge branch 'whitelist' into 'master'

Draft: Whitelist

See merge request ClearURLs/ClearUrls!110

README.md

@@ -77,8 +77,6 @@ Please push your translation into the folder `_locales/{language code}/messages.
 * [Unalix](https://github.com/AmanoTeam/Unalix) small, dependency-free, fast Python package for removing tracking fields from URLs
 * [Unalix-nim](https://github.com/AmanoTeam/Unalix-nim) small, dependency-free, fast Nim package and CLI tool for removing tracking fields from URLs
 * [UnalixAndroid](https://github.com/AmanoTeam/UnalixAndroid) simple Android app that removes link masking/tracking and optionally resolves shortened links
-* [pl-fe](https://github.com/mkljczk/pl-fe) is a Fediverse client which uses ClearURLs code to clean URLs from displayed posts and recommend cleaning URLs from created posts
-* [URLCheck](https://github.com/TrianguloY/URLCheck) is an Android app to review and edit URLs before opening them. Allows to use the ClearURLs catalog.
 
 ## Recommended by...
 *  [ghacks-user.js](https://github.com/ghacksuserjs/ghacks-user.js/wiki/4.1-Extensions)
@@ -106,12 +104,3 @@ We use some third-party scripts in our add-on. The authors and licenses are list
     [MIT](https://github.com/Simonwep/pickr/blob/master/LICENSE)
 -   [Font Awesome](https://github.com/FortAwesome/Font-Awesome/) | Copyright (c) @fontawesome |
     [Font Awesome Free License](https://github.com/FortAwesome/Font-Awesome/blob/master/LICENSE.txt)
-
-
-## Star History
-
- <picture>
-   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=ClearURLs/Addon&type=Date&theme=dark" />
-   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=ClearURLs/Addon&type=Date" />
-   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=ClearURLs/Addon&type=Date" />
- </picture>

_locales/en/messages.json

@@ -87,6 +87,14 @@
         "message": "Show numbers of cleaned urls",
         "description": "This string is used as title for the badges switch button on the popup page."
     },
+    "popup_html_configs_whitelist_button_add": {
+        "message": "Whitelist Site",
+        "description": "This string is used as name for the whitelist button on the popup page."
+    },
+    "popup_html_configs_whitelist_button_remove": {
+        "message": "Remove from Whitelist",
+        "description": "This string is used as name for the whitelist button on the popup page."
+    },
     "popup_html_statistics_head": {
         "message": "Statistics",
         "description": "This string is used as title for the statistics on the popup page."
@@ -179,6 +187,10 @@
         "message": "The url to the rules.hash file (hash)",
         "description": "This string is used as name for the rule.hash url label."
     },
+    "setting_whitelist_list_label": {
+        "message": "Whitelisted sites",
+        "description": "This string is used as name for the whitelisted sites list label."
+    },
     "setting_types_label": {
         "message": "<a href='https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webRequest/ResourceType' target='_blank'>Request types</a> (expert level)",
         "description": "This string is used as name for the types label."

clearurls.js

@@ -21,8 +21,6 @@
 * This script is responsible for the core functionalities.
 */
 var providers = [];
-var providersByToken = {}; // Map<string, Provider[]>
-var globalProviders = []; // Provider[]
 var prvKeys = [];
 var siteBlockedAlert = 'javascript:void(0)';
 var dataHash;
@@ -49,6 +47,19 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
     let rawRules = provider.getRawRules();
     let urlObject = new URL(url);
 
+    /*
+    * Skip whitelisted sites
+    */
+    for (const site of storage.whitelist) {
+        if (url.indexOf(site) !== -1) {
+            return {
+                "changes": false,
+                "url": url,
+                "cancel": false
+            }
+        }
+    }
+
     if (storage.localHostsSkipping && checkLocalURL(urlObject)) {
         return {
             "changes": false,
@@ -91,14 +102,14 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
     /*
     * Apply raw rules to the URL.
     */
-    rawRules.forEach(function ({ rule: rawRuleStr, regex: rawRuleRegex }) {
+    rawRules.forEach(function (rawRule) {
         let beforeReplace = url;
-        url = url.replace(rawRuleRegex, "");
+        url = url.replace(new RegExp(rawRule, "gi"), "");
 
         if (beforeReplace !== url) {
             //Log the action
             if (storage.loggingStatus && !quiet) {
-                pushToLog(beforeReplace, url, rawRuleStr);
+                pushToLog(beforeReplace, url, rawRule);
             }
 
             increaseBadged(quiet, request);
@@ -115,13 +126,13 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
      * Only test for matches, if there are fields or fragments that can be cleaned.
      */
     if (fields.toString() !== "" || fragments.toString() !== "") {
-        rules.forEach(({ rule, regex }) => {
+        rules.forEach(rule => {
             const beforeFields = fields.toString();
             const beforeFragments = fragments.toString();
             let localChange = false;
 
             for (const field of fields.keys()) {
-                if (regex.test(field)) {
+                if (new RegExp("^"+rule+"$", "gi").test(field)) {
                     fields.delete(field);
                     changes = true;
                     localChange = true;
@@ -129,7 +140,7 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
             }
 
             for (const fragment of fragments.keys()) {
-                if (regex.test(fragment)) {
+                if (new RegExp("^"+rule+"$", "gi").test(fragment)) {
                     fragments.delete(fragment);
                     changes = true;
                     localChange = true;
@@ -230,17 +241,6 @@ function start() {
             for (let re = 0; re < methods.length; re++) {
                 providers[p].addMethod(methods[re]);
             }
-
-            // Indexing logic
-            const token = providers[p].getLookupToken();
-            if (token) {
-                if (!providersByToken[token]) {
-                    providersByToken[token] = [];
-                }
-                providersByToken[token].push(providers[p]);
-            } else {
-                globalProviders.push(providers[p]);
-            }
         }
     }
 
@@ -373,7 +373,7 @@ function start() {
         let methods = [];
 
         if (_completeProvider) {
-            enabled_rules[".*"] = new RegExp("^.*$", "i");
+            enabled_rules[".*"] = true;
         }
 
         /**
@@ -392,38 +392,6 @@ function start() {
             return name;
         };
 
-        /**
-         * Returns the lookup token for this provider, or null if global.
-         * Extracts "domain" from patterns like ^https?://(?:[a-z0-9-]+\.)*?domain...
-         * @return {String|null}
-         */
-        this.getLookupToken = function () {
-            if (!urlPattern) return null;
-            const source = urlPattern.source;
-
-            // Case 1: Wildcard prefix pattern (e.g. ...*?amazon...)
-            const wildcardMatch = source.match(/\*\?([a-z0-9-]+)/i);
-            if (wildcardMatch && wildcardMatch[1]) {
-                return wildcardMatch[1].toLowerCase();
-            }
-
-            // Case 2: Explicit start pattern (e.g. ^https?://vk.com...)
-            // Matches ^https?://(optional www.)token
-            // We strip standard regex start structure to find the first meaningful domain token.
-            // This regex handles:
-            // - ^https?://  (start)
-            // - \/\/ or //  (slashes, potentially escaped)
-            // - www.        (optional www prefix, potentially escaped)
-            // - [a-z0-9-]+  (the token)
-            // It deliberately fails on patterns with groups (?:...) at the start, falling back to global.
-            const explicitMatch = source.match(/^(\^?https?:\\?\/\\?\/)(?:www(?:\\?\.))?([a-z0-9-]+)/i);
-            if (explicitMatch && explicitMatch[2]) {
-                return explicitMatch[2].toLowerCase();
-            }
-
-            return null;
-        };
-
         /**
          * Add URL pattern.
          *
@@ -451,21 +419,25 @@ function start() {
         };
 
         /**
-         * Helper to update rule maps with compiled regexes.
+         * Apply a rule to a given tuple of rule array.
+         * @param enabledRuleArray      array for enabled rules
+         * @param disabledRulesArray    array for disabled rules
+         * @param {String} rule         RegExp as string
+         * @param {boolean} isActive    Is this rule active?
          */
-        const updateRule = (enabledMap, disabledMap, rule, isActive, compileFn) => {
+        this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => {
             if (isActive) {
-                if (!enabledMap[rule]) {
-                    try {
-                        enabledMap[rule] = compileFn(rule);
-                    } catch (e) {
-                        console.error("Invalid regex", rule, e);
-                    }
+                enabledRuleArray[rule] = true;
+
+                if (disabledRulesArray[rule] !== undefined) {
+                    delete disabledRulesArray[rule];
                 }
-                if (disabledMap[rule]) delete disabledMap[rule];
             } else {
-                disabledMap[rule] = true;
-                if (enabledMap[rule]) delete enabledMap[rule];
+                disabledRulesArray[rule] = true;
+
+                if (enabledRuleArray[rule] !== undefined) {
+                    delete enabledRuleArray[rule];
+                }
             }
         };
 
@@ -477,22 +449,20 @@ function start() {
          * @param {boolean} isActive   Is this rule active?
          */
         this.addRule = function (rule, isActive = true) {
-            updateRule(enabled_rules, disabled_rules, rule, isActive, r => new RegExp("^" + r + "$", "i"));
+            this.applyRule(enabled_rules, disabled_rules, rule, isActive);
         };
 
         /**
-         * Return all active rules as an array of {rule, regex}.
+         * Return all active rules as an array.
          *
-         * @return Array Objects
+         * @return Array RegExp strings
          */
         this.getRules = function () {
-            let source = enabled_rules;
             if (!storage.referralMarketing) {
-                // Determine if we need to merge referral marketing rules
-                // We use a new object to avoid mutating enabled_rules via Object.assign if that was happening
-                source = Object.assign({}, enabled_rules, enabled_referralMarketing);
+                return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing));
             }
-            return Object.entries(source).map(([rule, regex]) => ({ rule, regex }));
+
+            return Object.keys(enabled_rules);
         };
 
         /**
@@ -503,17 +473,16 @@ function start() {
          * @param {boolean} isActive   Is this rule active?
          */
         this.addRawRule = function (rule, isActive = true) {
-            updateRule(enabled_rawRules, disabled_rawRules, rule, isActive, r => new RegExp(r, "gi"));
+            this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive);
         };
 
         /**
          * Return all active raw rules as an array.
          *
-         * @return Array Objects {rule, regex}
+         * @return Array RegExp strings
          */
         this.getRawRules = function () {
-            // return Object.keys(enabled_rawRules);
-            return Object.entries(enabled_rawRules).map(([rule, regex]) => ({ rule, regex }));
+            return Object.keys(enabled_rawRules);
         };
 
         /**
@@ -524,7 +493,7 @@ function start() {
          * @param {boolean} isActive   Is this rule active?
          */
         this.addReferralMarketing = function (rule, isActive = true) {
-            updateRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive, r => new RegExp("^" + r + "$", "i"));
+            this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive);
         };
 
         /**
@@ -535,7 +504,19 @@ function start() {
          * @param {Boolean} isActive   Is this exception active?
          */
         this.addException = function (exception, isActive = true) {
-            updateRule(enabled_exceptions, disabled_exceptions, exception, isActive, r => new RegExp(r, "i"));
+            if (isActive) {
+                enabled_exceptions[exception] = true;
+
+                if (disabled_exceptions[exception] !== undefined) {
+                    delete disabled_exceptions[exception];
+                }
+            } else {
+                disabled_exceptions[exception] = true;
+
+                if (enabled_exceptions[exception] !== undefined) {
+                    delete enabled_exceptions[exception];
+                }
+            }
         };
 
         /**
@@ -573,9 +554,11 @@ function start() {
             //Add the site blocked alert to every exception
             if (url === siteBlockedAlert) return true;
 
-            for (const [exception, regex] of Object.entries(enabled_exceptions)) {
+            for (const exception in enabled_exceptions) {
                 if (result) break;
-                result = regex.test(url);
+
+                let exception_regex = new RegExp(exception, "i");
+                result = exception_regex.test(url);
             }
 
             return result;
@@ -589,7 +572,19 @@ function start() {
          * @param {Boolean} isActive     Is this redirection active?
          */
         this.addRedirection = function (redirection, isActive = true) {
-            updateRule(enabled_redirections, disabled_redirections, redirection, isActive, r => new RegExp(r, "i"));
+            if (isActive) {
+                enabled_redirections[redirection] = true;
+
+                if (disabled_redirections[redirection] !== undefined) {
+                    delete disabled_redirections[redirection];
+                }
+            } else {
+                disabled_redirections[redirection] = true;
+
+                if (enabled_redirections[redirection] !== undefined) {
+                    delete enabled_redirections[redirection];
+                }
+            }
         };
 
         /**
@@ -600,11 +595,11 @@ function start() {
         this.getRedirection = function (url) {
             let re = null;
 
-            for (const [redirection, regex] of Object.entries(enabled_redirections)) {
-                let result = url.match(regex);
+            for (const redirection in enabled_redirections) {
+                let result = (url.match(new RegExp(redirection, "i")));
 
                 if (result && result.length > 0 && redirection) {
-                    re = result[1];
+                    re = (new RegExp(redirection, "i")).exec(url)[1];
 
                     break;
                 }
@@ -641,42 +636,16 @@ function start() {
                 pushToLog(request.url, request.url, translate('log_ping_blocked'));
                 increaseBadged(false, request);
                 increaseTotalCounter(1);
-                return { cancel: true };
+                return {cancel: true};
             }
 
-            let host = "";
-            try {
-                host = extractHost(new URL(request.url));
-            } catch (e) {
-                // If URL parsing fails, we falls back to empty host, relying on global providers or skipping
-            }
-
-            const hostTokens = host.split('.').map(t => t.toLowerCase());
-
-            // Collect candidate providers: Global + Key Matches
-            // Use a Set to avoid duplicates if multiple tokens map to same provider (unlikely but safe)
-            let candidateProviders = new Set(globalProviders);
-
-            for (const token of hostTokens) {
-                if (providersByToken[token]) {
-                    for (const p of providersByToken[token]) {
-                        candidateProviders.add(p);
-                    }
-                }
-            }
-
-            // "providers" global var is still used for legacy, but here we iterate candidates
-            // Converting Set to Array for iteration
-            const candidates = Array.from(candidateProviders);
-
             /*
             * Call for every provider the removeFieldsFormURL method.
             */
-            for (let i = 0; i < candidates.length; i++) {
-                const provider = candidates[i];
-                if (!provider.matchMethod(request)) continue;
-                if (provider.matchURL(request.url)) {
-                    result = removeFieldsFormURL(provider, request.url, false, request);
+            for (let i = 0; i < providers.length; i++) {
+                if (!providers[i].matchMethod(request)) continue;
+                if (providers[i].matchURL(request.url)) {
+                    result = removeFieldsFormURL(providers[i], request.url, false, request);
                 }
 
                 /*
@@ -684,10 +653,10 @@ function start() {
                 * Cancel the active request.
                 */
                 if (result.redirect) {
-                    if (provider.shouldForceRedirect() &&
+                    if (providers[i].shouldForceRedirect() &&
                         request.type === 'main_frame') {
-                        browser.tabs.update(request.tabId, { url: result.url }).catch(handleError);
-                        return { cancel: true };
+                        browser.tabs.update(request.tabId, {url: result.url}).catch(handleError);
+                        return {cancel: true};
                     }
 
                     return {
@@ -702,9 +671,9 @@ function start() {
                 if (result.cancel) {
                     if (request.type === 'main_frame') {
                         const blockingPage = browser.runtime.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url));
-                        browser.tabs.update(request.tabId, { url: blockingPage }).catch(handleError);
+                        browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError);
 
-                        return { cancel: true };
+                        return {cancel: true};
                     } else {
                         return {
                             redirectUrl: siteBlockedAlert
@@ -767,7 +736,7 @@ function start() {
      */
     browser.webRequest.onBeforeRequest.addListener(
         promise,
-        { urls: ["<all_urls>"], types: getData("types").concat(getData("pingRequestTypes")) },
+        {urls: ["<all_urls>"], types: getData("types").concat(getData("pingRequestTypes"))},
         ["blocking"]
     );
 }

core_js/popup.js

@@ -62,6 +62,34 @@ function changeStatistics()
     elTotal.textContent = totalCounter.toLocaleString();
 }
 
+/**
+* Set the whitelist button text
+*/
+function setWhitelistText()
+{
+    let element = document.getElementById('whitelist_btn');
+    let currentSite;
+    browser.tabs.query({active: true, currentWindow: true}, function(tabs) {
+        currentSite = tabs[0].url;
+    });
+
+    browser.runtime.sendMessage({
+        function: "getData",
+        params: ['whitelist']
+    }).then((data) => {
+        let siteFound = data.response.some(site => currentSite.indexOf(site) !== -1);
+        if (siteFound) {
+            element.classList.replace('btn-primary', 'btn-danger')
+            element.textContent = translate('popup_html_configs_whitelist_button_remove')
+            document.getElementById('whitelist_btn').onclick = () => {changeWhitelist(true)};
+        } else {
+            element.classList.replace('btn-danger', 'btn-primary')
+            element.textContent = translate('popup_html_configs_whitelist_button_add')
+            document.getElementById('whitelist_btn').onclick = () => {changeWhitelist(false)};
+        }
+    }).catch(handleError);
+}
+
 /**
 * Set the value for the hashStatus on startUp.
 */
@@ -155,6 +183,36 @@ function setSwitchButton(id, varname)
     element.checked = this[varname];
 }
 
+/**
+* Adds (or removes) the site the user is on to the whitelist
+* Whitelisted sites do not get filtered
+* @param {boolean} removeWl If true remove current site instead of adding
+*/
+function changeWhitelist(removeWl) {
+    let site;
+    browser.tabs.query({active: true, currentWindow: true}, function(tabs) { // Couldn't figure out how to access currentUrl var
+        site = tabs[0].url;                                                  // So this is used instead
+    });
+    browser.runtime.sendMessage({
+        function: "getData",
+        params: ['whitelist']
+    }).then((data) => {
+        let siteUrl = new URL(site)
+        let domain = siteUrl.hostname
+        if (removeWl) {
+            data.response = data.response.filter(wlSite => wlSite !== domain)
+        } else {
+            data.response.push(domain)
+        }
+        browser.runtime.sendMessage({
+            function: "setData",
+            params: ['whitelist', data.response]
+        }).then(() => {
+            setWhitelistText();
+        }).catch(handleError);
+    }).catch(handleError);
+}
+
 /**
 * Reset the global statistic
 */
@@ -220,6 +278,7 @@ function setText()
     injectText('configs_switch_filter','popup_html_configs_switch_filter');
     injectText('configs_head','popup_html_configs_head');
     injectText('configs_switch_statistics','configs_switch_statistics');
+    setWhitelistText();
     document.getElementById('donate').title = translate('donate_button');
 }
 

core_js/settings.js

@@ -82,6 +82,7 @@ function save() {
     saveData("badged_color", pickr.getColor().toHEXA().toString())
         .then(() => saveData("ruleURL", document.querySelector('input[name=ruleURL]').value))
         .then(() => saveData("hashURL", document.querySelector('input[name=hashURL]').value))
+        .then(() => saveData("whitelist", document.querySelector('input[name=whitelist]').value.split(',')))
         .then(() => saveData("types", document.querySelector('input[name=types]').value))
         .then(() => saveData("logLimit", Math.max(0, Math.min(5000, document.querySelector('input[name=logLimit]').value))))
         .then(() => browser.runtime.sendMessage({
@@ -122,6 +123,7 @@ function getData() {
 
     loadData("ruleURL")
         .then(() => loadData("hashURL"))
+        .then(() => loadData("whitelist"))
         .then(() => loadData("types"))
         .then(() => loadData("logLimit"))
         .then(logData => {
@@ -216,6 +218,7 @@ function setText() {
     document.getElementById('reset_settings_btn').setAttribute('title', translate('setting_html_reset_button_title'));
     document.getElementById('rule_url_label').textContent = translate('setting_rule_url_label');
     document.getElementById('hash_url_label').textContent = translate('setting_hash_url_label');
+    document.getElementById('whitelist_list_label').textContent = translate('setting_whitelist_list_label');
     document.getElementById('types_label').innerHTML = translate('setting_types_label');
     document.getElementById('save_settings_btn').textContent = translate('settings_html_save_button');
     document.getElementById('save_settings_btn').setAttribute('title', translate('settings_html_save_button_title'));

core_js/storage.js

@@ -216,6 +216,7 @@ function initSettings() {
     storage.badged_color = "#ffa500";
     storage.hashURL = "https://rules2.clearurls.xyz/rules.minify.hash";
     storage.ruleURL = "https://rules2.clearurls.xyz/data.minify.json";
+    storage.whitelist = []; // TODO: If we do whitelist per rule, this needs to be obj
     storage.contextMenuEnabled = true;
     storage.historyListenerEnabled = true;
     storage.localHostsSkipping = true;

html/popup.html

@@ -86,6 +86,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                         <label id="configs_switch_statistics"></label>
                     </label>
                     <div class="clearfix"></div>
+                    <div class="text-center">
+                        <button type="button" id="whitelist_btn" class="btn btn-primary btn-sm text-wrap"></button>
+                    </div>
                     <br />
                 </div>
             </div>

html/settings.html

@@ -105,6 +105,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                     <input type="url" id="hashURL" value="" name="hashURL" class="form-control" />
                 </p>
                 <br />
+                <p>
+                    <label id="whitelist_list_label"></label><br />
+                    <input type="text" id="whitelist" value="" name="whitelist" class="form-control" />
+                </p>
+                <br />
                 <p>
                     <label id="types_label"></label><br />
                     <input type="text" id="types" value="" name="types" class="form-control" />

manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 2,
   "name": "ClearURLs",
-  "version": "1.28.0",
+  "version": "1.27.3",
   "author": "Kevin Roebert",
   "description": "__MSG_extension_description__",
   "homepage_url": "https://docs.clearurls.xyz",
@@ -270,18 +270,18 @@
         "*://*.google.co.zw/*",
         "*://*.google.cat/*"
       ],
-      "include_globs": [
-        "http?://www.google.*/",
+      "include_globs": [ 
+        "http?://www.google.*/", 
         "http?://www.google.*/#hl=*",
-        "http?://www.google.*/search*",
+        "http?://www.google.*/search*", 
         "http?://www.google.*/webhp?hl=*",
-        "https://encrypted.google.*/",
+        "https://encrypted.google.*/", 
         "https://encrypted.google.*/#hl=*",
-        "https://encrypted.google.*/search*",
+        "https://encrypted.gogole.*/search*", 
         "https://encrypted.google.*/webhp?hl=*",
-        "http?://ipv6.google.com/",
+        "http?://ipv6.google.com/", 
         "http?://ipv6.google.com/search*"
-      ],
+       ],
       "js": [
         "core_js/google_link_fix.js"
       ],
@@ -303,4 +303,4 @@
   "options_ui": {
     "page": "html/settings.html"
   }
-}
+}