Added tokenzied provider lookup

Reduce lookup overhead by using map for provider matching

CHANGELOG.md

@@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Require Chrome >= 37
 
 
-## [1.27.3] - 2025-XX-XX
+## [1.27.3] - 2025-02-05
 
 ### Changed
 - Google Search `window.rwt` detection
@@ -33,7 +33,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     - [#1302](https://gitlab.com/ClearURLs/ClearUrls/-/issues/1302)
     - [#1305](https://gitlab.com/ClearURLs/ClearUrls/-/issues/1305)
     
-
 ### Compatibility note
 - Require Firefox >= 55
 - Require Chrome >= 37

README.md

@@ -77,6 +77,8 @@ Please push your translation into the folder `_locales/{language code}/messages.
 * [Unalix](https://github.com/AmanoTeam/Unalix) small, dependency-free, fast Python package for removing tracking fields from URLs
 * [Unalix-nim](https://github.com/AmanoTeam/Unalix-nim) small, dependency-free, fast Nim package and CLI tool for removing tracking fields from URLs
 * [UnalixAndroid](https://github.com/AmanoTeam/UnalixAndroid) simple Android app that removes link masking/tracking and optionally resolves shortened links
+* [pl-fe](https://github.com/mkljczk/pl-fe) is a Fediverse client which uses ClearURLs code to clean URLs from displayed posts and recommend cleaning URLs from created posts
+* [URLCheck](https://github.com/TrianguloY/URLCheck) is an Android app to review and edit URLs before opening them. Allows to use the ClearURLs catalog.
 
 ## Recommended by...
 *  [ghacks-user.js](https://github.com/ghacksuserjs/ghacks-user.js/wiki/4.1-Extensions)
@@ -104,3 +106,12 @@ We use some third-party scripts in our add-on. The authors and licenses are list
     [MIT](https://github.com/Simonwep/pickr/blob/master/LICENSE)
 -   [Font Awesome](https://github.com/FortAwesome/Font-Awesome/) | Copyright (c) @fontawesome |
     [Font Awesome Free License](https://github.com/FortAwesome/Font-Awesome/blob/master/LICENSE.txt)
+
+
+## Star History
+
+ <picture>
+   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=ClearURLs/Addon&type=Date&theme=dark" />
+   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=ClearURLs/Addon&type=Date" />
+   <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=ClearURLs/Addon&type=Date" />
+ </picture>

_locales/en/messages.json

@@ -87,14 +87,6 @@
         "message": "Show numbers of cleaned urls",
         "description": "This string is used as title for the badges switch button on the popup page."
     },
-    "popup_html_configs_whitelist_button_add": {
-        "message": "Whitelist Site",
-        "description": "This string is used as name for the whitelist button on the popup page."
-    },
-    "popup_html_configs_whitelist_button_remove": {
-        "message": "Remove from Whitelist",
-        "description": "This string is used as name for the whitelist button on the popup page."
-    },
     "popup_html_statistics_head": {
         "message": "Statistics",
         "description": "This string is used as title for the statistics on the popup page."
@@ -187,10 +179,6 @@
         "message": "The url to the rules.hash file (hash)",
         "description": "This string is used as name for the rule.hash url label."
     },
-    "setting_whitelist_list_label": {
-        "message": "Whitelisted sites",
-        "description": "This string is used as name for the whitelisted sites list label."
-    },
     "setting_types_label": {
         "message": "<a href='https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webRequest/ResourceType' target='_blank'>Request types</a> (expert level)",
         "description": "This string is used as name for the types label."

clearurls.js

@@ -21,6 +21,8 @@
 * This script is responsible for the core functionalities.
 */
 var providers = [];
+var providersByToken = {}; // Map<string, Provider[]>
+var globalProviders = []; // Provider[]
 var prvKeys = [];
 var siteBlockedAlert = 'javascript:void(0)';
 var dataHash;
@@ -47,19 +49,6 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
     let rawRules = provider.getRawRules();
     let urlObject = new URL(url);
 
-    /*
-    * Skip whitelisted sites
-    */
-    for (const site of storage.whitelist) {
-        if (url.indexOf(site) !== -1) {
-            return {
-                "changes": false,
-                "url": url,
-                "cancel": false
-            }
-        }
-    }
-
     if (storage.localHostsSkipping && checkLocalURL(urlObject)) {
         return {
             "changes": false,
@@ -102,14 +91,14 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
     /*
     * Apply raw rules to the URL.
     */
-    rawRules.forEach(function (rawRule) {
+    rawRules.forEach(function ({ rule: rawRuleStr, regex: rawRuleRegex }) {
         let beforeReplace = url;
-        url = url.replace(new RegExp(rawRule, "gi"), "");
+        url = url.replace(rawRuleRegex, "");
 
         if (beforeReplace !== url) {
             //Log the action
             if (storage.loggingStatus && !quiet) {
-                pushToLog(beforeReplace, url, rawRule);
+                pushToLog(beforeReplace, url, rawRuleStr);
             }
 
             increaseBadged(quiet, request);
@@ -126,13 +115,13 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
      * Only test for matches, if there are fields or fragments that can be cleaned.
      */
     if (fields.toString() !== "" || fragments.toString() !== "") {
-        rules.forEach(rule => {
+        rules.forEach(({ rule, regex }) => {
             const beforeFields = fields.toString();
             const beforeFragments = fragments.toString();
             let localChange = false;
 
             for (const field of fields.keys()) {
-                if (new RegExp("^"+rule+"$", "gi").test(field)) {
+                if (regex.test(field)) {
                     fields.delete(field);
                     changes = true;
                     localChange = true;
@@ -140,7 +129,7 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) {
             }
 
             for (const fragment of fragments.keys()) {
-                if (new RegExp("^"+rule+"$", "gi").test(fragment)) {
+                if (regex.test(fragment)) {
                     fragments.delete(fragment);
                     changes = true;
                     localChange = true;
@@ -241,6 +230,17 @@ function start() {
             for (let re = 0; re < methods.length; re++) {
                 providers[p].addMethod(methods[re]);
             }
+
+            // Indexing logic
+            const token = providers[p].getLookupToken();
+            if (token) {
+                if (!providersByToken[token]) {
+                    providersByToken[token] = [];
+                }
+                providersByToken[token].push(providers[p]);
+            } else {
+                globalProviders.push(providers[p]);
+            }
         }
     }
 
@@ -373,7 +373,7 @@ function start() {
         let methods = [];
 
         if (_completeProvider) {
-            enabled_rules[".*"] = true;
+            enabled_rules[".*"] = new RegExp("^.*$", "i");
         }
 
         /**
@@ -392,6 +392,38 @@ function start() {
             return name;
         };
 
+        /**
+         * Returns the lookup token for this provider, or null if global.
+         * Extracts "domain" from patterns like ^https?://(?:[a-z0-9-]+\.)*?domain...
+         * @return {String|null}
+         */
+        this.getLookupToken = function () {
+            if (!urlPattern) return null;
+            const source = urlPattern.source;
+
+            // Case 1: Wildcard prefix pattern (e.g. ...*?amazon...)
+            const wildcardMatch = source.match(/\*\?([a-z0-9-]+)/i);
+            if (wildcardMatch && wildcardMatch[1]) {
+                return wildcardMatch[1].toLowerCase();
+            }
+
+            // Case 2: Explicit start pattern (e.g. ^https?://vk.com...)
+            // Matches ^https?://(optional www.)token
+            // We strip standard regex start structure to find the first meaningful domain token.
+            // This regex handles:
+            // - ^https?://  (start)
+            // - \/\/ or //  (slashes, potentially escaped)
+            // - www.        (optional www prefix, potentially escaped)
+            // - [a-z0-9-]+  (the token)
+            // It deliberately fails on patterns with groups (?:...) at the start, falling back to global.
+            const explicitMatch = source.match(/^(\^?https?:\\?\/\\?\/)(?:www(?:\\?\.))?([a-z0-9-]+)/i);
+            if (explicitMatch && explicitMatch[2]) {
+                return explicitMatch[2].toLowerCase();
+            }
+
+            return null;
+        };
+
         /**
          * Add URL pattern.
          *
@@ -419,25 +451,21 @@ function start() {
         };
 
         /**
-         * Apply a rule to a given tuple of rule array.
+         * Helper to update rule maps with compiled regexes.
-         * @param enabledRuleArray      array for enabled rules
-         * @param disabledRulesArray    array for disabled rules
-         * @param {String} rule         RegExp as string
-         * @param {boolean} isActive    Is this rule active?
          */
-        this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => {
+        const updateRule = (enabledMap, disabledMap, rule, isActive, compileFn) => {
             if (isActive) {
-                enabledRuleArray[rule] = true;
+                if (!enabledMap[rule]) {
-
+                    try {
-                if (disabledRulesArray[rule] !== undefined) {
+                        enabledMap[rule] = compileFn(rule);
-                    delete disabledRulesArray[rule];
+                    } catch (e) {
+                        console.error("Invalid regex", rule, e);
+                    }
                 }
+                if (disabledMap[rule]) delete disabledMap[rule];
             } else {
-                disabledRulesArray[rule] = true;
+                disabledMap[rule] = true;
-
+                if (enabledMap[rule]) delete enabledMap[rule];
-                if (enabledRuleArray[rule] !== undefined) {
-                    delete enabledRuleArray[rule];
-                }
             }
         };
 
@@ -449,20 +477,22 @@ function start() {
          * @param {boolean} isActive   Is this rule active?
          */
         this.addRule = function (rule, isActive = true) {
-            this.applyRule(enabled_rules, disabled_rules, rule, isActive);
+            updateRule(enabled_rules, disabled_rules, rule, isActive, r => new RegExp("^" + r + "$", "i"));
         };
 
         /**
-         * Return all active rules as an array.
+         * Return all active rules as an array of {rule, regex}.
          *
-         * @return Array RegExp strings
+         * @return Array Objects
          */
         this.getRules = function () {
+            let source = enabled_rules;
             if (!storage.referralMarketing) {
-                return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing));
+                // Determine if we need to merge referral marketing rules
+                // We use a new object to avoid mutating enabled_rules via Object.assign if that was happening
+                source = Object.assign({}, enabled_rules, enabled_referralMarketing);
             }
-
+            return Object.entries(source).map(([rule, regex]) => ({ rule, regex }));
-            return Object.keys(enabled_rules);
         };
 
         /**
@@ -473,16 +503,17 @@ function start() {
          * @param {boolean} isActive   Is this rule active?
          */
         this.addRawRule = function (rule, isActive = true) {
-            this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive);
+            updateRule(enabled_rawRules, disabled_rawRules, rule, isActive, r => new RegExp(r, "gi"));
         };
 
         /**
          * Return all active raw rules as an array.
          *
-         * @return Array RegExp strings
+         * @return Array Objects {rule, regex}
          */
         this.getRawRules = function () {
-            return Object.keys(enabled_rawRules);
+            // return Object.keys(enabled_rawRules);
+            return Object.entries(enabled_rawRules).map(([rule, regex]) => ({ rule, regex }));
         };
 
         /**
@@ -493,7 +524,7 @@ function start() {
          * @param {boolean} isActive   Is this rule active?
          */
         this.addReferralMarketing = function (rule, isActive = true) {
-            this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive);
+            updateRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive, r => new RegExp("^" + r + "$", "i"));
         };
 
         /**
@@ -504,19 +535,7 @@ function start() {
          * @param {Boolean} isActive   Is this exception active?
          */
         this.addException = function (exception, isActive = true) {
-            if (isActive) {
+            updateRule(enabled_exceptions, disabled_exceptions, exception, isActive, r => new RegExp(r, "i"));
-                enabled_exceptions[exception] = true;
-
-                if (disabled_exceptions[exception] !== undefined) {
-                    delete disabled_exceptions[exception];
-                }
-            } else {
-                disabled_exceptions[exception] = true;
-
-                if (enabled_exceptions[exception] !== undefined) {
-                    delete enabled_exceptions[exception];
-                }
-            }
         };
 
         /**
@@ -554,11 +573,9 @@ function start() {
             //Add the site blocked alert to every exception
             if (url === siteBlockedAlert) return true;
 
-            for (const exception in enabled_exceptions) {
+            for (const [exception, regex] of Object.entries(enabled_exceptions)) {
                 if (result) break;
-
+                result = regex.test(url);
-                let exception_regex = new RegExp(exception, "i");
-                result = exception_regex.test(url);
             }
 
             return result;
@@ -572,19 +589,7 @@ function start() {
          * @param {Boolean} isActive     Is this redirection active?
          */
         this.addRedirection = function (redirection, isActive = true) {
-            if (isActive) {
+            updateRule(enabled_redirections, disabled_redirections, redirection, isActive, r => new RegExp(r, "i"));
-                enabled_redirections[redirection] = true;
-
-                if (disabled_redirections[redirection] !== undefined) {
-                    delete disabled_redirections[redirection];
-                }
-            } else {
-                disabled_redirections[redirection] = true;
-
-                if (enabled_redirections[redirection] !== undefined) {
-                    delete enabled_redirections[redirection];
-                }
-            }
         };
 
         /**
@@ -595,11 +600,11 @@ function start() {
         this.getRedirection = function (url) {
             let re = null;
 
-            for (const redirection in enabled_redirections) {
+            for (const [redirection, regex] of Object.entries(enabled_redirections)) {
-                let result = (url.match(new RegExp(redirection, "i")));
+                let result = url.match(regex);
 
                 if (result && result.length > 0 && redirection) {
-                    re = (new RegExp(redirection, "i")).exec(url)[1];
+                    re = result[1];
 
                     break;
                 }
@@ -636,16 +641,42 @@ function start() {
                 pushToLog(request.url, request.url, translate('log_ping_blocked'));
                 increaseBadged(false, request);
                 increaseTotalCounter(1);
-                return {cancel: true};
+                return { cancel: true };
             }
 
+            let host = "";
+            try {
+                host = extractHost(new URL(request.url));
+            } catch (e) {
+                // If URL parsing fails, we falls back to empty host, relying on global providers or skipping
+            }
+
+            const hostTokens = host.split('.').map(t => t.toLowerCase());
+
+            // Collect candidate providers: Global + Key Matches
+            // Use a Set to avoid duplicates if multiple tokens map to same provider (unlikely but safe)
+            let candidateProviders = new Set(globalProviders);
+
+            for (const token of hostTokens) {
+                if (providersByToken[token]) {
+                    for (const p of providersByToken[token]) {
+                        candidateProviders.add(p);
+                    }
+                }
+            }
+
+            // "providers" global var is still used for legacy, but here we iterate candidates
+            // Converting Set to Array for iteration
+            const candidates = Array.from(candidateProviders);
+
             /*
             * Call for every provider the removeFieldsFormURL method.
             */
-            for (let i = 0; i < providers.length; i++) {
+            for (let i = 0; i < candidates.length; i++) {
-                if (!providers[i].matchMethod(request)) continue;
+                const provider = candidates[i];
-                if (providers[i].matchURL(request.url)) {
+                if (!provider.matchMethod(request)) continue;
-                    result = removeFieldsFormURL(providers[i], request.url, false, request);
+                if (provider.matchURL(request.url)) {
+                    result = removeFieldsFormURL(provider, request.url, false, request);
                 }
 
                 /*
@@ -653,10 +684,10 @@ function start() {
                 * Cancel the active request.
                 */
                 if (result.redirect) {
-                    if (providers[i].shouldForceRedirect() &&
+                    if (provider.shouldForceRedirect() &&
                         request.type === 'main_frame') {
-                        browser.tabs.update(request.tabId, {url: result.url}).catch(handleError);
+                        browser.tabs.update(request.tabId, { url: result.url }).catch(handleError);
-                        return {cancel: true};
+                        return { cancel: true };
                     }
 
                     return {
@@ -671,9 +702,9 @@ function start() {
                 if (result.cancel) {
                     if (request.type === 'main_frame') {
                         const blockingPage = browser.runtime.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url));
-                        browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError);
+                        browser.tabs.update(request.tabId, { url: blockingPage }).catch(handleError);
 
-                        return {cancel: true};
+                        return { cancel: true };
                     } else {
                         return {
                             redirectUrl: siteBlockedAlert
@@ -736,7 +767,7 @@ function start() {
      */
     browser.webRequest.onBeforeRequest.addListener(
         promise,
-        {urls: ["<all_urls>"], types: getData("types").concat(getData("pingRequestTypes"))},
+        { urls: ["<all_urls>"], types: getData("types").concat(getData("pingRequestTypes")) },
         ["blocking"]
     );
 }

core_js/popup.js

@@ -62,34 +62,6 @@ function changeStatistics()
     elTotal.textContent = totalCounter.toLocaleString();
 }
 
-/**
-* Set the whitelist button text
-*/
-function setWhitelistText()
-{
-    let element = document.getElementById('whitelist_btn');
-    let currentSite;
-    browser.tabs.query({active: true, currentWindow: true}, function(tabs) {
-        currentSite = tabs[0].url;
-    });
-
-    browser.runtime.sendMessage({
-        function: "getData",
-        params: ['whitelist']
-    }).then((data) => {
-        let siteFound = data.response.some(site => currentSite.indexOf(site) !== -1);
-        if (siteFound) {
-            element.classList.replace('btn-primary', 'btn-danger')
-            element.textContent = translate('popup_html_configs_whitelist_button_remove')
-            document.getElementById('whitelist_btn').onclick = () => {changeWhitelist(true)};
-        } else {
-            element.classList.replace('btn-danger', 'btn-primary')
-            element.textContent = translate('popup_html_configs_whitelist_button_add')
-            document.getElementById('whitelist_btn').onclick = () => {changeWhitelist(false)};
-        }
-    }).catch(handleError);
-}
-
 /**
 * Set the value for the hashStatus on startUp.
 */
@@ -183,36 +155,6 @@ function setSwitchButton(id, varname)
     element.checked = this[varname];
 }
 
-/**
-* Adds (or removes) the site the user is on to the whitelist
-* Whitelisted sites do not get filtered
-* @param {boolean} removeWl If true remove current site instead of adding
-*/
-function changeWhitelist(removeWl) {
-    let site;
-    browser.tabs.query({active: true, currentWindow: true}, function(tabs) { // Couldn't figure out how to access currentUrl var
-        site = tabs[0].url;                                                  // So this is used instead
-    });
-    browser.runtime.sendMessage({
-        function: "getData",
-        params: ['whitelist']
-    }).then((data) => {
-        let siteUrl = new URL(site)
-        let domain = siteUrl.hostname
-        if (removeWl) {
-            data.response = data.response.filter(wlSite => wlSite !== domain)
-        } else {
-            data.response.push(domain)
-        }
-        browser.runtime.sendMessage({
-            function: "setData",
-            params: ['whitelist', data.response]
-        }).then(() => {
-            setWhitelistText();
-        }).catch(handleError);
-    }).catch(handleError);
-}
-
 /**
 * Reset the global statistic
 */
@@ -278,7 +220,6 @@ function setText()
     injectText('configs_switch_filter','popup_html_configs_switch_filter');
     injectText('configs_head','popup_html_configs_head');
     injectText('configs_switch_statistics','configs_switch_statistics');
-    setWhitelistText();
     document.getElementById('donate').title = translate('donate_button');
 }
 

core_js/settings.js

@@ -82,7 +82,6 @@ function save() {
     saveData("badged_color", pickr.getColor().toHEXA().toString())
         .then(() => saveData("ruleURL", document.querySelector('input[name=ruleURL]').value))
         .then(() => saveData("hashURL", document.querySelector('input[name=hashURL]').value))
-        .then(() => saveData("whitelist", document.querySelector('input[name=whitelist]').value.split(',')))
         .then(() => saveData("types", document.querySelector('input[name=types]').value))
         .then(() => saveData("logLimit", Math.max(0, Math.min(5000, document.querySelector('input[name=logLimit]').value))))
         .then(() => browser.runtime.sendMessage({
@@ -123,7 +122,6 @@ function getData() {
 
     loadData("ruleURL")
         .then(() => loadData("hashURL"))
-        .then(() => loadData("whitelist"))
         .then(() => loadData("types"))
         .then(() => loadData("logLimit"))
         .then(logData => {
@@ -218,7 +216,6 @@ function setText() {
     document.getElementById('reset_settings_btn').setAttribute('title', translate('setting_html_reset_button_title'));
     document.getElementById('rule_url_label').textContent = translate('setting_rule_url_label');
     document.getElementById('hash_url_label').textContent = translate('setting_hash_url_label');
-    document.getElementById('whitelist_list_label').textContent = translate('setting_whitelist_list_label');
     document.getElementById('types_label').innerHTML = translate('setting_types_label');
     document.getElementById('save_settings_btn').textContent = translate('settings_html_save_button');
     document.getElementById('save_settings_btn').setAttribute('title', translate('settings_html_save_button_title'));

core_js/storage.js

@@ -216,11 +216,10 @@ function initSettings() {
     storage.badged_color = "#ffa500";
     storage.hashURL = "https://rules2.clearurls.xyz/rules.minify.hash";
     storage.ruleURL = "https://rules2.clearurls.xyz/data.minify.json";
-    storage.whitelist = []; // TODO: If we do whitelist per rule, this needs to be obj
     storage.contextMenuEnabled = true;
     storage.historyListenerEnabled = true;
     storage.localHostsSkipping = true;
-    storage.referralMarketing = false;
+    storage.referralMarketing = true;
     storage.logLimit = 100;
     storage.domainBlocking = true;
     storage.pingBlocking = true;

html/popup.html

@@ -86,9 +86,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                         <label id="configs_switch_statistics"></label>
                     </label>
                     <div class="clearfix"></div>
-                    <div class="text-center">
-                        <button type="button" id="whitelist_btn" class="btn btn-primary btn-sm text-wrap"></button>
-                    </div>
                     <br />
                 </div>
             </div>

html/settings.html

@@ -105,11 +105,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                     <input type="url" id="hashURL" value="" name="hashURL" class="form-control" />
                 </p>
                 <br />
-                <p>
-                    <label id="whitelist_list_label"></label><br />
-                    <input type="text" id="whitelist" value="" name="whitelist" class="form-control" />
-                </p>
-                <br />
                 <p>
                     <label id="types_label"></label><br />
                     <input type="text" id="types" value="" name="types" class="form-control" />

manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 2,
   "name": "ClearURLs",
-  "version": "1.27.3",
+  "version": "1.28.0",
   "author": "Kevin Roebert",
   "description": "__MSG_extension_description__",
   "homepage_url": "https://docs.clearurls.xyz",
@@ -277,11 +277,11 @@
         "http?://www.google.*/webhp?hl=*",
         "https://encrypted.google.*/",
         "https://encrypted.google.*/#hl=*",
-        "https://encrypted.gogole.*/search*", 
+        "https://encrypted.google.*/search*",
         "https://encrypted.google.*/webhp?hl=*",
         "http?://ipv6.google.com/",
         "http?://ipv6.google.com/search*"
-       ],
+      ],
       "js": [
         "core_js/google_link_fix.js"
       ],