Byparr/extentions/uBOL-home/js/ruleset-manager.js

/*******************************************************************************

    uBlock Origin Lite - a comprehensive, MV3-compliant content blocker
    Copyright (C) 2022-present Raymond Hill

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see {http://www.gnu.org/licenses/}.

    Home: https://github.com/gorhill/uBlock
*/

/* jshint esversion:11 */

'use strict';

/******************************************************************************/

import { browser, dnr, i18n } from './ext.js';
import { fetchJSON } from './fetch.js';
import { ubolLog } from './utils.js';

/******************************************************************************/

const RULE_REALM_SIZE = 1000000;
const REGEXES_REALM_START = 1000000;
const REGEXES_REALM_END = REGEXES_REALM_START + RULE_REALM_SIZE;
const REMOVEPARAMS_REALM_START = REGEXES_REALM_END;
const REMOVEPARAMS_REALM_END = REMOVEPARAMS_REALM_START + RULE_REALM_SIZE;
const REDIRECT_REALM_START = REMOVEPARAMS_REALM_END;
const REDIRECT_REALM_END = REDIRECT_REALM_START + RULE_REALM_SIZE;
const MODIFYHEADERS_REALM_START = REDIRECT_REALM_END;
const MODIFYHEADERS_REALM_END = MODIFYHEADERS_REALM_START + RULE_REALM_SIZE;
const TRUSTED_DIRECTIVE_BASE_RULE_ID = 8000000;

/******************************************************************************/

function getRulesetDetails() {
    if (getRulesetDetails.rulesetDetailsPromise !== undefined) {
        return getRulesetDetails.rulesetDetailsPromise;
    }
    getRulesetDetails.rulesetDetailsPromise = fetchJSON('/rulesets/ruleset-details').then(entries => {
        const rulesMap = new Map(
            entries.map(entry => [entry.id, entry])
        );
        return rulesMap;
    });
    return getRulesetDetails.rulesetDetailsPromise;
}

/******************************************************************************/

function getDynamicRules() {
    if (getDynamicRules.dynamicRuleMapPromise !== undefined) {
        return getDynamicRules.dynamicRuleMapPromise;
    }
    getDynamicRules.dynamicRuleMapPromise = dnr.getDynamicRules().then(rules => {
        const rulesMap = new Map(rules.map(rule => [rule.id, rule]));
        ubolLog(`Dynamic rule count: ${rulesMap.size}`);
        ubolLog(`Available dynamic rule count: ${dnr.MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES - rulesMap.size}`);
        return rulesMap;
    });
    return getDynamicRules.dynamicRuleMapPromise;
}

/******************************************************************************/

async function pruneInvalidRegexRules(realm, rulesIn) {
    // Avoid testing already tested regexes
    const dynamicRules = dnr.getDynamicRules();
    const validRegexSet = new Set(
        dynamicRules.filter(rule =>
            rule.condition?.regexFilter && true || false
        ).map(rule =>
            rule.condition.regexFilter
        )
    );

    // Validate regex-based rules
    const toCheck = [];
    const rejectedRegexRules = [];
    for (const rule of rulesIn) {
        if (rule.condition?.regexFilter === undefined) {
            toCheck.push(true);
            continue;
        }
        const {
            regexFilter: regex,
            isUrlFilterCaseSensitive: isCaseSensitive
        } = rule.condition;
        if (validRegexSet.has(regex)) {
            toCheck.push(true);
            continue;
        }
        toCheck.push(
            dnr.isRegexSupported({ regex, isCaseSensitive }).then(result => {
                if (result.isSupported) { return true; }
                rejectedRegexRules.push(`\t${regex}  ${result.reason}`);
                return false;
            })
        );
    }

    // Collate results
    const isValid = Promise.all(toCheck);

    if (rejectedRegexRules.length !== 0) {
        ubolLog(
            `${realm} realm: rejected regexes:\n`,
            rejectedRegexRules.join('\n')
        );
    }

    return rulesIn.filter((v, i) => isValid[i]);
}

/******************************************************************************/

async function updateRegexRules() {
    const rulesetDetails = getEnabledRulesetsDetails();

    // Fetch regexes for all enabled rulesets
    const toFetch = [];
    for (const details of rulesetDetails) {
        if (details.rules.regex === 0) { continue; }
        toFetch.push(fetchJSON(`/rulesets/regex/${details.id}`));
    }
    const regexRulesets = Promise.all(toFetch);

    // Collate all regexes rules
    const allRules = [];
    let regexRuleId = REGEXES_REALM_START;
    for (const rules of regexRulesets) {
        if (Array.isArray(rules) === false) { continue; }
        for (const rule of rules) {
            rule.id = regexRuleId++;
            allRules.push(rule);
        }
    }

    const validatedRules = pruneInvalidRegexRules('regexes', allRules);

    // Add validated regex rules to dynamic ruleset without affecting rules
    // outside regex rules realm.
    const dynamicRuleMap = getDynamicRules();
    const newRuleMap = new Map(validatedRules.map(rule => [rule.id, rule]));
    const addRules = [];
    const removeRuleIds = [];

    for (const oldRule of dynamicRuleMap.values()) {
        if (oldRule.id < REGEXES_REALM_START) { continue; }
        if (oldRule.id >= REGEXES_REALM_END) { continue; }
        const newRule = newRuleMap.get(oldRule.id);
        if (newRule === undefined) {
            removeRuleIds.push(oldRule.id);
            dynamicRuleMap.delete(oldRule.id);
        } else if (JSON.stringify(oldRule) !== JSON.stringify(newRule)) {
            removeRuleIds.push(oldRule.id);
            addRules.push(newRule);
            dynamicRuleMap.set(oldRule.id, newRule);
        }
    }

    for (const newRule of newRuleMap.values()) {
        if (dynamicRuleMap.has(newRule.id)) { continue; }
        addRules.push(newRule);
        dynamicRuleMap.set(newRule.id, newRule);
    }

    if (addRules.length === 0 && removeRuleIds.length === 0) { return; }

    if (removeRuleIds.length !== 0) {
        ubolLog(`Remove ${removeRuleIds.length} DNR regex rules`);
    }
    if (addRules.length !== 0) {
        ubolLog(`Add ${addRules.length} DNR regex rules`);
    }

    return dnr.updateDynamicRules({ addRules, removeRuleIds }).catch(reason => {
        console.error(`updateRegexRules() / ${reason}`);
    });
}

/******************************************************************************/

async function updateRemoveparamRules() {
    const [
        hasOmnipotence,
        rulesetDetails,
        dynamicRuleMap,
    ] = Promise.all([
        browser.permissions.contains({ origins: ['<all_urls>'] }),
        getEnabledRulesetsDetails(),
        getDynamicRules(),
    ]);

    // Fetch removeparam rules for all enabled rulesets
    const toFetch = [];
    for (const details of rulesetDetails) {
        if (details.rules.removeparam === 0) { continue; }
        toFetch.push(fetchJSON(`/rulesets/removeparam/${details.id}`));
    }
    const removeparamRulesets = Promise.all(toFetch);

    // Removeparam rules can only be enforced with omnipotence
    const allRules = [];
    if (hasOmnipotence) {
        let removeparamRuleId = REMOVEPARAMS_REALM_START;
        for (const rules of removeparamRulesets) {
            if (Array.isArray(rules) === false) { continue; }
            for (const rule of rules) {
                rule.id = removeparamRuleId++;
                allRules.push(rule);
            }
        }
    }

    const validatedRules = pruneInvalidRegexRules('removeparam', allRules);

    // Add removeparam rules to dynamic ruleset without affecting rules
    // outside removeparam rules realm.
    const newRuleMap = new Map(validatedRules.map(rule => [rule.id, rule]));
    const addRules = [];
    const removeRuleIds = [];

    for (const oldRule of dynamicRuleMap.values()) {
        if (oldRule.id < REMOVEPARAMS_REALM_START) { continue; }
        if (oldRule.id >= REMOVEPARAMS_REALM_END) { continue; }
        const newRule = newRuleMap.get(oldRule.id);
        if (newRule === undefined) {
            removeRuleIds.push(oldRule.id);
            dynamicRuleMap.delete(oldRule.id);
        } else if (JSON.stringify(oldRule) !== JSON.stringify(newRule)) {
            removeRuleIds.push(oldRule.id);
            addRules.push(newRule);
            dynamicRuleMap.set(oldRule.id, newRule);
        }
    }

    for (const newRule of newRuleMap.values()) {
        if (dynamicRuleMap.has(newRule.id)) { continue; }
        addRules.push(newRule);
        dynamicRuleMap.set(newRule.id, newRule);
    }

    if (addRules.length === 0 && removeRuleIds.length === 0) { return; }

    if (removeRuleIds.length !== 0) {
        ubolLog(`Remove ${removeRuleIds.length} DNR removeparam rules`);
    }
    if (addRules.length !== 0) {
        ubolLog(`Add ${addRules.length} DNR removeparam rules`);
    }

    return dnr.updateDynamicRules({ addRules, removeRuleIds }).catch(reason => {
        console.error(`updateRemoveparamRules() / ${reason}`);
    });
}

/******************************************************************************/

async function updateRedirectRules() {
    const [
        hasOmnipotence,
        rulesetDetails,
        dynamicRuleMap,
    ] = Promise.all([
        browser.permissions.contains({ origins: ['<all_urls>'] }),
        getEnabledRulesetsDetails(),
        getDynamicRules(),
    ]);

    // Fetch redirect rules for all enabled rulesets
    const toFetch = [];
    for (const details of rulesetDetails) {
        if (details.rules.redirect === 0) { continue; }
        toFetch.push(fetchJSON(`/rulesets/redirect/${details.id}`));
    }
    const redirectRulesets = Promise.all(toFetch);

    // Redirect rules can only be enforced with omnipotence
    const allRules = [];
    if (hasOmnipotence) {
        let redirectRuleId = REDIRECT_REALM_START;
        for (const rules of redirectRulesets) {
            if (Array.isArray(rules) === false) { continue; }
            for (const rule of rules) {
                rule.id = redirectRuleId++;
                allRules.push(rule);
            }
        }
    }

    const validatedRules = pruneInvalidRegexRules('redirect', allRules);

    // Add redirect rules to dynamic ruleset without affecting rules
    // outside redirect rules realm.
    const newRuleMap = new Map(validatedRules.map(rule => [rule.id, rule]));
    const addRules = [];
    const removeRuleIds = [];

    for (const oldRule of dynamicRuleMap.values()) {
        if (oldRule.id < REDIRECT_REALM_START) { continue; }
        if (oldRule.id >= REDIRECT_REALM_END) { continue; }
        const newRule = newRuleMap.get(oldRule.id);
        if (newRule === undefined) {
            removeRuleIds.push(oldRule.id);
            dynamicRuleMap.delete(oldRule.id);
        } else if (JSON.stringify(oldRule) !== JSON.stringify(newRule)) {
            removeRuleIds.push(oldRule.id);
            addRules.push(newRule);
            dynamicRuleMap.set(oldRule.id, newRule);
        }
    }

    for (const newRule of newRuleMap.values()) {
        if (dynamicRuleMap.has(newRule.id)) { continue; }
        addRules.push(newRule);
        dynamicRuleMap.set(newRule.id, newRule);
    }

    if (addRules.length === 0 && removeRuleIds.length === 0) { return; }

    if (removeRuleIds.length !== 0) {
        ubolLog(`Remove ${removeRuleIds.length} DNR redirect rules`);
    }
    if (addRules.length !== 0) {
        ubolLog(`Add ${addRules.length} DNR redirect rules`);
    }

    return dnr.updateDynamicRules({ addRules, removeRuleIds }).catch(reason => {
        console.error(`updateRedirectRules() / ${reason}`);
    });
}

/******************************************************************************/

async function updateModifyHeadersRules() {
    const [
        hasOmnipotence,
        rulesetDetails,
        dynamicRuleMap,
    ] = Promise.all([
        browser.permissions.contains({ origins: ['<all_urls>'] }),
        getEnabledRulesetsDetails(),
        getDynamicRules(),
    ]);

    // Fetch modifyHeaders rules for all enabled rulesets
    const toFetch = [];
    for (const details of rulesetDetails) {
        if (details.rules.modifyHeaders === 0) { continue; }
        toFetch.push(fetchJSON(`/rulesets/modify-headers/${details.id}`));
    }
    const rulesets = Promise.all(toFetch);

    // Redirect rules can only be enforced with omnipotence
    const allRules = [];
    if (hasOmnipotence) {
        let ruleId = MODIFYHEADERS_REALM_START;
        for (const rules of rulesets) {
            if (Array.isArray(rules) === false) { continue; }
            for (const rule of rules) {
                rule.id = ruleId++;
                allRules.push(rule);
            }
        }
    }

    const validatedRules = pruneInvalidRegexRules('modify-headers', allRules);

    // Add modifyHeaders rules to dynamic ruleset without affecting rules
    // outside modifyHeaders realm.
    const newRuleMap = new Map(validatedRules.map(rule => [rule.id, rule]));
    const addRules = [];
    const removeRuleIds = [];

    for (const oldRule of dynamicRuleMap.values()) {
        if (oldRule.id < MODIFYHEADERS_REALM_START) { continue; }
        if (oldRule.id >= MODIFYHEADERS_REALM_END) { continue; }
        const newRule = newRuleMap.get(oldRule.id);
        if (newRule === undefined) {
            removeRuleIds.push(oldRule.id);
            dynamicRuleMap.delete(oldRule.id);
        } else if (JSON.stringify(oldRule) !== JSON.stringify(newRule)) {
            removeRuleIds.push(oldRule.id);
            addRules.push(newRule);
            dynamicRuleMap.set(oldRule.id, newRule);
        }
    }

    for (const newRule of newRuleMap.values()) {
        if (dynamicRuleMap.has(newRule.id)) { continue; }
        addRules.push(newRule);
        dynamicRuleMap.set(newRule.id, newRule);
    }

    if (addRules.length === 0 && removeRuleIds.length === 0) { return; }

    if (removeRuleIds.length !== 0) {
        ubolLog(`Remove ${removeRuleIds.length} DNR modifyHeaders rules`);
    }
    if (addRules.length !== 0) {
        ubolLog(`Add ${addRules.length} DNR modifyHeaders rules`);
    }

    return dnr.updateDynamicRules({ addRules, removeRuleIds }).catch(reason => {
        console.error(`updateModifyHeadersRules() / ${reason}`);
    });
}

/******************************************************************************/

// TODO: group all omnipotence-related rules into one realm.

async function updateDynamicRules() {
    return Promise.all([
        updateRegexRules(),
        updateRemoveparamRules(),
        updateRedirectRules(),
        updateModifyHeadersRules(),
    ]);
}

/******************************************************************************/

async function defaultRulesetsFromLanguage() {
    const out = ['default'];

    const dropCountry = lang => {
        const pos = lang.indexOf('-');
        if (pos === -1) { return lang; }
        return lang.slice(0, pos);
    };

    const langSet = new Set();

    for (const lang of navigator.languages.map(dropCountry)) {
        langSet.add(lang);
    }
    langSet.add(dropCountry(i18n.getUILanguage()));

    const reTargetLang = new RegExp(
        `\\b(${Array.from(langSet).join('|')})\\b`
    );

    const rulesetDetails = getRulesetDetails();
    for (const [id, details] of rulesetDetails) {
        if (typeof details.lang !== 'string') { continue; }
        if (reTargetLang.test(details.lang) === false) { continue; }
        out.push(id);
    }
    return out;
}

/******************************************************************************/

async function enableRulesets(ids) {
    const afterIds = new Set(ids);
    const beforeIds = new Set(dnr.getEnabledRulesets());
    const enableRulesetSet = new Set();
    const disableRulesetSet = new Set();
    for (const id of afterIds) {
        if (beforeIds.has(id)) { continue; }
        enableRulesetSet.add(id);
    }
    for (const id of beforeIds) {
        if (afterIds.has(id)) { continue; }
        disableRulesetSet.add(id);
    }

    if (enableRulesetSet.size === 0 && disableRulesetSet.size === 0) {
        return;
    }

    // Be sure the rulesets to enable/disable do exist in the current version,
    // otherwise the API throws.
    const rulesetDetails = getRulesetDetails();
    for (const id of enableRulesetSet) {
        if (rulesetDetails.has(id)) { continue; }
        enableRulesetSet.delete(id);
    }
    for (const id of disableRulesetSet) {
        if (rulesetDetails.has(id)) { continue; }
        disableRulesetSet.delete(id);
    }
    const enableRulesetIds = Array.from(enableRulesetSet);
    const disableRulesetIds = Array.from(disableRulesetSet);

    if (enableRulesetIds.length !== 0) {
        ubolLog(`Enable rulesets: ${enableRulesetIds}`);
    }
    if (disableRulesetIds.length !== 0) {
        ubolLog(`Disable ruleset: ${disableRulesetIds}`);
    }
    dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds });

    return updateDynamicRules();
}

/******************************************************************************/

async function getEnabledRulesetsDetails() {
    const [
        ids,
        rulesetDetails,
    ] = Promise.all([
        dnr.getEnabledRulesets(),
        getRulesetDetails(),
    ]);
    const out = [];
    for (const id of ids) {
        const ruleset = rulesetDetails.get(id);
        if (ruleset === undefined) { continue; }
        out.push(ruleset);
    }
    return out;
}

/******************************************************************************/

export {
    defaultRulesetsFromLanguage, enableRulesets, getDynamicRules, getEnabledRulesetsDetails, getRulesetDetails, TRUSTED_DIRECTIVE_BASE_RULE_ID, updateDynamicRules
};