From 847bab7e421656043010ccb401a830def69b775b Mon Sep 17 00:00:00 2001 From: uBlock <35694050+uBlockAdmin@users.noreply.github.com> Date: Fri, 24 May 2019 11:45:42 +0530 Subject: [PATCH] 1. Special handling is given to filters which had to match against document origin only. The filter will only be considered if it satisfies the given criteria. - Should be in the form of `/czf*.` (without token) or `|https://` or `|http://` or `*` - Should have `domain=` option - Should not have a negated domain in its `domain=` option - Should not have `csp=` option examples: |http://$image,script,subdocument,third-party,xmlhttprequest,domain=dwindly.io |https://$image,other,script,stylesheet,third-party,domain=movies123.xyz $websocket,domain=povw1deo.com|povwideo.net|powvideo.net /czf*.$image,domain=100percentfedup.com These filters had to be matched against each network request which was causing extra time. Now, these filters will only be considered when document origin matches the domain mentioned in the filter `domain=` option. 2. Added google keyword to badTokens list 3. Replaced substr with startWith method for string pattern matching 4. Validated regular expression based filter 5. Code optimizations --- platform/chromium/manifest.json | 2 +- src/js/background.js | 4 +- src/js/messaging.js | 1 + src/js/pagestore.js | 5 + src/js/static-net-filtering.js | 695 +++++++++++++++----------------- src/js/tab.js | 16 +- src/js/traffic.js | 5 + 7 files changed, 360 insertions(+), 368 deletions(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index be6c012f7..27013c055 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock", - "version": "0.9.5.15", + "version": "0.9.5.16", "default_locale": "en", "description": "__MSG_extShortDesc__", diff --git a/src/js/background.js b/src/js/background.js index 84652b40e..15c0f2037 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -91,8 +91,8 @@ return { // read-only systemSettings: { - compiledMagic: 'eopszukpsddd', - selfieMagic: 'menhiasrsddd' + compiledMagic: 'krpszukpsddd', + selfieMagic: 'ednhiasrsddd' }, restoreBackupSettings: { lastRestoreFile: '', diff --git a/src/js/messaging.js b/src/js/messaging.js index 352486e40..36015706f 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -477,6 +477,7 @@ var filterRequests = function(pageStore, details) { request = requests[i]; context.requestURL = vAPI.punycodeURL(request.url); context.requestHostname = µburi.hostnameFromURI(request.url); + context.requestDomain = µburi.domainFromHostname(context.requestHostname); context.requestType = tagNameToRequestTypeMap[request.tagName]; if ( isBlockResult(pageStore.filterRequest(context)) ) { request.collapse = true; diff --git a/src/js/pagestore.js b/src/js/pagestore.js index 5bbe5de5f..9beb8fb81 100644 --- a/src/js/pagestore.js +++ b/src/js/pagestore.js @@ -251,6 +251,7 @@ FrameStore.prototype.init = function(rootHostname, frameURL) { this.pageURL = frameURL; this.pageHostname = µburi.hostnameFromURI(frameURL); this.pageDomain = µburi.domainFromHostname(this.pageHostname) || this.pageHostname; + this.pageHostnameHashes = µb.getHostnameHashesFromLabelsBackward(this.pageHostname, this.pageDomain, false); return this; }; @@ -440,6 +441,7 @@ PageStore.prototype.createContextFromPage = function() { var context = new µb.tabContextManager.createContext(this.tabId); context.pageHostname = context.rootHostname; context.pageDomain = context.rootDomain; + context.pageHostnameHashes = context.rootHostnameHashes; return context; }; @@ -449,9 +451,11 @@ PageStore.prototype.createContextFromFrameId = function(frameId) { var frameStore = this.frames[frameId]; context.pageHostname = frameStore.pageHostname; context.pageDomain = frameStore.pageDomain; + context.pageHostnameHashes = frameStore.pageHostnameHashes; } else { context.pageHostname = context.rootHostname; context.pageDomain = context.rootDomain; + context.pageHostnameHashes = context.rootHostnameHashes; } return context; }; @@ -460,6 +464,7 @@ PageStore.prototype.createContextFromFrameHostname = function(frameHostname) { var context = new µb.tabContextManager.createContext(this.tabId); context.pageHostname = frameHostname; context.pageDomain = µb.URI.domainFromHostname(frameHostname) || frameHostname; + context.pageHostnameHashes = µb.getHostnameHashesFromLabelsBackward(context.pageHostname, context.pageDomain, false); return context; }; diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index f16396845..c394947ab 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -164,6 +164,15 @@ //console.debug('µBlock.staticNetFilteringEngine: created RegExp("%s")', reStr); return new RegExp(reStr); }; + const isValidRegEx = function(s) { + var isValid = true; + try { + new RegExp(s); + } catch(e) { + isValid = false; + } + return isValid; + }; /******************************************************************************* @@ -339,8 +348,11 @@ return hash >>> 0; }, match: function(details, hostnameOnlyFilter) { - let hostnamesView = new Uint32Array(this.objView.buffer.buffer, details['+'].offset, details['+'].length); - let notHostnamesView = new Uint32Array(this.objView.buffer.buffer, details['-'].offset, details['-'].length); + let hostnamesView, notHostnamesView; + if(details['+'].length > 0) + hostnamesView = new Uint32Array(this.objView.buffer.buffer, details['+'].offset, details['+'].length); + if(details['-'].length > 0) + notHostnamesView = new Uint32Array(this.objView.buffer.buffer, details['-'].offset, details['-'].length); let hostHashes; let blnStatus = false; let matchHostname = ''; @@ -349,7 +361,7 @@ } else { hostHashes = pageHostnameHashes; } - if(hostnamesView.length == 0) { + if(details['+'].length == 0) { blnStatus = true; } else { Array.from(hostHashes).some(function(element) { @@ -362,9 +374,9 @@ } }); } - if(blnStatus) + if(blnStatus && details['-'].length > 0) blnStatus = (blnStatus && (Array.from(hostHashes).some(element => µb.binSearch(notHostnamesView, element[0]) !== -1) === false)); - + return blnStatus +'|'+ matchHostname; } }; @@ -404,13 +416,38 @@ }; /******************************************************************************/ + var FilterPair = function(f0, f1) { + this.f0 = f0; + this.f1 = f1; + } + FilterPair.prototype.match = function(url, tokenBeg) { + return this.f0.match(url, tokenBeg) === true && this.f1.match(); + } + FilterPair.fid = FilterPair.prototype.fid = 'fp'; + + FilterPair.prototype.toString = function() { + return this.f0.toString() + "," + this.f1.toString(); + }; + FilterPair.prototype.toSelfie = function() { + return [this.f0.fid, this.f0.toSelfie(), this.f1.toSelfie()]; + }; + FilterPair.fromSelfie = function(s) { + let factory = FilterContainer.factories[s[0]]; + const f0 = factory.fromSelfie(s[1]); + const f1 = FilterDomain.fromSelfie(s[2]); + return new FilterPair(f0, f1); + }; + FilterPair.prototype.toJSON = function() { + return {[this.fid]: this.toSelfie()}; + } + var FilterPlain = function(s, tokenBeg) { this.s = s; this.tokenBeg = tokenBeg; }; FilterPlain.prototype.match = function(url, tokenBeg) { - return url.substr(tokenBeg - this.tokenBeg, this.s.length) === this.s; + return url.startsWith(this.s, tokenBeg - this.tokenBeg); }; FilterPlain.fid = FilterPlain.prototype.fid = 'a'; @@ -442,7 +479,7 @@ }; FilterPlainPrefix0.prototype.match = function(url, tokenBeg) { - return url.substr(tokenBeg, this.s.length) === this.s; + return url.startsWith(this.s, tokenBeg); }; FilterPlainPrefix0.fid = FilterPlainPrefix0.prototype.fid = '0a'; @@ -471,7 +508,7 @@ }; FilterPlainPrefix1.prototype.match = function(url, tokenBeg) { - return url.substr(tokenBeg - 1, this.s.length) === this.s; + return url.startsWith(this.s, tokenBeg - 1); }; FilterPlainPrefix1.fid = FilterPlainPrefix1.prototype.fid = '1a'; @@ -501,7 +538,7 @@ }; FilterPlainLeftAnchored.prototype.match = function(url) { - return url.slice(0, this.s.length) === this.s; + return url.startsWith(this.s); }; FilterPlainLeftAnchored.fid = FilterPlainLeftAnchored.prototype.fid = '|a'; @@ -530,7 +567,7 @@ this.s = s; }; FilterPlainRightAnchored.prototype.match = function(url) { - return url.slice(-this.s.length) === this.s; + return url.endsWith(this.s); }; FilterPlainRightAnchored.fid = FilterPlainRightAnchored.prototype.fid = 'a|'; @@ -564,7 +601,7 @@ this.s = s; }; FilterPlainHnAnchored.prototype.match = function(url, tokenBeg) { - if ( url.substr(tokenBeg, this.s.length) !== this.s ) { + if ( !url.startsWith(this.s, tokenBeg) ) { return false; } // Valid only if hostname-valid characters to the left of token @@ -837,30 +874,15 @@ let filters = this.filters; let n = filters.length; for ( let i = 0; i < n; i++ ) { - if(Array.isArray(filters[i])) { - let f0 = filters[i][0]; - let f1 = filters[i][1]; - if(f0.fid.indexOf('h') === -1 && skipGenericBlocking) { - continue; - } - if ( f0.match(url, tokenBeg) !== false && f1.match()) { - this.f = filters[i]; - if ( i >= this.vip ) { - this.promote(i); - } - return true; - } - } else { - if(filters[i].fid.indexOf('h') === -1 && skipGenericBlocking) { - continue; - } - if ( filters[i].match(url, tokenBeg) !== false) { - this.f = filters[i]; - if ( i >= this.vip ) { - this.promote(i); - } - return true; + if(filters[i].fid.indexOf('h') === -1 && skipGenericBlocking) { + continue; + } + if ( filters[i].match(url, tokenBeg) !== false) { + this.f = filters[i]; + if ( i >= this.vip ) { + this.promote(i); } + return true; } } return false; @@ -870,11 +892,7 @@ FilterBucket.prototype.toString = function() { if ( this.f !== null ) { - if(Array.isArray(this.f)) { - return this.f[0].toString() + this.f[1].toString(); - } else { - return this.f.toString(); - } + return this.f.toString(); } return ''; }; @@ -882,12 +900,7 @@ FilterBucket.prototype.toSelfie = function() { let str = this.filters.map( function(x) { - if(Array.isArray(x)) { - return [{[x[0].fid]:x[0].toSelfie()}, {[x[1].fid]:x[1].toSelfie()}] - } - else { - return {[x.fid]:x.toSelfie()} - } + return {[x.fid]:x.toSelfie()} } ); return JSON.stringify(str); @@ -900,21 +913,11 @@ for(let key in o) { let item = o[key]; let prop, value; - if(Array.isArray(item)) { - let arrF = []; - for(let p in item) { - prop = Object.keys(item[p])[0]; - value = Object.values(item[p])[0]; - arrF.push(FilterContainer.factories[prop].fromSelfie(value)); - } - arr.push(arrF); - } else { - prop = Object.keys(item)[0]; - value = Object.values(item)[0]; - let filter = FilterContainer.factories[prop].fromSelfie(value); - arr.push(filter); - } - } + prop = Object.keys(item)[0]; + value = Object.values(item)[0]; + let filter = FilterContainer.factories[prop].fromSelfie(value); + arr.push(filter); + } f.filters = arr; return f; }; @@ -1203,6 +1206,9 @@ if ( s.charAt(0) === '/' && s.slice(-1) === '/' && s.length > 2 ) { this.isRegex = true; this.f = s.slice(1, -1); + if (!isValidRegEx(this.f) ) { + this.unsupported = true; + } return this; } @@ -1276,6 +1282,7 @@ var badTokens = { 'com': true, + 'google': true, 'http': true, 'https': true, 'icon': true, @@ -1319,7 +1326,7 @@ FilterParser.prototype.makeToken = function() { if ( this.isRegex ) { - this.token = '*'; + this.token = µb.tokenHash('*r'); return; } @@ -1328,9 +1335,17 @@ // https://github.com/uBlockAdmin/uBlock/issues/1038 // Match any URL. if ( s === '*' ) { - this.token = '*'; + this.token = µb.tokenHash('*'); + return; + } + else if(s == "https://") { + this.token = µb.tokenHash('https://'); return; } + else if(s == "http://") { + this.token = µb.tokenHash('http://'); + return; + } let matches; @@ -1348,7 +1363,7 @@ matches = findFirstGoodToken(s); if ( matches === null || matches[0].length === 0 ) { - this.token = '*'; + this.token = µb.tokenHash('~'); return; } this.tokenBeg = matches.index; @@ -1371,6 +1386,12 @@ var FilterContainer = function() { this.reAnyToken = /[%0-9a-z]+/g; + this.httpsTokenHash = µb.tokenHash('https://'); + this.httpTokenHash = µb.tokenHash('http://'); + this.anyMatchTokenHash = µb.tokenHash('*'); + this.noTokenHash = µb.tokenHash('~'); + this.regexTokenHash = µb.tokenHash('*r'); + this.dotTokenHash = µb.tokenHash('.'); this.tokens = []; this.filterParser = new FilterParser(); this.reset(); @@ -1389,8 +1410,8 @@ this.blockFilterCount = 0; this.duplicateCount = 0; this.duplicateBuster = {}; - this.categories = Object.create(null); - this.cspFilters = Object.create(null); + this.categories = new Map(); + this.cspFilters = new Map(); this.filterParser.reset(); this.filterCounts = {}; this.cspSubsets = new Map(); @@ -1404,11 +1425,11 @@ this.filterParser.reset(); this.frozen = true; let pushToBuffer = false; - for(let category in this.categories) { - let notHostnames = []; - if(this.categories[category]['.'] !== undefined && this.categories[category]['.'].hasOwnProperty('dict')) { - let details = objDataView.pushToBuffer(Array.from(this.categories[category]['.'].dict), notHostnames, Array.from(this.categories[category]['.'].dict)); - this.categories[category]['.'] = FilterDomain.fromSelfie(JSON.stringify(details) + '\t' + true); + for ( const [ bits, bucket ] of this.categories ) { + let notHostnames = []; + if(bucket.has(this.dotTokenHash) && bucket.get(this.dotTokenHash).hasOwnProperty('dict')) { + let details = objDataView.pushToBuffer(Array.from(bucket.get(this.dotTokenHash).dict), notHostnames, Array.from(bucket.get(this.dotTokenHash).dict)); + bucket.set(this.dotTokenHash, FilterDomain.fromSelfie(JSON.stringify(details) + '\t' + true)); pushToBuffer = true; } } @@ -1433,12 +1454,26 @@ '_': FilterGeneric, '||_': FilterGenericHnAnchored, 'd': FilterDomain, - 'fc': FilterCSP + 'fc': FilterCSP, + 'fp': FilterPair }; /******************************************************************************/ FilterContainer.prototype.toSelfie = function() { + + const categoriesToSelfie = function(categories) { + const selfie = []; + for ( const [ bits, bucket ] of categories ) { + const tokens = []; + for ( const [ tokenhash, filter ] of bucket ) { + tokens.push([ tokenhash, filter.toJSON() ]); + } + selfie.push([ bits, tokens ]); + } + return selfie; + }; + return { processedFilterCount: this.processedFilterCount, acceptedCount: this.acceptedCount, @@ -1447,8 +1482,8 @@ blockFilterCount: this.blockFilterCount, duplicateCount: this.duplicateCount, hostnameFilterDataView: objDataView.toSelfie(), - categories: JSON.stringify(this.categories), - cspFilters: JSON.stringify(this.cspFilters) + categories: categoriesToSelfie(this.categories), + cspFilters: categoriesToSelfie(this.cspFilters) }; }; @@ -1463,44 +1498,28 @@ this.blockFilterCount = selfie.blockFilterCount; this.duplicateCount = selfie.duplicateCount; var fc = this; - var filterFromSelfie = function(s) { - - function getSelfie(tokenEntries) { - let selfie; - - for(let prop in tokenEntries) { - var item = tokenEntries[prop]; - selfie = FilterContainer.factories[prop].fromSelfie(item); - } - return selfie; - } - - var categories = JSON.parse(s); - - var categoriesDict = {}; - - for(let category in categories) { - - if(typeof categoriesDict[category] == "undefined" ) - categoriesDict[category] = Object.create(null); - - let categoryItem = categories[category]; - - for(let token in categoryItem) { - - if(typeof categoriesDict[category][token] == "undefined" ) - categoriesDict[category][token] = Object.create(null); - - if(Array.isArray(categoryItem[token])) { - categoriesDict[category][token] = [getSelfie(categoryItem[token][0]), getSelfie(categoryItem[token][1])]; - } - else { - categoriesDict[category][token] = getSelfie(categoryItem[token]); - } - }; + + const getSelfie = function(tokenEntries) { + let selfie; + for(let prop in tokenEntries) { + var item = tokenEntries[prop]; + selfie = FilterContainer.factories[prop].fromSelfie(item); } - return categoriesDict; + return selfie; + } + + const filterFromSelfie = function(scategories) { + var categories = new Map(); + for ( const [ bits, bucket ] of scategories) { + const tokens = new Map(); + for ( const [ tokenhash, filter ] of bucket ) { + tokens.set(tokenhash, getSelfie(filter)); + } + categories.set(bits, tokens); + } + return categories; } + objDataView.fromSelfie(selfie.hostnameFilterDataView); this.categories = filterFromSelfie(selfie.categories); this.cspFilters = filterFromSelfie(selfie.cspFilters); @@ -1576,14 +1595,14 @@ let type = parsed.types; if ( type === 0 ) { - out.push([this.makeCategoryKey(keyShard), '.', parsed.f]); + out.push([keyShard, this.dotTokenHash, parsed.f]); return true; } let bitOffset = 1; do { if ( type & 1 ) { - out.push([this.makeCategoryKey(keyShard | (bitOffset << 4)), '.', parsed.f]); + out.push([keyShard | (bitOffset << 4), this.dotTokenHash, parsed.f]); } bitOffset += 1; type >>>= 1; @@ -1622,7 +1641,8 @@ FilterContainer.prototype.compileToAtomicFilter = function(filterClass, parsed, party, out, hostname) { let bits = parsed.action | parsed.important | party; let type = parsed.types; - let obj = {}; + let obj = {}, domains; + obj.blnDomainSpecific = this.isDomainSpecific(parsed); obj.compiled = filterClass.compile(parsed, hostname); if(parsed.dataStr != "") { @@ -1631,25 +1651,50 @@ if(parsed.domainList != "") { obj.domains = parsed.domainList; + domains = obj.domains.split('|'); } - + if ( type === 0 ) { - out.push([this.makeCategoryKey(bits), parsed.token, filterClass.fid, JSON.stringify(obj)]); + if(obj.blnDomainSpecific) { + for ( const hn of domains ) { + if(hn !== "") { + obj.domains = hn; + out.push([bits, parsed.token, filterClass.fid, JSON.stringify(obj)]); + } + } + } else { + out.push([bits, parsed.token, filterClass.fid, JSON.stringify(obj)]); + } return; } var bitOffset = 1; do { if ( type & 1 ) { - out.push([this.makeCategoryKey(bits | (bitOffset << 4)), parsed.token, filterClass.fid, JSON.stringify(obj)]); + + if(obj.blnDomainSpecific) { + for ( const hn of domains ) { + if(hn !== "") { + obj.domains = hn; + out.push([bits | (bitOffset << 4), parsed.token, filterClass.fid, JSON.stringify(obj)]); + } + } + } else { + out.push([bits | (bitOffset << 4), parsed.token, filterClass.fid, JSON.stringify(obj)]); + } } bitOffset += 1; type >>>= 1; } while ( type !== 0 ); }; - + + FilterContainer.prototype.isDomainSpecific = function(parsed) { + return (parsed.token == this.noTokenHash || parsed.token == this.anyMatchTokenHash || parsed.token == this.httpTokenHash || parsed.token == this.httpsTokenHash) && + parsed.domainList != undefined && + parsed.domainList.indexOf('~') === -1 && + parsed.dataStr == ""; + } /******************************************************************************/ - - + FilterContainer.prototype.fromCompiledContent = function(text) { let bucket, entry, factory, filter; @@ -1666,6 +1711,8 @@ let blnCspMatch = false; + if(fields.length == 4) fields[3] = JSON.parse(fields[3]); + if(fields.length > 2) { if(line.indexOf("\"csp\":") !== -1) { blnCspMatch = true; @@ -1673,24 +1720,29 @@ } if(blnCspMatch) { - bucket = this.cspFilters[fields[0]]; + bucket = this.cspFilters.get(fields[0]); if ( bucket === undefined ) { - bucket = this.cspFilters[fields[0]] = Object.create(null); + bucket = new Map(); + this.cspFilters.set(fields[0], bucket); } } else { - bucket = this.categories[fields[0]]; + if(fields.length == 4 && fields[3].blnDomainSpecific) { + fields[1] = µb.tokenHash(fields[3].domains); + } + bucket = this.categories.get(fields[0]); if ( bucket === undefined ) { - bucket = this.categories[fields[0]] = Object.create(null); + bucket = new Map(); + this.categories.set(fields[0], bucket); } } - entry = bucket[fields[1]]; + entry = bucket.get(fields[1]); - if ( fields[1] === '.' ) { - + if ( fields[1] === this.dotTokenHash ) { if ( entry === undefined ) { - entry = bucket['.'] = new FilterHostnameDict(); + entry = new FilterHostnameDict(); + bucket.set(this.dotTokenHash, entry); } let hshash = µb.tokenHash(fields[2]); if ( entry.add(hshash) === false ) { @@ -1707,14 +1759,13 @@ this.duplicateBuster[line] = true; factory = FilterContainer.factories[fields[2]]; - - fields[3] = JSON.parse(fields[3]); if(fields[3].domains != undefined) { let [hostnames , notHostnames, allHostnames] = objDataView.parseOptHostnames(fields[3].domains); let details = objDataView.pushToBuffer(hostnames, notHostnames, allHostnames); - filter = [factory.fromSelfie(fields[3].compiled), FilterDomain.fromSelfie(JSON.stringify(details) + '\t' + false)]; - } else { + filter = FilterPair.fromSelfie([fields[2], fields[3].compiled, JSON.stringify(details) + '\t' + false]); + } + else { filter = factory.fromSelfie(fields[3].compiled); } @@ -1726,14 +1777,17 @@ } if ( entry === undefined ) { - bucket[fields[1]] = filter; + bucket.set(fields[1], filter); continue; } if ( entry.fid === '[]' ) { entry.add(filter); continue; } - bucket[fields[1]] = new FilterBucket(entry, filter); + bucket.set( + fields[1], + new FilterBucket(entry, filter) + ); } return true; }; @@ -1751,28 +1805,36 @@ let matches, tokenEntry; re.lastIndex = 0; let i = 0; - while ( matches = re.exec(url) ) { + const addToken = function (token, beg) { tokenEntry = tokens[i]; if ( tokenEntry === undefined ) { tokenEntry = tokens[i] = new TokenEntry(); } - tokenEntry.beg = matches.index; - tokenEntry.token = µb.tokenHash(matches[0]); + if(beg !== undefined) tokenEntry.beg = beg; + tokenEntry.token = token; i += 1; - + } + + for (let element of Array.from(pageHostnameHashes.keys())) { + addToken(element); + } + + while ( matches = re.exec(url) ) { + addToken(µb.tokenHash(matches[0]), matches.index); // https://github.com/uBlockAdmin/uBlock/issues/1118 // Crazy case... but I guess we have to expect the worst... if ( i === 2048 ) { break; } } - - // Sentinel - tokenEntry = tokens[i]; - if ( tokenEntry === undefined ) { - tokenEntry = tokens[i] = new TokenEntry(); + addToken(this.noTokenHash); + addToken(this.anyMatchTokenHash); + if ( url.startsWith('https://') ) { + addToken(this.httpsTokenHash); + } else if ( url.startsWith('http://') ) { + addToken(this.httpTokenHash); } - tokenEntry.token = ''; + addToken(''); }; /******************************************************************************/ @@ -1780,7 +1842,7 @@ FilterContainer.prototype.matchTokens = function(bucket, url) { // Hostname-only filters - let f = bucket['.']; + let f = bucket.get(this.dotTokenHash); if ( f !== undefined && !skipGenericBlocking && f.match() !== false) { return f; } @@ -1794,39 +1856,20 @@ if ( token === '' ) { break; } - f = bucket[token]; + f = bucket.get(token); - if(Array.isArray(f)) { - let f0 = f[0]; - let f1 = f[1]; - if(f0 !== undefined && f0.fid.indexOf('h') === -1 && f0.fid != "[]" && skipGenericBlocking) { - continue; - } - if ( f0 !== undefined && f0.match(url, tokenEntry.beg) !== false && f1.match()) { - return f; - } - } else { - if(f !== undefined && f.fid.indexOf('h') === -1 && f.fid != "[]" && skipGenericBlocking) { - continue; - } - if ( f !== undefined && f.match(url, tokenEntry.beg) !== false ) { - return f; - } + if(f !== undefined && f.fid.indexOf('h') === -1 && f.fid != "[]" && skipGenericBlocking) { + continue; + } + if ( f !== undefined && f.match(url, tokenEntry.beg) !== false ) { + return f; } } // Regex-based filters - f = bucket['*']; - if(f !== undefined && Array.isArray(f)) { - let f0 = f[0]; - let f1 = f[1]; - if (f0.match(url) !== false && f1.match()) { - return f; - } - } else { - if ( f !== undefined && f.match(url) !== false ) { - return f; - } + f = bucket.get(this.regexTokenHash); + if ( f !== undefined && f.match(url) !== false ) { + return f; } return false; @@ -1843,17 +1886,13 @@ FilterContainer.prototype.matchStringExactType = function(context, requestURL, requestType) { let url = requestURL.toLowerCase(); - // These registers will be used by various filters pageHostnameRegister = context.pageHostname || ''; - pageDomainRegister = µb.URI.domainFromHostname(pageHostnameRegister) || pageHostnameRegister; - pageHostnameHashes = µb.getHostnameHashesFromLabelsBackward(pageHostnameRegister, pageDomainRegister, false); - - requestHostnameRegister = decode(encode(µb.URI.hostnameFromURI(requestURL).trim())); - requestDomainRegister = µb.URI.domainFromHostname(requestHostnameRegister) || requestHostnameRegister; + pageDomainRegister = context.pageDomain || pageHostnameRegister; + pageHostnameHashes = context.pageHostnameHashes || ''; + requestHostnameRegister = context.requestHostname; + requestDomainRegister = context.requestDomain; requestHostnameHashes = µb.getHostnameHashesFromLabelsBackward(requestHostnameRegister, requestDomainRegister, false); - - skipGenericBlocking = context.skipGenericBlocking; - + let party = isFirstParty(context.pageDomain, requestHostnameRegister) ? FirstParty : ThirdParty; // Be prepared to support unknown types @@ -1861,71 +1900,37 @@ if ( type === 0 ) { return ''; } - - let categories = this.categories; - let bf = false, bucket; - // Tokenize only once this.tokenize(url); - - // https://github.com/uBlockAdmin/uBlock/issues/139 - // Test against important block filters - if ( bucket = categories[this.makeCategoryKey(BlockAnyParty | Important | type)] ) { - bf = this.matchTokens(bucket, url); - if ( bf !== false ) { - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + '$important' + bf[1].toString() : 'sb:' + bf.toString() + '$important'; - } - } - if ( bucket = categories[this.makeCategoryKey(BlockAction | Important | type | party)] ) { - bf = this.matchTokens(bucket, url); - if ( bf !== false ) { - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + '$important' + bf[1].toString() : 'sb:' + bf.toString() + '$important'; - } - } - - // Test against block filters - if ( bucket = categories[this.makeCategoryKey(BlockAnyParty | type)] ) { - bf = this.matchTokens(bucket, url); - } - if ( bf === false ) { - if ( bucket = categories[this.makeCategoryKey(BlockAction | type | party)] ) { - bf = this.matchTokens(bucket, url); - } - } - // If there is no block filter, no need to test against allow filters - if ( bf === false ) { - return ''; - } - - skipGenericBlocking = false; - // Test against allow filters - let af; - if ( bucket = categories[this.makeCategoryKey(AllowAnyParty | type)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - if ( bucket = categories[this.makeCategoryKey(AllowAction | type | party)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + bf[1].toString() : 'sb:' + bf.toString(); + + let categoryKeysMap = new Map([ + ["important", [ + BlockAnyParty | Important | type, + BlockAction | Important | type | party + ] + ], + ["blocked", [ + BlockAnyParty | type, + BlockAction | type | party + ] + ], + ["allowed", [ + AllowAnyParty | type, + AllowAction | type | party + ] + ] + ]); + + return this.match(url, categoryKeysMap, context); }; - FilterContainer.prototype.matchStringExceptionOnlyRule = function(url,requestType) { + FilterContainer.prototype.matchStringExceptionOnlyRule = function(url, requestType) { pageHostnameRegister = µb.URI.hostnameFromURI(url) || ''; pageDomainRegister = µb.URI.domainFromHostname(pageHostnameRegister) || pageHostnameRegister; pageHostnameHashes = µb.getHostnameHashesFromLabelsBackward(pageHostnameRegister, pageDomainRegister, false); - requestHostnameRegister = µb.URI.hostnameFromURI(url); requestDomainRegister = µb.URI.domainFromHostname(requestHostnameRegister) || requestHostnameRegister; requestHostnameHashes = µb.getHostnameHashesFromLabelsBackward(requestHostnameRegister, requestDomainRegister, false); - let categories = this.categories; - let af = false,bf = false, bucket; - skipGenericBlocking = false; let party = FirstParty; let type = typeNameToTypeValue[requestType] || 0; @@ -1933,26 +1938,21 @@ return ''; } this.tokenize(url); - - if ( bucket = categories[this.makeCategoryKey(BlockAction | AnyParty | type | Important)] ) { - bf = this.matchTokens(bucket, url); - if ( bf !== false ) { - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + bf[1].toString() : 'sb:' + bf.toString(); - } - } - if ( bucket = categories[this.makeCategoryKey(AllowAnyParty | type)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - if ( bucket = categories[this.makeCategoryKey(AllowAction | type | party)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - return ''; + + let categoryKeysMap = new Map([ + ["important", [ + BlockAction | AnyParty | type | Important + ] + ], + ["allowed", [ + AllowAnyParty | type, + AllowAction | type | party + ] + ] + ]); + let context = {}; + context.skipGenericBlocking = false; + return this.match(url, categoryKeysMap, context); }; FilterContainer.prototype.matchCspRules = function(bucket, url,out) { this.tokenize(url); @@ -1967,36 +1967,44 @@ if ( token === '' ) { break; } - f = bucket[token]; + f = bucket.get(token); if(f !== undefined && f.fid.indexOf('h') === -1 && f.fid != "[]" && skipGenericBlocking) { continue; } if ( f !== undefined && f.match(url, tokenEntry.beg) !== false ) { - out.set(f.dataStr,f); + if(f.fid == "[]") + out.set(f.f.dataStr,f.f); + else + out.set(f.dataStr,f); } } - f = bucket['*']; + f = bucket.get(this.regexTokenHash); if ( f !== undefined && f.match(url) !== false ) { - out.set(f.f.dataStr,f.f); + if(f.fid == "[]") + out.set(f.f.dataStr,f.f); + else + out.set(f.dataStr,f); } }; FilterContainer.prototype.matchAndFetchCspData = function(context) { pageHostnameRegister = context.pageHostname || ''; - pageDomainRegister = µb.URI.domainFromHostname(pageHostnameRegister) || pageHostnameRegister; - pageHostnameHashes = µb.getHostnameHashesFromLabelsBackward(pageHostnameRegister, pageDomainRegister, false); + pageDomainRegister = context.pageDomain || pageHostnameRegister; + pageHostnameHashes = context.pageHostnameHashes || ''; requestHostnameRegister = context.requestHostname; - requestDomainRegister = µb.URI.domainFromHostname(requestHostnameRegister) || requestHostnameRegister; + requestDomainRegister = context.requestDomain; requestHostnameHashes = µb.getHostnameHashesFromLabelsBackward(requestHostnameRegister, requestDomainRegister, false); - let bucket; + let bucket, categoryKey; let type = typeNameToTypeValue["csp"]; let toBlockCSP = new Map(); let toAllowCSP = new Map(); - if ( bucket = this.cspFilters[this.makeCategoryKey(BlockAnyParty | type)] ) { + categoryKey = BlockAnyParty | type; + if ( bucket = this.cspFilters.get(categoryKey) ) { this.matchCspRules(bucket, context.requestURL, toBlockCSP); } - if ( bucket = this.cspFilters[this.makeCategoryKey(AllowAnyParty | type)] ) { + categoryKey = AllowAnyParty | type; + if ( bucket = this.cspFilters.get(categoryKey) ) { this.matchCspRules(bucket, context.requestURL, toAllowCSP); } @@ -2019,6 +2027,39 @@ } /******************************************************************************/ + FilterContainer.prototype.match = function(url, categoryKeysMap, context) { + + let bucket, bf = false, tf; + skipGenericBlocking = context.skipGenericBlocking; + + for (let [key, categoryKeys] of Array.from(categoryKeysMap)) { + + if(key == "allowed" && bf === false && categoryKeysMap.has("blocked")) { + return ''; + } else if(key == "allowed" && bf !== false && categoryKeysMap.has("blocked")) { + tf = bf; + } + else if(key == "allowed") { + skipGenericBlocking = false; + } + + for (let i = 0; i < categoryKeys.length; i++) { + if ( bucket = this.categories.get(categoryKeys[i]) ) { + bf = this.matchTokens(bucket, url); + if ( bf !== false && key == "important") { + return 'sb:' + bf.toString() + '$important'; + } + else if( bf !== false && key == "allowed") { + return 'sa:' + bf.toString(); + } else if(bf !== false) { + break; + } + } + } + } + if(tf === undefined) return ''; + return 'sb:' + tf.toString(); + } FilterContainer.prototype.matchString = function(context) { // https://github.com/uBlockAdmin/uBlock/issues/519 // Use exact type match for anything beyond `other` @@ -2033,7 +2074,7 @@ // `match-case` option not supported, but then, I saw only one // occurrence of it in all the supported lists (bulgaria list). let url = context.requestURL.toLowerCase(); - + // The logic here is simple: // // block = !whitelisted && blacklisted @@ -2058,109 +2099,41 @@ // These registers will be used by various filters pageHostnameRegister = context.pageHostname || ''; - pageDomainRegister = µb.URI.domainFromHostname(pageHostnameRegister) || pageHostnameRegister; - pageHostnameHashes = µb.getHostnameHashesFromLabelsBackward(pageHostnameRegister, pageDomainRegister, false); + pageDomainRegister = context.pageDomain || pageHostnameRegister; + pageHostnameHashes = context.pageHostnameHashes || ''; requestHostnameRegister = context.requestHostname; - requestDomainRegister = µb.URI.domainFromHostname(requestHostnameRegister) || requestHostnameRegister; + requestDomainRegister = context.requestDomain; requestHostnameHashes = µb.getHostnameHashesFromLabelsBackward(requestHostnameRegister, requestDomainRegister, false); - skipGenericBlocking = context.skipGenericBlocking; let party = isFirstParty(context.pageDomain, context.requestHostname) ? FirstParty : ThirdParty; - let filterClasses = this.categories; - let bucket; // Tokenize only once this.tokenize(url); - let bf = false; - - // https://github.com/uBlockAdmin/uBlock/issues/139 - // Test against important block filters. - // The purpose of the `important` option is to reverse the order of - // evaluation. Normally, it is "evaluate block then evaluate allow", with - // the `important` property it is "evaluate allow then evaluate block". - if ( bucket = filterClasses[this.makeCategoryKey(BlockAnyTypeAnyParty | Important)] ) { - bf = this.matchTokens(bucket, url); - if ( bf !== false ) { - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + '$important' + bf[1].toString() : 'sb:' + bf.toString() + '$important'; - } - } - if ( bucket = filterClasses[this.makeCategoryKey(BlockAnyType | Important | party)] ) { - bf = this.matchTokens(bucket, url); - if ( bf !== false ) { - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + '$important' + bf[1].toString() : 'sb:' + bf.toString() + '$important'; - } - } - if ( bucket = filterClasses[this.makeCategoryKey(BlockAnyParty | Important | type)] ) { - bf = this.matchTokens(bucket, url); - if ( bf !== false ) { - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + '$important' + bf[1].toString() : 'sb:' + bf.toString() + '$important'; - } - } - if ( bucket = filterClasses[this.makeCategoryKey(BlockAction | Important | type | party)] ) { - bf = this.matchTokens(bucket, url); - if ( bf !== false ) { - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + '$important' + bf[1].toString() : 'sb:' + bf.toString() + '$important'; - } - } - - // Test against block filters - if ( bf === false ) { - if ( bucket = filterClasses[this.makeCategoryKey(BlockAnyTypeAnyParty)] ) { - bf = this.matchTokens(bucket, url); - } - } - if ( bf === false ) { - if ( bucket = filterClasses[this.makeCategoryKey(BlockAnyType | party)] ) { - bf = this.matchTokens(bucket, url); - } - } - if ( bf === false ) { - if ( bucket = filterClasses[this.makeCategoryKey(BlockAnyParty | type)] ) { - bf = this.matchTokens(bucket, url); - } - } - if ( bf === false ) { - if ( bucket = filterClasses[this.makeCategoryKey(BlockAction | type | party)] ) { - bf = this.matchTokens(bucket, url); - } - } - - // If there is no block filter, no need to test against allow filters - if ( bf === false ) { - return ''; - } - - // Test against allow filters - let af; - skipGenericBlocking = false; - - if ( bucket = filterClasses[this.makeCategoryKey(AllowAnyTypeAnyParty)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - if ( bucket = filterClasses[this.makeCategoryKey(AllowAnyType | party)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - if ( bucket = filterClasses[this.makeCategoryKey(AllowAnyParty | type)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - if ( bucket = filterClasses[this.makeCategoryKey(AllowAction | type | party)] ) { - af = this.matchTokens(bucket, url); - if ( af !== false ) { - return Array.isArray(af) ? 'sa:' + af[0].toString() + af[1].toString() : 'sa:' + af.toString(); - } - } - - return Array.isArray(bf) ? 'sb:' + bf[0].toString() + bf[1].toString() : 'sb:' + bf.toString(); + let categoryKeysMap = new Map([ + ["important", [ + BlockAnyTypeAnyParty | Important, + BlockAnyType | Important | party, + BlockAnyParty | Important | type, + BlockAction | Important | type | party + ] + ], + ["blocked", [ + BlockAnyTypeAnyParty, + BlockAnyType | party, + BlockAnyParty | type, + BlockAction | type | party + ] + ], + ["allowed", [ + AllowAnyTypeAnyParty, + AllowAnyType | party, + AllowAnyParty | type, + AllowAction | type | party + ] + ] + ]); + return this.match(url, categoryKeysMap, context); }; /******************************************************************************/ diff --git a/src/js/tab.js b/src/js/tab.js index dd42883eb..3fa019cdc 100644 --- a/src/js/tab.js +++ b/src/js/tab.js @@ -145,6 +145,7 @@ housekeep itself. this.normalURL = this.rootHostname = this.rootDomain = ''; + this.rootHostnameHashes = new Map(); this.timer = null; this.onTabCallback = null; this.onTimerCallback = null; @@ -201,6 +202,7 @@ housekeep itself. this.normalURL = µb.normalizePageURL(this.tabId, this.rawURL); this.rootHostname = µb.URI.hostnameFromURI(this.normalURL); this.rootDomain = µb.URI.domainFromHostname(this.rootHostname); + this.rootHostnameHashes = µb.getHostnameHashesFromLabelsBackward(this.rootHostname, this.rootDomain, false); } }; @@ -340,6 +342,7 @@ housekeep itself. entry.normalURL = µb.normalizePageURL(entry.tabId); entry.rootHostname = µb.URI.hostnameFromURI(entry.normalURL); entry.rootDomain = µb.URI.domainFromHostname(entry.rootHostname); + entry.rootHostnameHashes = µb.getHostnameHashesFromLabelsBackward(entry.rootHostname, entry.rootDomain, false); })(); // Context object, typically to be used to feed filtering engines. @@ -347,6 +350,7 @@ housekeep itself. var tabContext = lookup(tabId); this.rootHostname = tabContext.rootHostname; this.rootDomain = tabContext.rootDomain; + this.rootHostnameHashes = tabContext.rootHostnameHashes; this.pageHostname = this.pageDomain = this.requestURL = @@ -459,17 +463,21 @@ vAPI.tabs.onPopup = function(details) { var µburi = µb.URI; var openerHostname = µburi.hostnameFromURI(openerURL); var openerDomain = µburi.domainFromHostname(openerHostname); - + let pageHostnameHashes = µb.getHostnameHashesFromLabelsBackward(openerHostname, openerDomain, false); var targetURL = details.targetURL; - + let requestHostname = µb.URI.hostnameFromURI(targetURL); + let requestDomain = µburi.domainFromHostname(requestHostname); + var context = { pageHostname: openerHostname, pageDomain: openerDomain, rootHostname: openerHostname, rootDomain: openerDomain, requestURL: targetURL, - requestHostname: µb.URI.hostnameFromURI(targetURL), - requestType: 'popup' + requestHostname: requestHostname, + requestDomain: requestDomain, + requestType: 'popup', + pageHostnameHashes: pageHostnameHashes }; var result = ''; diff --git a/src/js/traffic.js b/src/js/traffic.js index 1bbbbc265..a0b441352 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -89,6 +89,7 @@ var onBeforeRequest = function(details) { var requestURL = details.url; requestContext.requestURL = requestURL; requestContext.requestHostname = details.hostname; + requestContext.requestDomain = µb.URI.domainFromHostname(requestContext.requestHostname); requestContext.requestType = requestType; var result = pageStore.filterRequest(requestContext); @@ -149,6 +150,7 @@ var onBeforeRootFrameRequest = function(details) { pageDomain: requestDomain, requestURL: requestURL, requestHostname: requestHostname, + requestDomain: requestDomain, requestType: 'main_frame' }; @@ -218,6 +220,7 @@ var onBeforeBehindTheSceneRequest = function(details) { var context = pageStore.createContextFromPage(); context.requestURL = details.url; context.requestHostname = details.hostname; + context.requestDomain = µb.URI.domainFromHostname(context.requestHostname); context.requestType = details.type; // Blocking behind-the-scene requests can break a lot of stuff: prevent @@ -273,6 +276,7 @@ var onHeadersReceived = function(details) { var context = pageStore.createContextFromFrameId(details.parentFrameId); context.requestURL = details.url; context.requestHostname = details.hostname; + context.requestDomain = µb.URI.domainFromHostname(context.requestHostname); context.requestType = 'inline-script'; µb.staticNetFilteringEngine.cspSubsets = new Map(); var result = pageStore.filterRequestNoCache(context); @@ -325,6 +329,7 @@ var onRootFrameHeadersReceived = function(details) { var context = pageStore.createContextFromPage(); context.requestURL = requestURL; context.requestHostname = requestHostname; + context.requestDomain = µb.URI.domainFromHostname(context.requestHostname); context.requestType = 'inline-script'; µb.staticNetFilteringEngine.cspSubsets = new Map();