diff --git a/audit.log b/audit.log index 41614f097..157cb1c12 100644 --- a/audit.log +++ b/audit.log @@ -13,6 +13,20 @@ │ More info │ https://github.com/advisories/GHSA-j3q9-mxjg-w52f │ └─────────────────────┴────────────────────────────────────────────────────────┘ ┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Mongoose's Improper Sanitization of $nor in │ +│ │ sanitizeFilter May Allow NoSQL Injection │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ mongoose │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=8.0.0 <=8.22.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=8.22.1 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>mongoose │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-wpg9-53fq-2r8h │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ │ moderate │ brace-expansion: Zero-step sequence causes process │ │ │ hang and memory exhaustion │ ├─────────────────────┼────────────────────────────────────────────────────────┤ @@ -68,5 +82,5 @@ ├─────────────────────┼────────────────────────────────────────────────────────┤ │ More info │ https://github.com/advisories/GHSA-qj8w-gfj5-8c6v │ └─────────────────────┴────────────────────────────────────────────────────────┘ -5 vulnerabilities found -Severity: 4 moderate | 1 high +6 vulnerabilities found +Severity: 4 moderate | 2 high diff --git a/config.js b/config.js index 4a320562a..c4dd73afa 100644 --- a/config.js +++ b/config.js @@ -367,7 +367,13 @@ 'ENET', 'HPE_INVALID_', 'ERR_SSL_' - ] + ], + + GET_VARS_METHODS: { + 'getSignals': 'signals', + 'getPolicy': 'policy', + 'getSources': 'sources' + } }; // Providers config loader. diff --git a/lib/core.js b/lib/core.js index 94ae22480..ee3cca066 100644 --- a/lib/core.js +++ b/lib/core.js @@ -458,13 +458,19 @@ // Use all generic plugins. for(var i = 0; i < pluginsList.length; i++) { var plugin = pluginsList[i]; - if (!plugin.domain && !plugin.custom) { + if (!plugin.domain + && !plugin.custom + // dataMode - enables data plugins + && (options.dataMode || !plugin.data)) { + var match = plugin.pluginReMatchesUrl(domain, uri); if (match) { pluginsUrlMatches[plugin.id] = match; } - // false - has re, no match. + // match = result; - url matched plugin.re. + // match = undefined; - domain has no plugin.re. + // match = false - plugin has re, and no match. if (match !== false) { initialPlugins.push(plugin); } @@ -473,7 +479,7 @@ } // In domain debug: add all plugins before domain plugins to make them low priority. - if (options.mixAllWithDomainPlugin) { + if (options.mixAllWithDomainPlugin || options.dataMode) { addAllGeneric(); } @@ -513,6 +519,15 @@ addAllGeneric(); } + function addOneMorePlugin(pluginId) { + var exists = initialPlugins.find(function(plugin) { + return plugin.id === pluginId; + }); + if (!exists) { + initialPlugins.push(plugins[pluginId]); + } + } + if (options.forceParams) { // TODO: replace forEach options.forceParams.forEach(function(param) { @@ -525,13 +540,7 @@ for(var k = 0; k < paramPlugins.length; k++) { var foundPluginId = paramPlugins[k]; - - var exists = initialPlugins.find(function(plugin) { - return plugin.id === foundPluginId; - }); - if (!exists) { - initialPlugins.push(plugins[foundPluginId]); - } + addOneMorePlugin(foundPluginId); } } }); @@ -943,9 +952,20 @@ } } } - } else if (r.method.name === "getVars") { - for(var key in r.data) { - var v = r.data[key]; + } else if (r.method.name === "getVars" || CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { + + var result_data = r.data; + + // Set specific alias as key. + if (CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { + var key = CONFIG.GET_VARS_METHODS[r.method.name]; + result_data = { + [key]: r.data + }; + } + + for(var key in result_data) { + var v = result_data[key]; // TODO: work with arrays? if (v !== '' && v !== null && ((typeof v === 'string' && !/^\s+$/.test(v)) || typeof v === 'number' || typeof v === 'object')) { @@ -1134,7 +1154,7 @@ return hasDomainData; } - const BIG_CONTEXT = ['htmlparser', 'readability', 'decode', 'cheerio']; + const BIG_CONTEXT = ['readability', 'decode', 'cheerio']; function prepareResultData(uri, result, options) { @@ -1312,6 +1332,11 @@ if (allData) { for(var i = 0; i < allData.length; i++) { var r = allData[i]; + for(var key in r.data) { + if (r.data[key].abortController) { + r.data[key] = 'BIG_CONTEXT'; + } + } for(var j = 0; j < BIG_CONTEXT.length; j++) { var d = BIG_CONTEXT[j]; if (r.data && r.data[d]) { diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js index e9b7b807c..497f2bf8a 100644 --- a/lib/loader/pluginLoader.js +++ b/lib/loader/pluginLoader.js @@ -370,6 +370,9 @@ // Plugins in folder 'custom' or 'core' will be run only on explicit dependency. pluginDeclaration.custom = (bits.indexOf('custom') > -1 || bits.indexOf('system') > -1) && !plugin.generic; + // Plugins in folder 'data' will be run only in data_mode. + pluginDeclaration.data = bits.indexOf('data') > -1; + var stat = fs.statSync(pluginPath); pluginDeclaration.modified = new Date(stat.mtime); pluginDeclaration.getPluginLastModifiedDate = getPluginLastModifiedDate; @@ -665,6 +668,7 @@ await loadPlugins(pluginsRoot, 'custom'); await loadPlugins(pluginsRoot, 'links'); await loadPlugins(pluginsRoot, 'meta'); + await loadPlugins(pluginsRoot, 'data'); await loadPlugins(pluginsRoot, 'templates'); // TODO: if has multiple modules_listing - CUSTOM_PLUGINS_PATH will be loaded multiple times. @@ -674,6 +678,7 @@ await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'data'); await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates'); } else { console.warn('Custom plugin folder "' + CONFIG.CUSTOM_PLUGINS_PATH + '" not found.'); diff --git a/lib/loader/utils.js b/lib/loader/utils.js index d6decc3e7..f10a6da66 100644 --- a/lib/loader/utils.js +++ b/lib/loader/utils.js @@ -1,3 +1,5 @@ + import CONFIG from '../../config.loader.js'; + export const DEFAULT_PARAMS = [ "url", "urlMatch", @@ -43,6 +45,12 @@ "prepareLink" ]; + + // Adds vars methods to PLUGIN_METHODS` as `getSignals`. + CONFIG.GET_VARS_METHODS && Object.keys(CONFIG.GET_VARS_METHODS).forEach(methodName => { + PLUGIN_METHODS.push(methodName); + }); + export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([ "mixins" ]); diff --git a/lib/plugins/system/htmlparser/__allowHtmlparser.js b/lib/plugins/system/htmlparser/__allowHtmlparser.js index 0f4a79e45..d65d35b7a 100644 --- a/lib/plugins/system/htmlparser/__allowHtmlparser.js +++ b/lib/plugins/system/htmlparser/__allowHtmlparser.js @@ -4,6 +4,7 @@ export default { provides: [ // Run for all who requests htmlparser or meta. 'htmlparser', + 'meta', '__allowHtmlparser' ], diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index 0633fceae..2e1e0c000 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -8,15 +8,41 @@ import { extendCookiesJar } from '../../../fetch.js'; var getUrlFunctional = utils.getUrlFunctional; import { CollectingHandlerForMutliTarget } from './CollectingHandlerForMutliTarget.js'; +// Check if first request `Accept` header matches response `content-type`. +function matchContentTypeHeaders(requestHeaders, responseHeaders) { + var requestAcceptHeaders = requestHeaders?.['Accept']; + if (!requestAcceptHeaders) { + return; + } + var responseContentType = responseHeaders['content-type']; + if (!responseContentType) { + return; + } + + var requestContentType = requestAcceptHeaders.split(/[;, ]/)[0]; + + if (requestContentType) { + if (/html|\*/gi.test(requestContentType)) { + // Skip "any" or "html" type. + return; + } + if (responseContentType.indexOf(requestContentType) > - 1 ) { + return true; + } + } + +} + export default { provides: [ 'self', '__nonHtmlContentData', + '__nonHtmlContentResponse', '__statusCode' ], - getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, __allowHtmlparser, cb) { + getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, cb) { var options2 = {...options, ...{ followRedirect: !!options.followHTTPRedirect, @@ -159,7 +185,13 @@ export default { } } - if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){ + if (matchContentTypeHeaders(options.requestHeaders, headers)) { + return cb(null, { + __nonHtmlContentResponse: resp + }); + } + + if ('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])) { abortController.abort(); return cacheAndRespond(null, { diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js index 1a1497968..0df1dcb61 100644 --- a/lib/plugins/system/meta/cachedMeta.js +++ b/lib/plugins/system/meta/cachedMeta.js @@ -19,7 +19,7 @@ export default { '__statusCode' ], - getData: function(url, options, whitelistRecord, cb) { + getData: function(url, __allowHtmlparser, options, whitelistRecord, cb) { // Ignore proxy.cache_ttl, if options.cache_ttl === 0 - do not read from cache. if (options.refresh || options.cache_ttl === 0) { diff --git a/lib/plugins/system/meta/utils.js b/lib/plugins/system/meta/utils.js index ff5c1fa70..7187ab536 100644 --- a/lib/plugins/system/meta/utils.js +++ b/lib/plugins/system/meta/utils.js @@ -1,6 +1,12 @@ export function getMetaCacheKey(url, whitelistRecord, options) { - var meta_key = 'meta:' + url; + var meta_key = 'meta'; + + if (options.metaKeyPrefix) { + meta_key += ':' + options.metaKeyPrefix; + } + + meta_key += ':' + url; var whitelistHash = whitelistRecord && whitelistRecord.getRecordHash(); if (whitelistHash) { @@ -15,4 +21,4 @@ export function getMetaCacheKey(url, whitelistRecord, options) { return meta_key; }; -export const notPlugin = true; \ No newline at end of file +export const notPlugin = true; diff --git a/lib/utils.js b/lib/utils.js index 94098c49a..fd6f36706 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -105,6 +105,7 @@ export function prepareRequestOptions(request_options, options) { } } + // Some calls (like oembed) use basic options without `getDomainOptions`. var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); if (enable_domain_prerender) { setPrerender(enable_domain_prerender); @@ -241,7 +242,7 @@ export function prepareRequestOptions(request_options, options) { follow = 0; } - // Custom redirect logit for cookies. + // Custom redirect logic for cookies. if (options.followRedirect && options.reuseCookies) { redirect = 'manual'; follow = 0; @@ -254,7 +255,10 @@ export function prepareRequestOptions(request_options, options) { uri: url, method: 'GET', headers: { - 'Accept': '*/*' + ...{ + 'Accept': '*/*' + }, + ...options.requestHeaders }, timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, redirect: redirect, diff --git a/modules/api/views.js b/modules/api/views.js index 421ab7d85..00bdbe74a 100644 --- a/modules/api/views.js +++ b/modules/api/views.js @@ -137,6 +137,7 @@ export default function(app) { debug: getBooleanParam(req, 'debug'), returnProviderOptionsUsage: getBooleanParam(req, 'debug'), mixAllWithDomainPlugin: getBooleanParam(req, 'mixAllWithDomainPlugin'), + dataMode: getBooleanParam(req, 'dataMode'), forceParams: req.query.meta === "true" ? CONFIG.DEBUG_CONTEXTS : null, whitelist: getBooleanParam(req, 'whitelist'), readability: getBooleanParam(req, 'readability'), diff --git a/modules/debug/views.js b/modules/debug/views.js index 9aae734da..bb68e71e4 100644 --- a/modules/debug/views.js +++ b/modules/debug/views.js @@ -10,12 +10,14 @@ export default function(app) { DEBUG = {"on":1, "true":1}[req.query.debug]; } - res.render('debug',{ + res.render('debug', { uri: req.query.uri, mixAllWithDomainPlugin: !!{"on":1, "true":1}[req.query.mixAllWithDomainPlugin], + dataMode: !!{"on":1, "true":1}[req.query.dataMode], refresh: !!{"on":1, "true":1}[req.query.refresh], DEBUG: DEBUG, + DATA_DEBUG_ENABLED: CONFIG.DATA_DEBUG_ENABLED, QUERY: getProviderOptionsQuery(req.query) }); }); -}; \ No newline at end of file +}; diff --git a/static/js/debug.js b/static/js/debug.js index 457751dac..aac044dc9 100644 --- a/static/js/debug.js +++ b/static/js/debug.js @@ -252,6 +252,19 @@ function showEmbeds($embeds, data, filterByRel) { $meta.append('' + (DEBUG ? ('' + pluginId + '') : '') + '' + key + '' + linkify(data.meta[key]) + '') }); + // Render "vars". + if (data.vars) { + metaKeys = Object.keys(data.vars); + metaKeys.forEach(function(key) { + if (key == "_sources") { + return; + } + var pluginId = data.vars._sources && data.vars._sources[key] || ''; + usedPlugins[pluginId] = true; + + $meta.append('' + (DEBUG ? ('' + pluginId + '') : '') + 'vars: ' + key + '
' + JSON.stringify(data.vars[key], null, 4) + '
') + }); + } if (DEBUG && data.vary && data.vary.join) { $meta.append('vary' + data.vary.join('
') + '') } @@ -326,6 +339,7 @@ function processUrl() { var query = { debug: true, group: false, + dataMode: $('[name="dataMode"]').is(":checked"), mixAllWithDomainPlugin: $('[name="mixAllWithDomainPlugin"]').is(":checked"), refresh: $('[name="refresh"]').is(":checked") }; diff --git a/static/js/iframely.js b/static/js/iframely.js index f13861da6..d7ca4d06e 100644 --- a/static/js/iframely.js +++ b/static/js/iframely.js @@ -151,6 +151,7 @@ uri: !options.url ? uri : undefined, url: options.url ? uri : undefined, debug: options.debug, + dataMode: options.dataMode, mixAllWithDomainPlugin: options.mixAllWithDomainPlugin, refresh: options.refresh, meta: options.meta, diff --git a/views/debug.ejs b/views/debug.ejs index 0022d83eb..065a5b999 100644 --- a/views/debug.ejs +++ b/views/debug.ejs @@ -47,6 +47,11 @@ <% if (DEBUG) { %>
+ <% if (DATA_DEBUG_ENABLED) { %> + + <% } %> @@ -98,4 +103,4 @@
- \ No newline at end of file +