diff --git a/config.js b/config.js index 034fcceec..66f52c5af 100644 --- a/config.js +++ b/config.js @@ -68,8 +68,11 @@ T: { text_html: "text/html", + text_markdown: "text/markdown", + text_plain: "text/plain", maybe_text_html: "maybe_text_html", javascript: "application/javascript", + application_pdf: "application/pdf", safe_html: "text/x-safe-html", image_jpeg: "image/jpeg", flash: "application/x-shockwave-flash", // Adobe Flash Player is no longer supported diff --git a/lib/core.js b/lib/core.js index d5443d459..eb797ee1a 100644 --- a/lib/core.js +++ b/lib/core.js @@ -1242,10 +1242,6 @@ var links = result.links; - if (!result.meta.canonical) { - result.meta.canonical = uri; - } - // Remove canonical links. // Remove _meta data. var canonical = result.meta.canonical; @@ -2176,6 +2172,9 @@ options.domainOptions = options.getDomainOptionsByUrl(uri); } + // Reset `acceptHeaders` as empty array for each call. + options.acceptHeaders = []; + asyncMethodCb('initial'); }; diff --git a/lib/fetch.js b/lib/fetch.js index d17cce1cb..b7f29f8dd 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -45,7 +45,8 @@ export function skipLoggingFetchError(error) { || error?.code === 'ERR_HTTP2_STREAM_ERROR' || error?.code === 'ERR_HTTP2_SESSION_ERROR' || error?.code === 'Z_DATA_ERROR' - || error?.code === 'Z_BUF_ERROR'; + || error?.code === 'Z_BUF_ERROR' + || error?.code === 'ETIMEDOUT'; } function doFetch(fetch_func, h1_fetch_func, options) { diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index ffee38009..1c53a9883 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -164,7 +164,8 @@ export default { if ('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])) { - if (options.requestHeaders?.['Accept']?.indexOf(headers['content-type'].split(';')[0]) > -1) { + if (options.acceptHeaders?.indexOf + && options.acceptHeaders.indexOf(headers['content-type'].split(';')[0]) > -1) { return cb(null, { __nonHtmlContentResponse: resp }); diff --git a/lib/plugins/system/htmlparser/nonHtmlContentData.js b/lib/plugins/system/htmlparser/nonHtmlContentData.js index 4d7f4da9d..f12c8a75f 100644 --- a/lib/plugins/system/htmlparser/nonHtmlContentData.js +++ b/lib/plugins/system/htmlparser/nonHtmlContentData.js @@ -9,9 +9,14 @@ export default { ], getLink: function(url, __nonHtmlContentData, options, cb) { + var nonHtmlContentType = __nonHtmlContentData.type; var nonHtmlContentLength = __nonHtmlContentData.content_length; + if (options.dataMode && !/video|image/i.test(nonHtmlContentType)) { + return cb(); + } + // HEADS UP: do not ever remove the below check for 'javascript' or 'flash' in content type // if left allowed, it'll make apps vulnerable for XSS attacks as such files will be rendered as regular embeds if (/javascript|flash|application\/json|text\/xml|application\/xml/i.test(nonHtmlContentType)) { @@ -30,5 +35,4 @@ export default { }); } } - }; diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js index 0df1dcb61..432ae812c 100644 --- a/lib/plugins/system/meta/cachedMeta.js +++ b/lib/plugins/system/meta/cachedMeta.js @@ -57,7 +57,7 @@ export default { var record_age_sec = (new Date().getTime() / 1000) - data._sys_created_at; if (record_age_sec > ttl) { // Ignore cache older then requested ttl. - log(' -- Disable using old cache for: ' + url); + log(' -- Disable using old cache for: ' + url + '. Seconds old: ' + (record_age_sec - ttl)); return noCacheFound(); } } diff --git a/lib/utils.js b/lib/utils.js index fe8f93133..b7fa71f75 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -66,6 +66,7 @@ export function getMaxCacheTTLOverride(url, options) { }; export function getMaxCacheTTL(url, options, default_min_ttl) { + default_min_ttl = default_min_ttl || CONFIG.CACHE_TTL; var proxy = getMaxCacheTTLOverride(url, options); var result = Math.max(fixTTL(options && options.cache_ttl), fixTTL(proxy && proxy.cache_ttl), fixTTL(default_min_ttl)); result = maxTTL(result); @@ -971,7 +972,7 @@ export function getContentType(uriForCache, uriOriginal, options, cb) { }, { // Ignore proxy.cache_ttl, if options.cache_ttl === 0 - do not read from cache. refresh: options.refresh || options.cache_ttl === 0, - ttl: getMaxCacheTTL(uriOriginal, options, CONFIG.CACHE_TTL), + ttl: getMaxCacheTTL(uriOriginal, options), }, cb); }; diff --git a/plugins/domains/ted.com/ted.com.js b/plugins/domains/ted.com/ted.com.js index ab6c8b13c..c54be9694 100644 --- a/plugins/domains/ted.com/ted.com.js +++ b/plugins/domains/ted.com/ted.com.js @@ -7,7 +7,8 @@ export default { provides: [ "oembedLinks", "tedLangs", - "__allowEmbedURL" + "__allowEmbedURL", + "__isYouTube" ], mixins: [ @@ -189,4 +190,4 @@ export default { // Should work, but let's skip from tests not to avoid all oembed-* mixins // "https://www.ted.com/talks/su_santidad_el_papa_francisco_nuestro_imperativo_moral_para_actuar_sobre_el_cambio_climatico_y_3_pasos_que_podemos_dar", ] -}; \ No newline at end of file +}; diff --git a/plugins/domains/youtube.com/youtube.video.js b/plugins/domains/youtube.com/youtube.video.js index 652fdaa23..42f0e1a3a 100644 --- a/plugins/domains/youtube.com/youtube.video.js +++ b/plugins/domains/youtube.com/youtube.video.js @@ -23,8 +23,8 @@ export default { var api_key = options.getProviderOptions('youtube.api_key'); - if (!api_key) { - return cb (new Error ("No youtube.api_key configured")); + if (!api_key || options.dataMode) { + return cb('YouTube Data API is disabled'); } var parts = options.getProviderOptions('youtube.parts') || [ diff --git a/plugins/links/google-docs-viewer.js b/plugins/links/google-docs-viewer.js index 8c28d46d5..6065c512f 100644 --- a/plugins/links/google-docs-viewer.js +++ b/plugins/links/google-docs-viewer.js @@ -1,6 +1,6 @@ export default { - getLink: function(url, __nonHtmlContentData, utils, headers, options, iframelyRun, cb) { + getLink: function(__nonDataMode, url, __nonHtmlContentData, utils, headers, options, iframelyRun, cb) { if (!options.getProviderOptions('disableDocViewers', false) && /application\/pdf|text\/rtf/.test(__nonHtmlContentData.type)) { diff --git a/plugins/links/ms-doc-viewer.js b/plugins/links/ms-doc-viewer.js index 7c4876d95..a1a84df9d 100644 --- a/plugins/links/ms-doc-viewer.js +++ b/plugins/links/ms-doc-viewer.js @@ -2,7 +2,7 @@ export default { - getLink: function(url, __nonHtmlContentData, options) { + getLink: function(__nonDataMode, url, __nonHtmlContentData, options) { if (!options.getProviderOptions('disableDocViewers', false) && /application\/vnd\.openxmlformats\-officedocument|ms\-powerpoint|msword|ms\-excel|ms\-office/.test(__nonHtmlContentData.type) diff --git a/plugins/meta/canonical.js b/plugins/meta/canonical.js index 0699849f1..eb4d843f4 100644 --- a/plugins/meta/canonical.js +++ b/plugins/meta/canonical.js @@ -1,13 +1,14 @@ export default { - highestPriority: true, - - getMeta: function(url, meta) { + getMeta: function(url, meta, options) { var canonical = (meta.canonical && meta.canonical.href || meta.canonical) || (meta.og && meta.og.url) || (meta.twitter && meta.twitter.url); - if (canonical && typeof canonical === 'string' && /^https?:\/\//i.test(canonical)) { + if ((!canonical || /^https?:\/\//i.test(canonical)) && !options.dataMode) { + canonical = url; + } + if (typeof canonical === 'string') { return { canonical: canonical }; diff --git a/plugins/meta/non-data-mode.js b/plugins/meta/non-data-mode.js new file mode 100644 index 000000000..f27ccdbd1 --- /dev/null +++ b/plugins/meta/non-data-mode.js @@ -0,0 +1,13 @@ +export default { + + provides: "__nonDataMode", + + getData: function(options) { + + if (!options.dataMode) { + return { + __nonDataMode: true + } + } + } +} \ No newline at end of file