From dd993b626be4282af86da866a2fe4af623397bcc Mon Sep 17 00:00:00 2001 From: leon <25666053+ployt0@users.noreply.github.com> Date: Sat, 20 Aug 2022 15:25:50 +0100 Subject: [PATCH 1/2] Add webp support. Provide the option to disable strict SSL to facilitate local testing. --- src/parser.js | 4 ++-- src/translator.js | 2 +- src/wizard.js | 6 ++++++ src/writer.js | 11 +++++++---- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/parser.js b/src/parser.js index 683344c..99ca1ba 100644 --- a/src/parser.js +++ b/src/parser.js @@ -113,7 +113,7 @@ function getPostCoverImageId(postData) { function collectAttachedImages(channelData) { const images = getItemsOfType(channelData, 'attachment') // filter to certain image file types - .filter(attachment => attachment.attachment_url && (/\.(gif|jpe?g|png)$/i).test(attachment.attachment_url[0])) + .filter(attachment => attachment.attachment_url && (/\.(gif|jpe?g|png|webp)$/i).test(attachment.attachment_url[0])) .map(attachment => ({ id: attachment.post_id[0], postId: attachment.post_parent[0], @@ -132,7 +132,7 @@ function collectScrapedImages(channelData, postTypes) { const postContent = postData.encoded[0]; const postLink = postData.link[0]; - const matches = [...postContent.matchAll(/]*src="(.+?\.(?:gif|jpe?g|png))"[^>]*>/gi)]; + const matches = [...postContent.matchAll(/]*src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)]; matches.forEach(match => { // base the matched image URL relative to the post URL const url = new URL(match[1], postLink).href; diff --git a/src/translator.js b/src/translator.js index 2a24452..c974599 100644 --- a/src/translator.js +++ b/src/translator.js @@ -105,7 +105,7 @@ function getPostContent(postData, turndownService, config) { if (config.saveScrapedImages) { // writeImageFile() will save all content images to a relative /images // folder so update references in post content to match - content = content.replace(/(]*src=").*?([^/"]+\.(?:gif|jpe?g|png))("[^>]*>)/gi, '$1images/$2$3'); + content = content.replace(/(]*src=").*?([^/"]+\.(?:gif|jpe?g|png|webp))("[^>]*>)/gi, '$1images/$2$3'); } // preserve "more" separator, max one per post, optionally with custom label diff --git a/src/wizard.js b/src/wizard.js index 5285b8d..88a722d 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -74,6 +74,12 @@ const options = [ type: 'boolean', description: 'Include custom post types and pages', default: false + }, + { + name: 'disable-strict-ssl', + type: 'boolean', + description: 'Strict SSL prevents image retrieval from self-signed servers', + default: false } ]; diff --git a/src/writer.js b/src/writer.js index b782756..fa02ce1 100644 --- a/src/writer.js +++ b/src/writer.js @@ -16,7 +16,7 @@ async function processPayloadsPromise(payloads, loadFunc) { const promises = payloads.map(payload => new Promise((resolve, reject) => { setTimeout(async () => { try { - const data = await loadFunc(payload.item); + const data = await loadFunc(payload.item, payload.strictSSL); await writeFile(payload.destinationPath, data); console.log(chalk.green('[OK]') + ' ' + payload.name); resolve(); @@ -55,6 +55,7 @@ async function writeMarkdownFilesPromise(posts, config ) { const payload = { item: post, name: (config.includeOtherTypes ? post.meta.type + ' - ' : '') + post.meta.slug, + strictSSL: !config.disableStrictSsl, destinationPath, delay }; @@ -72,7 +73,7 @@ async function writeMarkdownFilesPromise(posts, config ) { } } -async function loadMarkdownFilePromise(post) { +async function loadMarkdownFilePromise(post, strictSSL) { let output = '---\n'; Object.entries(post.frontmatter).forEach(([key, value]) => { @@ -117,6 +118,7 @@ async function writeImageFilesPromise(posts, config) { const payload = { item: imageUrl, name: filename, + strictSSL: !config.disableStrictSsl, destinationPath, delay }; @@ -135,7 +137,7 @@ async function writeImageFilesPromise(posts, config) { } } -async function loadImageFilePromise(imageUrl) { +async function loadImageFilePromise(imageUrl, strictSSL) { // only encode the URL if it doesn't already have encoded characters const url = (/%[\da-f]{2}/i).test(imageUrl) ? imageUrl : encodeURI(imageUrl); @@ -146,7 +148,8 @@ async function loadImageFilePromise(imageUrl) { encoding: null, // preserves binary encoding headers: { 'User-Agent': 'wordpress-export-to-markdown' - } + }, + strictSSL: strictSSL }); } catch (ex) { if (ex.name === 'StatusCodeError') { From d579d2604e8cefe18d697123bd1753551dd4a582 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Thu, 29 Feb 2024 14:39:13 -0500 Subject: [PATCH 2/2] Refactor from wizard option to advanced setting --- src/settings.js | 12 ++++++++---- src/wizard.js | 6 ------ src/writer.js | 12 +++++------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/settings.js b/src/settings.js index 02f51db..39e6334 100644 --- a/src/settings.js +++ b/src/settings.js @@ -18,10 +18,6 @@ exports.image_file_request_delay = 500; // overloaded. exports.markdown_file_write_delay = 25; -// Specify the timezone used for post dates. See available zone values and examples here: -// https://moment.github.io/luxon/#/zones?id=specifying-a-zone. -exports.custom_date_timezone = 'utc'; - // Enable this to include time with post dates. For example, "2020-12-25" would become // "2020-12-25T11:20:35.000Z". exports.include_time_with_date = false; @@ -31,6 +27,14 @@ exports.include_time_with_date = false; // set, this takes precedence over include_time_with_date. exports.custom_date_formatting = ''; +// Specify the timezone used for post dates. See available zone values and examples here: +// https://moment.github.io/luxon/#/zones?id=specifying-a-zone. +exports.custom_date_timezone = 'utc'; + // Categories to be excluded from post frontmatter. This does not filter out posts themselves, // just the categories listed in their frontmatter. exports.filter_categories = ['uncategorized']; + +// Strict SSL is enabled as the safe default when downloading images, but will not work with +// self-signed servers. You can disable it if you're getting a "self-signed certificate" error. +exports.strict_ssl = true; diff --git a/src/wizard.js b/src/wizard.js index 88a722d..5285b8d 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -74,12 +74,6 @@ const options = [ type: 'boolean', description: 'Include custom post types and pages', default: false - }, - { - name: 'disable-strict-ssl', - type: 'boolean', - description: 'Strict SSL prevents image retrieval from self-signed servers', - default: false } ]; diff --git a/src/writer.js b/src/writer.js index fa02ce1..7b0efce 100644 --- a/src/writer.js +++ b/src/writer.js @@ -16,7 +16,7 @@ async function processPayloadsPromise(payloads, loadFunc) { const promises = payloads.map(payload => new Promise((resolve, reject) => { setTimeout(async () => { try { - const data = await loadFunc(payload.item, payload.strictSSL); + const data = await loadFunc(payload.item); await writeFile(payload.destinationPath, data); console.log(chalk.green('[OK]') + ' ' + payload.name); resolve(); @@ -41,7 +41,7 @@ async function writeFile(destinationPath, data) { await fs.promises.writeFile(destinationPath, data); } -async function writeMarkdownFilesPromise(posts, config ) { +async function writeMarkdownFilesPromise(posts, config) { // package up posts into payloads let skipCount = 0; let delay = 0; @@ -55,7 +55,6 @@ async function writeMarkdownFilesPromise(posts, config ) { const payload = { item: post, name: (config.includeOtherTypes ? post.meta.type + ' - ' : '') + post.meta.slug, - strictSSL: !config.disableStrictSsl, destinationPath, delay }; @@ -73,7 +72,7 @@ async function writeMarkdownFilesPromise(posts, config ) { } } -async function loadMarkdownFilePromise(post, strictSSL) { +async function loadMarkdownFilePromise(post) { let output = '---\n'; Object.entries(post.frontmatter).forEach(([key, value]) => { @@ -118,7 +117,6 @@ async function writeImageFilesPromise(posts, config) { const payload = { item: imageUrl, name: filename, - strictSSL: !config.disableStrictSsl, destinationPath, delay }; @@ -137,7 +135,7 @@ async function writeImageFilesPromise(posts, config) { } } -async function loadImageFilePromise(imageUrl, strictSSL) { +async function loadImageFilePromise(imageUrl) { // only encode the URL if it doesn't already have encoded characters const url = (/%[\da-f]{2}/i).test(imageUrl) ? imageUrl : encodeURI(imageUrl); @@ -149,7 +147,7 @@ async function loadImageFilePromise(imageUrl, strictSSL) { headers: { 'User-Agent': 'wordpress-export-to-markdown' }, - strictSSL: strictSSL + strictSSL: settings.strict_ssl }); } catch (ex) { if (ex.name === 'StatusCodeError') {