Merge pull request #116 from lonekorean/ssl-and-webp

SSL and .webp

src/parser.js

@@ -113,7 +113,7 @@ function getPostCoverImageId(postData) {
 function collectAttachedImages(channelData) {
 	const images = getItemsOfType(channelData, 'attachment')
 		// filter to certain image file types
-		.filter(attachment => attachment.attachment_url && (/\.(gif|jpe?g|png)$/i).test(attachment.attachment_url[0]))
+		.filter(attachment => attachment.attachment_url && (/\.(gif|jpe?g|png|webp)$/i).test(attachment.attachment_url[0]))
 		.map(attachment => ({
 			id: attachment.post_id[0],
 			postId: attachment.post_parent[0],
@@ -132,7 +132,7 @@ function collectScrapedImages(channelData, postTypes) {
 			const postContent = postData.encoded[0];
 			const postLink = postData.link[0];
 
-			const matches = [...postContent.matchAll(/<img[^>]*src="(.+?\.(?:gif|jpe?g|png))"[^>]*>/gi)];
+			const matches = [...postContent.matchAll(/<img[^>]*src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)];
 			matches.forEach(match => {
 				// base the matched image URL relative to the post URL
 				const url = new URL(match[1], postLink).href;

src/settings.js

@@ -18,10 +18,6 @@ exports.image_file_request_delay = 500;
 // overloaded.
 exports.markdown_file_write_delay = 25;
 
-// Specify the timezone used for post dates. See available zone values and examples here:
-// https://moment.github.io/luxon/#/zones?id=specifying-a-zone.
-exports.custom_date_timezone = 'utc';
-
 // Enable this to include time with post dates. For example, "2020-12-25" would become
 // "2020-12-25T11:20:35.000Z".
 exports.include_time_with_date = false;
@@ -31,6 +27,14 @@ exports.include_time_with_date = false;
 // set, this takes precedence over include_time_with_date.
 exports.custom_date_formatting = '';
 
+// Specify the timezone used for post dates. See available zone values and examples here:
+// https://moment.github.io/luxon/#/zones?id=specifying-a-zone.
+exports.custom_date_timezone = 'utc';
+
 // Categories to be excluded from post frontmatter. This does not filter out posts themselves,
 // just the categories listed in their frontmatter.
 exports.filter_categories = ['uncategorized'];
+
+// Strict SSL is enabled as the safe default when downloading images, but will not work with
+// self-signed servers. You can disable it if you're getting a "self-signed certificate" error.
+exports.strict_ssl = true;

src/translator.js

@@ -105,7 +105,7 @@ function getPostContent(postData, turndownService, config) {
 	if (config.saveScrapedImages) {
 		// writeImageFile() will save all content images to a relative /images
 		// folder so update references in post content to match
-		content = content.replace(/(<img[^>]*src=").*?([^/"]+\.(?:gif|jpe?g|png))("[^>]*>)/gi, '$1images/$2$3');
+		content = content.replace(/(<img[^>]*src=").*?([^/"]+\.(?:gif|jpe?g|png|webp))("[^>]*>)/gi, '$1images/$2$3');
 	}
 
 	// preserve "more" separator, max one per post, optionally with custom label

src/writer.js

@@ -41,7 +41,7 @@ async function writeFile(destinationPath, data) {
 	await fs.promises.writeFile(destinationPath, data);
 }
 
-async function writeMarkdownFilesPromise(posts, config ) {
+async function writeMarkdownFilesPromise(posts, config) {
 	// package up posts into payloads
 	let skipCount = 0;
 	let delay = 0;
@@ -146,7 +146,8 @@ async function loadImageFilePromise(imageUrl) {
 			encoding: null, // preserves binary encoding
 			headers: {
 				'User-Agent': 'wordpress-export-to-markdown'
-			}
+			},
+			strictSSL: settings.strict_ssl
 		});
 	} catch (ex) {
 		if (ex.name === 'StatusCodeError') {