diff --git a/src/parser.js b/src/parser.js
index b252dc9..1b705cc 100644
--- a/src/parser.js
+++ b/src/parser.js
@@ -13,6 +13,8 @@ async function parseFilePromise(config) {
tagNameProcessors: [xml2js.processors.stripPrefix]
});
+ let posts = collectPosts(data);
+
let images = [];
if (config.saveattachedimages) {
images.push(...collectAttachedImages(data));
@@ -21,55 +23,20 @@ async function parseFilePromise(config) {
images.push(...collectScrapedImages(data));
}
- let posts = collectPosts(data);
mergeImagesIntoPosts(images, posts);
return Promise.resolve(posts);
}
-function collectAttachedImages(data) {
- let images = getItemsOfType(data, 'attachment')
- // filter to certain image file types
- .filter(attachment => (/\.(gif|jpe?g|png)$/i).test(attachment.attachment_url[0]))
- .map(attachment => ({
- id: attachment.post_id[0],
- postId: attachment.post_parent[0],
- url: attachment.attachment_url[0]
- }));
-
- return images;
-}
-
-function collectScrapedImages(data) {
- let images = [];
-
- getItemsOfType(data, 'post').forEach(post => {
- let postId = post.post_id[0];
- let postContent = post.encoded[0];
- let postLink = post.link[0];
-
- let matches = [...postContent.matchAll(/
]*src="(.+?\.(?:gif|jpe?g|png))"[^>]*>/gi)];
- matches.forEach(match => {
- // base the matched image URL relative to the post URL
- let url = new URL(match[1], postLink).href;
-
- images.push({
- id: -1,
- postId: postId,
- url: url
- });
- console.log('Scraped ' + url + '.');
- });
- });
-
- return images;
+function getItemsOfType(data, type) {
+ return data.rss.channel[0].item.filter(item => item.post_type[0] === type);
}
function collectPosts(data) {
// this is passed into getPostContent() for the markdown conversion
turndownService = translator.initTurndownService();
- return getItemsOfType(data, 'post')
+ let posts = getItemsOfType(data, 'post')
.map(post => ({
// meta data isn't written to file, but is used to help with other things
meta: {
@@ -84,10 +51,9 @@ function collectPosts(data) {
},
content: translator.getPostContent(post, turndownService, config)
}));
-}
-function getItemsOfType(data, type) {
- return data.rss.channel[0].item.filter(item => item.post_type[0] === type);
+ console.log(posts.length + ' posts found.');
+ return posts;
}
function getPostId(post) {
@@ -113,6 +79,44 @@ function getPostDate(post) {
return luxon.DateTime.fromRFC2822(post.pubDate[0], { zone: 'utc' }).toISODate();
}
+function collectAttachedImages(data) {
+ let images = getItemsOfType(data, 'attachment')
+ // filter to certain image file types
+ .filter(attachment => (/\.(gif|jpe?g|png)$/i).test(attachment.attachment_url[0]))
+ .map(attachment => ({
+ id: attachment.post_id[0],
+ postId: attachment.post_parent[0],
+ url: attachment.attachment_url[0]
+ }));
+
+ console.log(images.length + ' attached images found.');
+ return images;
+}
+
+function collectScrapedImages(data) {
+ let images = [];
+ getItemsOfType(data, 'post').forEach(post => {
+ let postId = post.post_id[0];
+ let postContent = post.encoded[0];
+ let postLink = post.link[0];
+
+ let matches = [...postContent.matchAll(/
]*src="(.+?\.(?:gif|jpe?g|png))"[^>]*>/gi)];
+ matches.forEach(match => {
+ // base the matched image URL relative to the post URL
+ let url = new URL(match[1], postLink).href;
+
+ images.push({
+ id: -1,
+ postId: postId,
+ url: url
+ });
+ });
+ });
+
+ console.log(images.length + ' images scraped from post body content.');
+ return images;
+}
+
function mergeImagesIntoPosts(images, posts) {
// create lookup table for quicker traversal
let postsLookup = posts.reduce((lookup, post) => {
diff --git a/src/translator.js b/src/translator.js
index f5df86f..3b90d50 100644
--- a/src/translator.js
+++ b/src/translator.js
@@ -1,7 +1,5 @@
const turndown = require('turndown');
-const shared = require('./shared');
-
function initTurndownService() {
let turndownService = new turndown({
headingStyle: 'atx',
@@ -47,8 +45,7 @@ function initTurndownService() {
turndownService.addRule('iframe', {
filter: 'iframe',
replacement: (content, node) => {
- let html = node.outerHTML
- .replace('allowfullscreen=""', 'allowfullscreen');
+ let html = node.outerHTML.replace('allowfullscreen=""', 'allowfullscreen');
return '\n\n' + html + '\n\n';
}
});