import fs from 'fs'; import * as luxon from 'luxon'; import * as data from './data.js'; import * as frontmatter from './frontmatter.js'; import * as shared from './shared.js'; import * as translator from './translator.js'; export async function parseFilePromise() { console.log('\nParsing...'); const content = await fs.promises.readFile(shared.config.input, 'utf8'); const rssData = await data.load(content); const allPostData = rssData.child('channel').children('item'); const postTypes = getPostTypes(allPostData); const posts = collectPosts(allPostData, postTypes); const images = []; if (shared.config.saveImages === 'attached' || shared.config.saveImages === 'all') { images.push(...collectAttachedImages(allPostData)); } if (shared.config.saveImages === 'scraped' || shared.config.saveImages === 'all') { images.push(...collectScrapedImages(allPostData, postTypes)); } mergeImagesIntoPosts(images, posts); populateFrontmatter(posts); return posts; } function getPostTypes(allPostData) { // search export file for all post types minus some specific types we don't want const postTypes = allPostData .map((postData) => postData.childValue('post_type')) .filter((postType) => ![ 'attachment', 'revision', 'nav_menu_item', 'custom_css', 'customize_changeset', 'oembed_cache', 'user_request', 'wp_block', 'wp_global_styles', 'wp_navigation', 'wp_template', 'wp_template_part' ].includes(postType)); return [...new Set(postTypes)]; // remove duplicates } function getItemsOfType(allPostData, type) { return allPostData.filter(item => item.childValue('post_type') === type); } function collectPosts(allPostData, postTypes) { let allPosts = []; postTypes.forEach(postType => { const postsForType = getItemsOfType(allPostData, postType) .filter(postData => postData.childValue('status') !== 'trash') .filter(postData => !(postType === 'page' && postData.childValue('post_name') === 'sample-page')) .map(postData => buildPost(postData)); if (postsForType.length > 0) { console.log(`${postsForType.length} posts of type "${postType}" found.`); } allPosts.push(...postsForType); }); return allPosts; } function buildPost(data) { return { // full raw post data data, // body content converted to markdown content: translator.getPostContent(data.childValue('encoded')), // particularly useful values for all sorts of things type: data.childValue('post_type'), id: data.childValue('post_id'), isDraft: data.childValue('status') === 'draft', slug: decodeURIComponent(data.childValue('post_name')), date: getPostDate(data), coverImageId: getPostMetaValue(data, '_thumbnail_id'), // these are possibly set later in mergeImagesIntoPosts() coverImage: undefined, imageUrls: [] }; } function getPostDate(data) { const date = luxon.DateTime.fromRFC2822(data.childValue('pubDate'), { zone: shared.config.customDateTimezone }); return date.isValid ? date : undefined; } function getPostMetaValue(data, key) { const metas = data.children('postmeta'); const meta = metas.find((meta) => meta.childValue('meta_key') === key); return meta ? meta.childValue('meta_value') : undefined; } function collectAttachedImages(allPostData) { const images = getItemsOfType(allPostData, 'attachment') // filter to certain image file types .filter(attachment => { const url = attachment.childValue('attachment_url'); return url && (/\.(gif|jpe?g|png|webp)$/i).test(url); }) .map(attachment => ({ id: attachment.childValue('post_id'), postId: attachment.optionalChildValue('post_parent') ?? 'nope', // may not exist (cover image in a squarespace export, for example) url: attachment.childValue('attachment_url') })); console.log(images.length + ' attached images found.'); return images; } function collectScrapedImages(allPostData, postTypes) { const images = []; postTypes.forEach(postType => { getItemsOfType(allPostData, postType).forEach(postData => { const postId = postData.childValue('post_id'); const postContent = postData.childValue('encoded'); const scrapedUrls = [...postContent.matchAll(/]*?src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)].map((match) => match[1]); scrapedUrls.forEach((scrapedUrl) => { let url; if (isAbsoluteUrl(scrapedUrl)) { url = scrapedUrl; } else { const postLink = postData.childValue('link'); if (isAbsoluteUrl(postLink)) { url = new URL(scrapedUrl, postLink).href; } else { throw new Error(`Unable to determine absolute URL from scraped image URL '${scrapedUrl}' and post link URL '${postLink}'.`); } } images.push({ id: 'nope', // scraped images don't have an id postId, url }); }); }); }); console.log(images.length + ' images scraped from post body content.'); return images; } function mergeImagesIntoPosts(images, posts) { images.forEach(image => { posts.forEach(post => { let shouldAttach = false; // this image was uploaded as an attachment to this post if (image.postId === post.id) { shouldAttach = true; } // this image was set as the featured image for this post if (image.id === post.coverImageId) { shouldAttach = true; post.coverImage = shared.getFilenameFromUrl(image.url); } if (shouldAttach && !post.imageUrls.includes(image.url)) { post.imageUrls.push(image.url); } }); }); } function populateFrontmatter(posts) { posts.forEach(post => { post.frontmatter = {}; shared.config.frontmatterFields.forEach(field => { const [key, alias] = field.split(':'); let frontmatterGetter = frontmatter[key]; if (!frontmatterGetter) { throw `Could not find a frontmatter getter named "${key}".`; } post.frontmatter[alias ?? key] = frontmatterGetter(post); }); }); } function isAbsoluteUrl(url) { return (/^https?:\/\//i).test(url); }