wordpress-export-to-markdown/src/writer.js

const chalk = require('chalk');
const fs = require('fs');
const luxon = require('luxon');
const path = require('path');
const requestPromiseNative = require('request-promise-native');

const shared = require('./shared');
const settings = require('./settings');

async function writeFilesPromise(posts, config) {
	await writeMarkdownFilesPromise(posts, config);
	await writeImageFilesPromise(posts, config);
}

async function processPayloadsPromise(payloads, loadFunc, config) {
	const promises = payloads.map(payload => new Promise((resolve, reject) => {
		setTimeout(async () => {
			try {
				const data = await loadFunc(payload.item, config);
				await writeFile(payload.destinationPath, data);
				console.log(chalk.green('[OK]') + ' ' + payload.name);
				resolve();
			} catch (ex) {
				console.log(chalk.red('[FAILED]') + ' ' + payload.name + ' ' + chalk.red('(' + ex.toString() + ')'));
				reject();
			}
		}, payload.delay);
	}));
	
	const results = await Promise.allSettled(promises);
	const failedCount = results.filter(result => result.status === 'rejected').length;
	if (failedCount === 0) {
		console.log('Done, got them all!');
	} else {
		console.log('Done, but with ' + chalk.red(failedCount + ' failed') + '.');
	}
}

async function writeFile(destinationPath, data) {
	await fs.promises.mkdir(path.dirname(destinationPath), { recursive: true });
	await fs.promises.writeFile(destinationPath, data);
}

async function writeMarkdownFilesPromise(posts, config ) {
	// package up posts into payloads
	const payloads = posts.map((post, index) => ({
		item: post,
		name: post.meta.slug,
		destinationPath: getPostPath(post, config),
		delay: index * settings.markdown_file_write_delay
	}));

	console.log('\nSaving posts...');
	await processPayloadsPromise(payloads, loadMarkdownFilePromise, config);
}

async function loadMarkdownFilePromise(post) {
	let output = '---\n';
	Object.entries(post.frontmatter).forEach(pair => {
		const key = pair[0];
		const value = (pair[1] || '').replace(/"/g, '\\"');
		output += key + ': "' + value + '"\n';
	});
	output += '---\n\n' + post.content + '\n';
	return output;
}

async function writeImageFilesPromise(posts, config) {
	// collect image data from all posts into a single flattened array of payloads
	let delay = 0;
	const payloads = posts.flatMap(post => {
		const postPath = getPostPath(post, config);
		const imagesDir = path.join(path.dirname(postPath), 'images');
		return post.meta.imageUrls.map(imageUrl => {
			const filename = shared.getFilenameFromUrl(imageUrl);
			const payload = {
				item: imageUrl,
				name: filename,
				destinationPath: path.join(imagesDir, filename),
				delay
			};
			delay += settings.image_file_request_delay;
			return payload;
		});
	});

	if (payloads.length > 0) {
		console.log('\nDownloading and saving images...');
		await processPayloadsPromise(payloads, loadImageFilePromise);
	} else {
		console.log('\nNo images to download and save...');
	}
}

async function loadImageFilePromise(imageUrl) {
	let buffer;
	try {
		buffer = await requestPromiseNative.get({
			url: imageUrl,
			encoding: null, // preserves binary encoding
			headers: {
				'User-Agent' : 'wordpress-export-to-markdown'
			}
		});
	} catch (ex) {
		if (ex.name === 'StatusCodeError') {
			// these errors contain a lot of noise, simplify to just the status code
			ex.message = ex.statusCode;
		}
		throw ex;
	}
	return buffer;
}

function getPostPath(post, config) {
	const dt = luxon.DateTime.fromISO(post.frontmatter.date);

	// start with base output dir
	const pathSegments = [config.output];

	if (config.yearFolders) {
		pathSegments.push(dt.toFormat('yyyy'));
	}

	if (config.monthFolders) {
		pathSegments.push(dt.toFormat('LL'));
	}

	// create slug fragment, possibly date prefixed
	let slugFragment = post.meta.slug;
	if (config.prefixDate) {
		slugFragment = dt.toFormat('yyyy-LL-dd') + '-' + slugFragment;
	}

	// use slug fragment as folder or filename as specified
	if (config.postFolders) {
		pathSegments.push(slugFragment, 'index.md');
	} else {
		pathSegments.push(slugFragment + '.md');
	}

	return path.join(...pathSegments);
}

exports.writeFilesPromise = writeFilesPromise;