Rewrite of async logic in writer

index.js

@@ -6,7 +6,7 @@ async function init() {
 	try {
 		config = wizard.getConfig();
 		let posts = await parser.parseFilePromise(config)
-		writer.writeFiles(posts, config);
+		await writer.writeFilesPromise(posts, config);
 	} catch (ex) {
 		// appease the UnhandledPromiseRejectionWarning
 		console.error(ex);

package.json

@@ -21,6 +21,7 @@
 		"luxon": "^1.21.3",
 		"minimist": "^1.2.0",
 		"request": "^2.88.0",
+		"request-promise-native": "^1.0.8",
 		"turndown": "^5.0.3",
 		"xml2js": "^0.4.22"
 	}

src/parser.js

@@ -6,8 +6,7 @@ const shared = require('./shared');
 const translator = require('./translator');
 
 async function parseFilePromise(config) {
-	const content = fs.readFileSync(config.input, 'utf8');
-
+	const content = await fs.promises.readFile(config.input, 'utf8');
 	const data = await xml2js.parseStringPromise(content, {
 		trim: true,
 		tagNameProcessors: [xml2js.processors.stripPrefix]
@@ -25,7 +24,7 @@ async function parseFilePromise(config) {
 
 	mergeImagesIntoPosts(images, posts);
 
-	return Promise.resolve(posts);
+	return posts;
 }
 
 function getItemsOfType(data, type) {
@@ -72,7 +71,7 @@ function getPostSlug(post) {
 }
 
 function getPostTitle(post) {
-	return post.title[0].replace(/"/g, '\\"');
+	return post.title[0];
 }
 
 function getPostDate(post) {

src/writer.js

@@ -1,74 +1,97 @@
 const fs = require('fs');
 const luxon = require('luxon');
 const path = require('path');
-const request = require('request');
+const requestPromiseNative = require('request-promise-native');
 
 const shared = require('./shared');
 
-function writeFiles(posts, config) {
+async function writeFilesPromise(posts, config) {
+    await writeMarkdownFilesPromise(posts);
+    await writeImageFilesPromise(posts, config);
+}
+
+async function writeMarkdownFilesPromise(posts) {
+    const promises = posts.map(writeMardownFilePromise);
+    const result = await Promise.allSettled(promises);
+}
+
+async function writeMardownFilePromise(post) {
+    const postDir = getPostDir(post, config);
+    await createDirPromise(postDir);
+
+    let output = '---\n';
+    Object.entries(post.frontmatter).forEach(pair => {
+        const key = pair[0];
+        const value = pair[1].replace(/"/g, '\\"');
+        output += key + ': "' + value + '"\n';
+    });
+    output += '---\n\n' + post.content + '\n';
+    
+    const postPath = path.join(postDir, getPostFilename(post, config));
+    await fs.promises.writeFile(postPath, output);
+}
+
+async function writeImageFilesPromise(posts, config) {
+    // collect image data from all posts into a single flattened array
     let delay = 0;
-	posts.forEach(post => {
-		const postDir = getPostDir(post, config);
-		createDir(postDir);
-		writeMarkdownFile(post, postDir, config);
+    let images = posts.flatMap(post => {
+        const postDir = getPostDir(post, config);
+        return post.meta.imageUrls.map(imageUrl => ({
+            postDir,
+            url: imageUrl,
+            delay: delay += 25
+        }));
+    });
 
-        post.meta.imageUrls.forEach(imageUrl => {
-            const imageDir = path.join(postDir, 'images');
-            createDir(imageDir);
-            writeImageFile(imageUrl, imageDir, delay);
-            delay += 25;
+    let progress = {
+        current: 0,
+        total: images.length
+    };
+
+    const promises = images.map(writeImageFileDelayPromise.bind(this, progress));
+    const result = await Promise.allSettled(promises);
+}
+
+async function writeImageFileDelayPromise(progress, image) {
+    await new Promise((resolve, reject) => {
+        setTimeout(async () => {
+            await writeImageFilePromise(progress, image);
+            resolve();
+        }, image.delay);
+    });
+}
+
+async function writeImageFilePromise(progress, image) {
+    const imageDir = path.join(image.postDir, 'images');
+    await createDirPromise(imageDir);
+
+    const imagePath = path.join(imageDir, shared.getFilenameFromUrl(image.url));
+	const stream = fs.createWriteStream(imagePath);
+
+    try {
+        const buffer = await requestPromiseNative.get({
+            url: image.url,
+            encoding: null // preserves binary encoding
         });
-	});
+        stream.write(buffer);
+    } catch(ex) {
+        if (ex.statusCode !== undefined && ex.statusCode !== 200) {
+            process.stdout.clearLine();
+            process.stdout.cursorTo(0);
+            process.stdout.write('Response status code ' + ex.statusCode + ' received for ' + image.url + '.\n');
+        } else {
+            console.log(ex);
+        }
+    } finally {
+        progress.current++;
+        process.stdout.clearLine();
+        process.stdout.cursorTo(0);
+        process.stdout.write('Saving images: ' + progress.current + ' / ' + progress.total);
+    }
 }
 
-function writeMarkdownFile(post, postDir, config) {
-	const frontmatter = Object.entries(post.frontmatter)
-		.reduce((accumulator, pair) => {
-			return accumulator + pair[0] + ': "' + pair[1] + '"\n'
-		}, '');
-	const data = '---\n' + frontmatter + '---\n\n' + post.content + '\n';
-	
-	const postPath = path.join(postDir, getPostFilename(post, config));
-	fs.writeFile(postPath, data, (err) => {
-		if (err) {
-			console.log('Unable to write file.')
-			console.log(err);
-		} else {
-			console.log('Wrote ' + postPath + '.');
-		}
-	});
-}
-
-function writeImageFile(imageUrl, imageDir, delay) {
-	let imagePath = path.join(imageDir, shared.getFilenameFromUrl(imageUrl));
-	let stream = fs.createWriteStream(imagePath);
-	stream.on('finish', () => {
-		console.log('Saved ' + imagePath + '.');
-	});
-
-	// stagger image requests so we don't piss off hosts
-	setTimeout(() => {
-		request
-			.get(imageUrl)
-			.on('response', response => {
-				if (response.statusCode !== 200) {
-					console.log('Response status code ' + response.statusCode + ' received for ' + imageUrl + '.');
-				}
-			})
-			.on('error', err => {
-				console.log('Unable to download image.');
-				console.log(err);
-			})
-			.pipe(stream);
-	}, delay);
-}
-
-function createDir(dir) {
-	try {
-		fs.accessSync(dir, fs.constants.F_OK);
-	} catch (ex) {
-		fs.mkdirSync(dir, { recursive: true });
-	}
+async function createDirPromise(dir) {
+	return fs.promises.mkdir(dir, { recursive: true });
 }
 
 function getPostDir(post, config) {
@@ -106,4 +129,4 @@ function getPostFilename(post, config) {
 	}
 }
 
-exports.writeFiles = writeFiles;
+exports.writeFilesPromise = writeFilesPromise;