10h30/wordpress-export-to-markdown
1
0
Fork 0
mirror of https://github.com/10h30/wordpress-export-to-markdown.git synced 2026-07-14 12:16:50 +09:00
Code Issues Packages Projects Releases Wiki Activity

Error handling and logging for image downloads

Browse Source
This commit is contained in:
Will Boyd
2019-12-25 14:25:13 -05:00
parent 1b2153ad8b
commit a593d223f4
6 changed files with 105 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
index.js
+3
View File
@@ -1,8 +1,11 @@
const package = require('./package.json');
const wizard = require('./src/wizard');
const parser = require('./src/parser');
const writer = require('./src/writer');
(async () => {
console.log('Starting ' + package.name + ' v' + package.version + '...');
config = wizard.getConfig();
let posts = await parser.parseFilePromise(config)
await writer.writeFilesPromise(posts, config);
package-lock.json
Generated
+49
View File
@@ -4,6 +4,11 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@types/color-name": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
"integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ=="
},
"abab": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/abab/-/abab-2.0.3.tgz",
@@ -46,6 +51,15 @@
"uri-js": "^4.2.2"
}
},
"ansi-styles": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.0.tgz",
"integrity": "sha512-7kFQgnEaMdRtwf6uSfUnVr9gSGC7faurn+J/Mv90/W+iTtN0405/nLdopfMWwchyxhbGYl6TC4Sccn9TUkGAgg==",
"requires": {
"@types/color-name": "^1.1.1",
"color-convert": "^2.0.1"
}
},
"array-equal": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz",
@@ -102,6 +116,28 @@
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"chalk": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
"integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
"requires": {
"ansi-styles": "^4.1.0",
"supports-color": "^7.1.0"
}
},
"color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"requires": {
"color-name": "~1.1.4"
}
},
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -322,6 +358,11 @@
"function-bind": "^1.1.1"
}
},
"has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
},
"has-symbols": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz",
@@ -682,6 +723,14 @@
"function-bind": "^1.1.1"
}
},
"supports-color": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz",
"integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==",
"requires": {
"has-flag": "^4.0.0"
}
},
"symbol-tree": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
package.json
+1
View File
@@ -18,6 +18,7 @@
"node": ">= 12.14.0"
},
"dependencies": {
"chalk": "^3.0.0",
"luxon": "^1.21.3",
"minimist": "^1.2.0",
"request": "^2.88.0",
src/parser.js
+1
View File
@@ -6,6 +6,7 @@ const shared = require('./shared');
const translator = require('./translator');
async function parseFilePromise(config) {
console.log('\nParsing export file...');
const content = await fs.promises.readFile(config.input, 'utf8');
const data = await xml2js.parseStringPromise(content, {
trim: true,
src/wizard.js
+1 -1
View File
@@ -40,7 +40,7 @@ function getConfig() {
function checkFileExists(path) {
try {
return fs.existsSync(path);
} catch(ex) {
} catch (ex) {
return false;
}
}
src/writer.js
+50 -31
View File
@@ -1,3 +1,4 @@
const chalk = require('chalk');
const fs = require('fs');
const luxon = require('luxon');
const path = require('path');
@@ -11,8 +12,15 @@ async function writeFilesPromise(posts, config) {
}
async function writeMarkdownFilesPromise(posts) {
console.log('\nSaving posts...');
const promises = posts.map(writeMardownFilePromise);
const result = await Promise.allSettled(promises);
const result = await Promise.allSettled(promises);
const failedCount = results.filter(result => result.status === 'rejected').length;
if (failedCount === 0) {
console.log('Done, got them all!');
} else {
console.log('Done, but with ' + chalk.red(failedCount + ' failed') + '.');
}
}
async function writeMardownFilePromise(post) {
@@ -32,62 +40,73 @@ async function writeMardownFilePromise(post) {
}
async function writeImageFilesPromise(posts, config) {
// collect image data from all posts into a single flattened array
// collect image data from all posts into a single flattened array
let delay = 0;
let images = posts.flatMap(post => {
const postDir = getPostDir(post, config);
return post.meta.imageUrls.map(imageUrl => ({
postDir,
url: imageUrl,
url: imageUrl,
filename: shared.getFilenameFromUrl(imageUrl),
delay: delay += 25
}));
});
let progress = {
current: 0,
current: 1,
total: images.length
};
console.log('\nSaving images...');
const promises = images.map(writeImageFileDelayPromise.bind(this, progress));
const result = await Promise.allSettled(promises);
const results = await Promise.allSettled(promises);
const failedCount = results.filter(result => result.status === 'rejected').length;
if (failedCount === 0) {
console.log('Done, got them all!');
} else {
console.log('Done, but with ' + chalk.red(failedCount + ' failed') + '.');
}
}
async function writeImageFileDelayPromise(progress, image) {
await new Promise((resolve, reject) => {
setTimeout(async () => {
await writeImageFilePromise(progress, image);
resolve();
try {
await writeImageFilePromise(image);
console.log(chalk.green('[OK]') + ' ' + image.filename);
resolve();
} catch (ex) {
console.error(chalk.red('[FAILED]') + ' ' + image.filename + ' ' + chalk.red('(' + ex.toString() + ')'));
reject();
} finally {
progress.current++;
}
}, image.delay);
});
}
async function writeImageFilePromise(progress, image) {
const imageDir = path.join(image.postDir, 'images');
await createDirPromise(imageDir);
async function writeImageFilePromise(image) {
const imageDir = path.join(image.postDir, 'images');
await createDirPromise(imageDir);
const imagePath = path.join(imageDir, shared.getFilenameFromUrl(image.url));
const imagePath = path.join(imageDir, image.filename);
const stream = fs.createWriteStream(imagePath);
try {
const buffer = await requestPromiseNative.get({
url: image.url,
encoding: null // preserves binary encoding
});
stream.write(buffer);
} catch(ex) {
if (ex.statusCode !== undefined && ex.statusCode !== 200) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write('Response status code ' + ex.statusCode + ' received for ' + image.url + '.\n');
} else {
console.log(ex);
}
} finally {
progress.current++;
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write('Saving images: ' + progress.current + ' / ' + progress.total);
}
let buffer;
try {
buffer = await requestPromiseNative.get({
url: image.url,
encoding: null // preserves binary encoding
});
} catch (ex) {
if (ex.name === 'StatusCodeError') {
// these errors contain a lot of noise, simplify to just the status code
ex.message = ex.statusCode;
}
throw ex;
}
stream.write(buffer);
}
async function createDirPromise(dir) {