mirror of
https://github.com/10h30/wordpress-export-to-markdown.git
synced 2026-07-13 03:36:36 +09:00
Will Boyd
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"env": {
|
||||
"browser": true,
|
||||
"commonjs": true,
|
||||
"es6": true
|
||||
},
|
||||
"extends": "eslint:recommended",
|
||||
"globals": {
|
||||
"Atomics": "readonly",
|
||||
"SharedArrayBuffer": "readonly"
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2018
|
||||
},
|
||||
"rules": {
|
||||
}
|
||||
}
|
||||
@@ -1,135 +1,156 @@
|
||||
# wordpress-export-to-markdown
|
||||
|
||||
Converts a WordPress export XML file into Markdown files.
|
||||
A script that converts a WordPress export XML file into Markdown files suitable for a static site generator ([Gatsby](https://www.gatsbyjs.org/), [Hugo](https://gohugo.io/), [Jekyll](https://jekyllrb.com/), etc.).
|
||||
|
||||
Useful if you want to migrate from WordPress to a static site generator ([Gatsby](https://www.gatsbyjs.org/), [Hugo](https://gohugo.io/), [Jekyll](https://jekyllrb.com/), etc.).
|
||||
Each post is saved as a separate Markdown file with appropriate frontmatter. Images are also downloaded and saved. Embedded content from YouTube, Twitter, CodePen, etc. is carefully preserved.
|
||||
|
||||
Saves each post as a separate file with appropriate frontmatter. Also saves attached images and (optionally) any additional images found in post body content. Posts and images can be saved into a variety of folder structures.
|
||||
|
||||
|
||||
## Quick Start
|
||||
|
||||
You'll need:
|
||||
- [Node.js](https://nodejs.org/) v10.12 or later
|
||||
- Your [WordPress export file](https://codex.wordpress.org/Tools_Export_Screen)
|
||||
- [Node.js](https://nodejs.org/) v12.14 or later
|
||||
- Your [WordPress export file](https://wordpress.org/support/article/tools-export-screen/)
|
||||
|
||||
There are a few ways you can use this package:
|
||||
It is recommended that you drop your WordPress export file into the same directory that you run this script from so it's easy to find.
|
||||
|
||||
1. To run via `npx`, run `npx wordpress-export-to-markdown`
|
||||
2. To add to an existing repo, run `npm i wordpress-export-to-markdown` and then `wordpress-export-to-markdown`
|
||||
3. Clone this repo, open your terminal to this package's directory, then run `npm install` and then `node index.js`
|
||||
|
||||
This will create an `/output` folder filled with your posts and images.
|
||||
|
||||
## Customization
|
||||
|
||||
You can use command line arguments to control options for how the script runs. For example, this will give you [Jekyll](https://jekyllrb.com/)-style output in terms of folder structure and filenames:
|
||||
You can run this script immediately in your terminal with `npx`:
|
||||
|
||||
```
|
||||
node index.js --postfolders=false --prefixdate=true
|
||||
npx wordpress-export-to-markdown
|
||||
```
|
||||
|
||||
### --input
|
||||
Or you can clone and run (this makes repeated runs faster and allows you to tinker with the code). After cloning this repo, open your terminal to the package's directory and run:
|
||||
|
||||
- Type: String
|
||||
```
|
||||
npm install && node index.js
|
||||
```
|
||||
|
||||
Either way you run it, the script will start the wizard. Answer the questions and off you go!
|
||||
|
||||
## Command Line
|
||||
|
||||
The wizard makes it easy to configure your options, but you can also do so via the command line if you want. For example, the following will give you [Jekyll](https://jekyllrb.com/)-style output in terms of folder structure and filenames.
|
||||
|
||||
Using `npx`:
|
||||
|
||||
```
|
||||
npx wordpress-export-to-markdown --post-folders=false --prefix-date=true
|
||||
```
|
||||
|
||||
Using a locally cloned repo:
|
||||
|
||||
```
|
||||
node index.js --post-folders=false --prefix-date=true
|
||||
```
|
||||
|
||||
The wizard will still ask you about any options not specifed on the command line. To skip the wizard entirely and use default values for unspecified options, add `--wizard=false`.
|
||||
|
||||
## Options
|
||||
|
||||
### Use wizard?
|
||||
|
||||
- Argument: `--wizard`
|
||||
- Type: `boolean`
|
||||
- Default: `true`
|
||||
|
||||
Enable to have the script prompt you for each option. Disable to skip the wizard and use default values for any options not specified via the command line.
|
||||
|
||||
### Path to WordPress export file?
|
||||
|
||||
- Argument: `--input`
|
||||
- Type: `file` (as a path string)
|
||||
- Default: `export.xml`
|
||||
|
||||
The file to parse. This should be the WordPress export XML file that you downloaded.
|
||||
The path to the WordPress export file that you want to parse. It is recommended that you drop your WordPress export file into the same directory that you run this script from so it's easy to find.
|
||||
|
||||
### --output
|
||||
### Path to output folder?
|
||||
|
||||
- Type: String
|
||||
- Argument: `--output`
|
||||
- Type: `folder` (as a path string)
|
||||
- Default: `output`
|
||||
|
||||
The output directory where Markdown and image files will be saved.
|
||||
The path to the output directory where Markdown and image files will be saved. If it does not exist, it will be created for you.
|
||||
|
||||
### --yearmonthfolders
|
||||
### Create year folders?
|
||||
|
||||
- Type: Boolean
|
||||
- Argument: `--year-folders`
|
||||
- Type: `boolean`
|
||||
- Default: `false`
|
||||
|
||||
Whether or not to organize output files into year and month folders.
|
||||
Whether or not to organize output files into folders by year.
|
||||
|
||||
/output
|
||||
/2017
|
||||
/01
|
||||
/02
|
||||
/2018
|
||||
/01
|
||||
### Create month folders?
|
||||
|
||||
### --yearfolders
|
||||
|
||||
- Type: Boolean
|
||||
- Argument: `--month-folders`
|
||||
- Type: `boolean`
|
||||
- Default: `false`
|
||||
|
||||
Whether or not to organize output files into year folders.
|
||||
Whether or not to organize output files into folders by month. You'll probably want to combine this with `--year-folders` to organize files by year then month.
|
||||
|
||||
/output
|
||||
/2017
|
||||
/2018
|
||||
### Create a folder for each post?
|
||||
|
||||
### --postfolders
|
||||
|
||||
- Type: Boolean
|
||||
- Argument: `--post-folders`
|
||||
- Type: `boolean`
|
||||
- Default: `true`
|
||||
|
||||
Whether or not to save files and images into post folders.
|
||||
|
||||
If `true`, the post slug is used for the folder name and the post's Markdown file is named `index.md`. Each post folder will have its own `/images` folder.
|
||||
|
||||
/output
|
||||
/first-post
|
||||
/images
|
||||
potato.png
|
||||
index.md
|
||||
/oh-look-another-post
|
||||
/images
|
||||
cat1.gif
|
||||
cat2.gif
|
||||
index.md
|
||||
/first-post
|
||||
/images
|
||||
potato.png
|
||||
index.md
|
||||
/second-post
|
||||
/images
|
||||
carrot.jpg
|
||||
celery.jpg
|
||||
index.md
|
||||
|
||||
If `false`, the post slug is used to name the post's Markdown file. These files will be side-by-side and images will go into a shared `/images` folder.
|
||||
|
||||
/output
|
||||
/images
|
||||
cat1.gif
|
||||
cat2.gif
|
||||
potato.png
|
||||
first-post.md
|
||||
oh-look-another-post.md
|
||||
/images
|
||||
carrot.jpg
|
||||
celery.jpg
|
||||
potato.png
|
||||
first-post.md
|
||||
second-post.md
|
||||
|
||||
Either way, this can be combined with with `--yearmonthfolderes` and `--yearfolders`, in which case the above output will be organized under the appropriate year and month folders.
|
||||
Either way, this can be combined with with `--year-folders` and `--month-folders`, in which case the above output will be organized under the appropriate year and month folders.
|
||||
|
||||
### --prefixdate
|
||||
### Prefix post folders/files with date?
|
||||
|
||||
- Type: Boolean
|
||||
- Argument: `--prefix-date`
|
||||
- Type: `boolean`
|
||||
- Default: `false`
|
||||
|
||||
Whether or not to prepend the post date to the post slug when naming a post's folder or file.
|
||||
|
||||
If `--postfolders` is `true`, this affects the folder.
|
||||
If `--post-folders` is `true`, this affects the folder.
|
||||
|
||||
/output
|
||||
/2017-01-14-first-post
|
||||
index.md
|
||||
/2017-01-23-oh-look-another-post
|
||||
index.md
|
||||
/2019-10-14-first-post
|
||||
index.md
|
||||
/2019-10-23-second-post
|
||||
index.md
|
||||
|
||||
If `--postfolders` is `false`, this affects the file.
|
||||
If `--post-folders` is `false`, this affects the file.
|
||||
|
||||
/output
|
||||
2017-01-14-first-post.md
|
||||
2017-01-23-oh-look-another-post.md
|
||||
2019-10-14-first-post.md
|
||||
2019-10-23-second-post.md
|
||||
|
||||
### --saveimages
|
||||
### Save images attached to posts?
|
||||
|
||||
- Type: Boolean
|
||||
- Argument: `--save-attached-images`
|
||||
- Type: `boolean`
|
||||
- Default: `true`
|
||||
|
||||
Whether or not to download and save images attached to posts. Generally speaking, these are images that were added by dragging/dropping or clicking **Add Media** or **Set Featured Image** when editing a post in WordPress. Images are saved into `/images`. See `--postfolders` for more details.
|
||||
Whether or not to download and save images attached to posts. Generally speaking, these are images that were uploaded by using **Add Media** or **Set Featured Image** when editing a post in WordPress. Images are saved into `/images`.
|
||||
|
||||
### --addcontentimages
|
||||
### Save images scraped from post body content?
|
||||
|
||||
- Type: Boolean
|
||||
- Default: `false`
|
||||
- Argument: `--save-scraped-images`
|
||||
- Type: `boolean`
|
||||
- Default: `true`
|
||||
|
||||
Whether or not to also include images scraped from <img> tags in post body content. These images are downloaded and saved along with other images as dictated by `--saveimages`. The <img> tags are updated to point to where the images are saved.
|
||||
Whether or not to download and save images scraped from <img> tags in post body content. Images are saved into `/images`. The <img> tags are updated to point to where the images are saved.
|
||||
|
||||
@@ -1,383 +1,27 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const luxon = require('luxon');
|
||||
const minimist = require('minimist');
|
||||
const path = require('path');
|
||||
const request = require('request');
|
||||
const turndown = require('turndown');
|
||||
const xml2js = require('xml2js');
|
||||
const process = require('process');
|
||||
|
||||
// global so various functions can access arguments
|
||||
let argv;
|
||||
const wizard = require('./src/wizard');
|
||||
const parser = require('./src/parser');
|
||||
const writer = require('./src/writer');
|
||||
|
||||
function init() {
|
||||
argv = minimist(process.argv.slice(2), {
|
||||
string: [
|
||||
'input',
|
||||
'output'
|
||||
],
|
||||
boolean: [
|
||||
'yearmonthfolders',
|
||||
'yearfolders',
|
||||
'postfolders',
|
||||
'prefixdate',
|
||||
'saveimages',
|
||||
'addcontentimages'
|
||||
],
|
||||
default: {
|
||||
input: 'export.xml',
|
||||
output: 'output',
|
||||
yearmonthfolders: false,
|
||||
yearfolders: false,
|
||||
postfolders: true,
|
||||
prefixdate: false,
|
||||
saveimages: true,
|
||||
addcontentimages: false
|
||||
}
|
||||
});
|
||||
(async () => {
|
||||
// parse any command line arguments and run wizard
|
||||
const config = await wizard.getConfig(process.argv);
|
||||
|
||||
let content = readFile(argv.input);
|
||||
parseFileContent(content);
|
||||
}
|
||||
// parse data from XML and do Markdown translations
|
||||
const posts = await parser.parseFilePromise(config)
|
||||
|
||||
function readFile(path) {
|
||||
try {
|
||||
return fs.readFileSync(path, 'utf8');
|
||||
} catch (ex) {
|
||||
console.log('Unable to read file.');
|
||||
console.log(ex.message);
|
||||
}
|
||||
}
|
||||
// write files, downloading images as needed
|
||||
await writer.writeFilesPromise(posts, config);
|
||||
|
||||
function parseFileContent(content) {
|
||||
const processors = { tagNameProcessors: [xml2js.processors.stripPrefix] };
|
||||
xml2js.parseString(content, processors, (err, data) => {
|
||||
if (err) {
|
||||
console.log('Unable to parse file content.');
|
||||
console.log(err);
|
||||
} else {
|
||||
processData(data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function processData(data) {
|
||||
let images = collectImages(data);
|
||||
let posts = collectPosts(data);
|
||||
mergeImagesIntoPosts(images, posts);
|
||||
writeFiles(posts);
|
||||
}
|
||||
|
||||
function collectImages(data) {
|
||||
// start by collecting all attachment images
|
||||
let images = getItemsOfType(data, 'attachment')
|
||||
// filter to certain image file types
|
||||
.filter(attachment => (/\.(gif|jpg|png)$/i).test(attachment.attachment_url[0]))
|
||||
.map(attachment => ({
|
||||
id: attachment.post_id[0],
|
||||
postId: attachment.post_parent[0],
|
||||
url: attachment.attachment_url[0]
|
||||
}));
|
||||
|
||||
// optionally add images scraped from <img> tags in post content
|
||||
if (argv.addcontentimages) {
|
||||
addContentImages(data, images);
|
||||
}
|
||||
|
||||
return images;
|
||||
}
|
||||
|
||||
function addContentImages(data, images) {
|
||||
let regex = (/<img[^>]*src="(.+?\.(?:gif|jpg|png))"[^>]*>/gi);
|
||||
let match;
|
||||
|
||||
getItemsOfType(data, 'post').forEach(post => {
|
||||
let postId = post.post_id[0];
|
||||
let postContent = post.encoded[0];
|
||||
let postLink = post.link[0];
|
||||
|
||||
// reset lastIndex since we're reusing the same regex object
|
||||
regex.lastIndex = 0;
|
||||
while ((match = regex.exec(postContent)) !== null) {
|
||||
// base the matched image URL relative to the post URL
|
||||
let url = new URL(match[1], postLink).href;
|
||||
|
||||
// add image if it hasn't already been added for this post
|
||||
let exists = images.some(image => image.postId === postId && image.url === url);
|
||||
if (!exists) {
|
||||
images.push({
|
||||
id: -1,
|
||||
postId: postId,
|
||||
url: url
|
||||
});
|
||||
console.log('Scraped ' + url + '.');
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function collectPosts(data) {
|
||||
// this is passed into getPostContent() for the markdown conversion
|
||||
turndownService = initTurndownService();
|
||||
|
||||
return getItemsOfType(data, 'post')
|
||||
.map(post => ({
|
||||
// meta data isn't written to file, but is used to help with other things
|
||||
meta: {
|
||||
id: getPostId(post),
|
||||
slug: getPostSlug(post),
|
||||
coverImageId: getPostCoverImageId(post)
|
||||
},
|
||||
frontmatter: {
|
||||
title: getPostTitle(post),
|
||||
date: getPostDate(post)
|
||||
},
|
||||
content: getPostContent(post, turndownService)
|
||||
}));
|
||||
}
|
||||
|
||||
function initTurndownService() {
|
||||
let turndownService = new turndown({
|
||||
headingStyle: 'atx',
|
||||
bulletListMarker: '-',
|
||||
codeBlockStyle: 'fenced'
|
||||
});
|
||||
|
||||
// preserve embedded tweets
|
||||
turndownService.addRule('tweet', {
|
||||
filter: node => node.nodeName === 'BLOCKQUOTE' && node.getAttribute('class') === 'twitter-tweet',
|
||||
replacement: (content, node) => '\n\n' + node.outerHTML
|
||||
});
|
||||
|
||||
// preserve embedded codepens
|
||||
turndownService.addRule('codepen', {
|
||||
filter: node => {
|
||||
// codepen embed snippets have changed over the years
|
||||
// but this series of checks should find the commonalities
|
||||
return (
|
||||
['P', 'DIV'].includes(node.nodeName) &&
|
||||
node.attributes['data-slug-hash'] &&
|
||||
node.getAttribute('class') === 'codepen'
|
||||
);
|
||||
},
|
||||
replacement: (content, node) => '\n\n' + node.outerHTML
|
||||
});
|
||||
|
||||
// preserve embedded scripts (for tweets, codepens, gists, etc.)
|
||||
turndownService.addRule('script', {
|
||||
filter: 'script',
|
||||
replacement: (content, node) => {
|
||||
let before = '\n\n';
|
||||
let src = node.getAttribute('src');
|
||||
if (node.previousSibling && node.previousSibling.nodeName !== '#text') {
|
||||
// keep twitter and codepen <script> tags snug with the element above them
|
||||
before = '\n';
|
||||
}
|
||||
let html = node.outerHTML.replace('async=""', 'async');
|
||||
return before + html + '\n\n';
|
||||
}
|
||||
});
|
||||
|
||||
// preserve iframes (common for embedded audio/video)
|
||||
turndownService.addRule('iframe', {
|
||||
filter: 'iframe',
|
||||
replacement: (content, node) => {
|
||||
let html = node.outerHTML
|
||||
.replace('allowfullscreen=""', 'allowfullscreen');
|
||||
return '\n\n' + html + '\n\n';
|
||||
}
|
||||
});
|
||||
|
||||
return turndownService;
|
||||
}
|
||||
|
||||
function getItemsOfType(data, type) {
|
||||
return data.rss.channel[0].item.filter(item => item.post_type[0] === type);
|
||||
}
|
||||
|
||||
function getPostId(post) {
|
||||
return post.post_id[0];
|
||||
}
|
||||
|
||||
function getPostCoverImageId(post) {
|
||||
if (post.postmeta === undefined) return;
|
||||
let postmeta = post.postmeta.find(postmeta => postmeta.meta_key[0] === '_thumbnail_id');
|
||||
let id = postmeta ? postmeta.meta_value[0] : undefined;
|
||||
return id;
|
||||
}
|
||||
|
||||
function getPostSlug(post) {
|
||||
return post.post_name[0];
|
||||
}
|
||||
|
||||
function getPostTitle(post) {
|
||||
return post.title[0].trim().replace(/"/g, '\\"');
|
||||
}
|
||||
|
||||
function getPostDate(post) {
|
||||
return luxon.DateTime.fromRFC2822(post.pubDate[0], { zone: 'utc' }).toISODate();
|
||||
}
|
||||
|
||||
function getPostContent(post, turndownService) {
|
||||
let content = post.encoded[0].trim();
|
||||
|
||||
// insert an empty div element between double line breaks
|
||||
// this nifty trick causes turndown to keep adjacent paragraphs separated
|
||||
// without mucking up content inside of other elemnts (like <code> blocks)
|
||||
content = content.replace(/(\r?\n){2}/g, '\n<div></div>\n');
|
||||
|
||||
if (argv.addcontentimages) {
|
||||
// writeImageFile() will save all content images to a relative /images
|
||||
// folder so update references in post content to match
|
||||
content = content.replace(/(<img[^>]*src=").*?([^\/"]+\.(?:gif|jpg|png))("[^>]*>)/gi, '$1images/$2$3');
|
||||
}
|
||||
|
||||
// this is a hack to make <iframe> nodes non-empty by inserting a "." which
|
||||
// allows the iframe rule declared in initTurndownService() to take effect
|
||||
// (using turndown's blankRule() and keep() solution did not work for me)
|
||||
content = content.replace(/(<\/iframe>)/gi, '.$1');
|
||||
|
||||
// use turndown to convert HTML to Markdown
|
||||
content = turndownService.turndown(content);
|
||||
|
||||
// clean up extra spaces in list items
|
||||
content = content.replace(/(-|\d+\.) +/g, '$1 ');
|
||||
|
||||
// clean up the "." from the iframe hack above
|
||||
content = content.replace(/\.(<\/iframe>)/gi, '$1');
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
function mergeImagesIntoPosts(images, posts) {
|
||||
// create lookup table for quicker traversal
|
||||
let postsLookup = posts.reduce((lookup, post) => {
|
||||
lookup[post.meta.id] = post;
|
||||
return lookup;
|
||||
}, {});
|
||||
|
||||
images.forEach(image => {
|
||||
let post = postsLookup[image.postId];
|
||||
if (post) {
|
||||
// save full image URLs for downloading later
|
||||
post.meta.imageUrls = post.meta.imageUrls || [];
|
||||
post.meta.imageUrls.push(image.url);
|
||||
|
||||
if (image.id === post.meta.coverImageId) {
|
||||
// save cover image filename to frontmatter
|
||||
post.frontmatter.coverImage = getFilenameFromUrl(image.url);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function writeFiles(posts) {
|
||||
let delay = 0;
|
||||
posts.forEach(post => {
|
||||
const postDir = getPostDir(post);
|
||||
createDir(postDir);
|
||||
writeMarkdownFile(post, postDir);
|
||||
|
||||
if (argv.saveimages && post.meta.imageUrls) {
|
||||
post.meta.imageUrls.forEach(imageUrl => {
|
||||
const imageDir = path.join(postDir, 'images');
|
||||
createDir(imageDir);
|
||||
writeImageFile(imageUrl, imageDir, delay);
|
||||
delay += 25;
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function writeMarkdownFile(post, postDir) {
|
||||
const frontmatter = Object.entries(post.frontmatter)
|
||||
.reduce((accumulator, pair) => {
|
||||
return accumulator + pair[0] + ': "' + pair[1] + '"\n'
|
||||
}, '');
|
||||
const data = '---\n' + frontmatter + '---\n\n' + post.content + '\n';
|
||||
|
||||
const postPath = path.join(postDir, getPostFilename(post));
|
||||
fs.writeFile(postPath, data, (err) => {
|
||||
if (err) {
|
||||
console.log('Unable to write file.')
|
||||
console.log(err);
|
||||
} else {
|
||||
console.log('Wrote ' + postPath + '.');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function writeImageFile(imageUrl, imageDir, delay) {
|
||||
let imagePath = path.join(imageDir, getFilenameFromUrl(imageUrl));
|
||||
let stream = fs.createWriteStream(imagePath);
|
||||
stream.on('finish', () => {
|
||||
console.log('Saved ' + imagePath + '.');
|
||||
});
|
||||
|
||||
// stagger image requests so we don't piss off hosts
|
||||
setTimeout(() => {
|
||||
request
|
||||
.get(imageUrl)
|
||||
.on('response', response => {
|
||||
if (response.statusCode !== 200) {
|
||||
console.log('Response status code ' + response.statusCode + ' received for ' + imageUrl + '.');
|
||||
}
|
||||
})
|
||||
.on('error', err => {
|
||||
console.log('Unable to download image.');
|
||||
console.log(err);
|
||||
})
|
||||
.pipe(stream);
|
||||
}, delay);
|
||||
}
|
||||
|
||||
function getFilenameFromUrl(url) {
|
||||
return url.split('/').slice(-1)[0];
|
||||
}
|
||||
|
||||
function createDir(dir) {
|
||||
try {
|
||||
fs.accessSync(dir, fs.constants.F_OK);
|
||||
} catch (ex) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
function getPostDir(post) {
|
||||
let dir = argv.output;
|
||||
let dt = luxon.DateTime.fromISO(post.frontmatter.date);
|
||||
|
||||
if (argv.yearmonthfolders) {
|
||||
dir = path.join(dir, dt.toFormat('yyyy'), dt.toFormat('LL'));
|
||||
} else if (argv.yearfolders) {
|
||||
dir = path.join(dir, dt.toFormat('yyyy'));
|
||||
}
|
||||
|
||||
if (argv.postfolders) {
|
||||
let folder = post.meta.slug;
|
||||
if (argv.prefixdate) {
|
||||
folder = dt.toFormat('yyyy-LL-dd') + '-' + folder;
|
||||
}
|
||||
dir = path.join(dir, folder);
|
||||
}
|
||||
|
||||
return dir;
|
||||
}
|
||||
|
||||
function getPostFilename(post) {
|
||||
if (argv.postfolders) {
|
||||
// the containing folder name will be unique, just use index.md here
|
||||
return 'index.md';
|
||||
} else {
|
||||
let filename = post.meta.slug + '.md';
|
||||
if (argv.prefixdate) {
|
||||
let dt = luxon.DateTime.fromISO(post.frontmatter.date);
|
||||
filename = dt.toFormat('yyyy-LL-dd') + '-' + filename;
|
||||
}
|
||||
return filename;
|
||||
}
|
||||
}
|
||||
|
||||
// it's go time!
|
||||
init();
|
||||
// happy goodbye
|
||||
console.log('\nAll done!');
|
||||
console.log('Look for your output files in: ' + path.resolve(config.output));
|
||||
})().catch(ex => {
|
||||
// sad goodbye
|
||||
console.log('\nSomething went wrong, execution halted early.');
|
||||
console.error(ex);
|
||||
});
|
||||
|
||||
Generated
+1029
-3
File diff suppressed because it is too large
Load Diff
+13
-4
@@ -5,9 +5,11 @@
|
||||
"main": "index.js",
|
||||
"repository": "https://github.com/lonekorean/wordpress-export-to-markdown.git",
|
||||
"keywords": [
|
||||
"wordpress",
|
||||
"blog",
|
||||
"convert",
|
||||
"export",
|
||||
"markdown",
|
||||
"export"
|
||||
"wordpress"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
@@ -15,15 +17,22 @@
|
||||
"author": "Will Boyd <will@codersblock.com> (https://codersblock.com)",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 10.12.0"
|
||||
"node": ">= 12.14.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"camelcase": "^5.3.1",
|
||||
"chalk": "^3.0.0",
|
||||
"commander": "^4.0.1",
|
||||
"inquirer": "^7.0.3",
|
||||
"luxon": "^1.21.3",
|
||||
"minimist": "^1.2.0",
|
||||
"request": "^2.88.0",
|
||||
"request-promise-native": "^1.0.8",
|
||||
"turndown": "^5.0.3",
|
||||
"xml2js": "^0.4.22"
|
||||
},
|
||||
"devDependencies": {
|
||||
"eslint": "^6.8.0"
|
||||
},
|
||||
"bin": {
|
||||
"wordpress-export-to-markdown": "./index.js"
|
||||
}
|
||||
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
const fs = require('fs');
|
||||
const luxon = require('luxon');
|
||||
const xml2js = require('xml2js');
|
||||
|
||||
const shared = require('./shared');
|
||||
const translator = require('./translator');
|
||||
|
||||
async function parseFilePromise(config) {
|
||||
console.log('\nParsing...');
|
||||
const content = await fs.promises.readFile(config.input, 'utf8');
|
||||
const data = await xml2js.parseStringPromise(content, {
|
||||
trim: true,
|
||||
tagNameProcessors: [xml2js.processors.stripPrefix]
|
||||
});
|
||||
|
||||
const posts = collectPosts(data, config);
|
||||
|
||||
const images = [];
|
||||
if (config.saveAttachedImages) {
|
||||
images.push(...collectAttachedImages(data));
|
||||
}
|
||||
if (config.saveScrapedImages) {
|
||||
images.push(...collectScrapedImages(data));
|
||||
}
|
||||
|
||||
mergeImagesIntoPosts(images, posts);
|
||||
|
||||
return posts;
|
||||
}
|
||||
|
||||
function getItemsOfType(data, type) {
|
||||
return data.rss.channel[0].item.filter(item => item.post_type[0] === type);
|
||||
}
|
||||
|
||||
function collectPosts(data, config) {
|
||||
// this is passed into getPostContent() for the markdown conversion
|
||||
const turndownService = translator.initTurndownService();
|
||||
|
||||
const posts = getItemsOfType(data, 'post')
|
||||
.map(post => ({
|
||||
// meta data isn't written to file, but is used to help with other things
|
||||
meta: {
|
||||
id: getPostId(post),
|
||||
slug: getPostSlug(post),
|
||||
coverImageId: getPostCoverImageId(post),
|
||||
imageUrls: []
|
||||
},
|
||||
frontmatter: {
|
||||
title: getPostTitle(post),
|
||||
date: getPostDate(post)
|
||||
},
|
||||
content: translator.getPostContent(post, turndownService, config)
|
||||
}));
|
||||
|
||||
console.log(posts.length + ' posts found.');
|
||||
return posts;
|
||||
}
|
||||
|
||||
function getPostId(post) {
|
||||
return post.post_id[0];
|
||||
}
|
||||
|
||||
function getPostSlug(post) {
|
||||
return post.post_name[0];
|
||||
}
|
||||
|
||||
function getPostCoverImageId(post) {
|
||||
if (post.postmeta === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const postmeta = post.postmeta.find(postmeta => postmeta.meta_key[0] === '_thumbnail_id');
|
||||
const id = postmeta ? postmeta.meta_value[0] : undefined;
|
||||
return id;
|
||||
}
|
||||
|
||||
function getPostTitle(post) {
|
||||
return post.title[0];
|
||||
}
|
||||
|
||||
function getPostDate(post) {
|
||||
return luxon.DateTime.fromRFC2822(post.pubDate[0], { zone: 'utc' }).toISODate();
|
||||
}
|
||||
|
||||
function collectAttachedImages(data) {
|
||||
const images = getItemsOfType(data, 'attachment')
|
||||
// filter to certain image file types
|
||||
.filter(attachment => (/\.(gif|jpe?g|png)$/i).test(attachment.attachment_url[0]))
|
||||
.map(attachment => ({
|
||||
id: attachment.post_id[0],
|
||||
postId: attachment.post_parent[0],
|
||||
url: attachment.attachment_url[0]
|
||||
}));
|
||||
|
||||
console.log(images.length + ' attached images found.');
|
||||
return images;
|
||||
}
|
||||
|
||||
function collectScrapedImages(data) {
|
||||
const images = [];
|
||||
getItemsOfType(data, 'post').forEach(post => {
|
||||
const postId = post.post_id[0];
|
||||
const postContent = post.encoded[0];
|
||||
const postLink = post.link[0];
|
||||
|
||||
const matches = [...postContent.matchAll(/<img[^>]*src="(.+?\.(?:gif|jpe?g|png))"[^>]*>/gi)];
|
||||
matches.forEach(match => {
|
||||
// base the matched image URL relative to the post URL
|
||||
const url = new URL(match[1], postLink).href;
|
||||
|
||||
images.push({
|
||||
id: -1,
|
||||
postId: postId,
|
||||
url: url
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
console.log(images.length + ' images scraped from post body content.');
|
||||
return images;
|
||||
}
|
||||
|
||||
function mergeImagesIntoPosts(images, posts) {
|
||||
// create lookup table for quicker traversal
|
||||
const postsLookup = posts.reduce((lookup, post) => {
|
||||
lookup[post.meta.id] = post;
|
||||
return lookup;
|
||||
}, {});
|
||||
|
||||
images.forEach(image => {
|
||||
const post = postsLookup[image.postId];
|
||||
if (post) {
|
||||
if (image.id === post.meta.coverImageId) {
|
||||
// save cover image filename to frontmatter
|
||||
post.frontmatter.coverImage = shared.getFilenameFromUrl(image.url);
|
||||
}
|
||||
|
||||
// save (unique) full image URLs for downloading later
|
||||
if (!post.meta.imageUrls.includes(image.url)) {
|
||||
post.meta.imageUrls.push(image.url);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
exports.parseFilePromise = parseFilePromise;
|
||||
@@ -0,0 +1,5 @@
|
||||
function getFilenameFromUrl(url) {
|
||||
return url.split('/').slice(-1)[0];
|
||||
}
|
||||
|
||||
exports.getFilenameFromUrl = getFilenameFromUrl;
|
||||
@@ -0,0 +1,88 @@
|
||||
const turndown = require('turndown');
|
||||
|
||||
function initTurndownService() {
|
||||
const turndownService = new turndown({
|
||||
headingStyle: 'atx',
|
||||
bulletListMarker: '-',
|
||||
codeBlockStyle: 'fenced'
|
||||
});
|
||||
|
||||
// preserve embedded tweets
|
||||
turndownService.addRule('tweet', {
|
||||
filter: node => node.nodeName === 'BLOCKQUOTE' && node.getAttribute('class') === 'twitter-tweet',
|
||||
replacement: (content, node) => '\n\n' + node.outerHTML
|
||||
});
|
||||
|
||||
// preserve embedded codepens
|
||||
turndownService.addRule('codepen', {
|
||||
filter: node => {
|
||||
// codepen embed snippets have changed over the years
|
||||
// but this series of checks should find the commonalities
|
||||
return (
|
||||
['P', 'DIV'].includes(node.nodeName) &&
|
||||
node.attributes['data-slug-hash'] &&
|
||||
node.getAttribute('class') === 'codepen'
|
||||
);
|
||||
},
|
||||
replacement: (content, node) => '\n\n' + node.outerHTML
|
||||
});
|
||||
|
||||
// preserve embedded scripts (for tweets, codepens, gists, etc.)
|
||||
turndownService.addRule('script', {
|
||||
filter: 'script',
|
||||
replacement: (content, node) => {
|
||||
let before = '\n\n';
|
||||
if (node.previousSibling && node.previousSibling.nodeName !== '#text') {
|
||||
// keep twitter and codepen <script> tags snug with the element above them
|
||||
before = '\n';
|
||||
}
|
||||
const html = node.outerHTML.replace('async=""', 'async');
|
||||
return before + html + '\n\n';
|
||||
}
|
||||
});
|
||||
|
||||
// preserve iframes (common for embedded audio/video)
|
||||
turndownService.addRule('iframe', {
|
||||
filter: 'iframe',
|
||||
replacement: (content, node) => {
|
||||
const html = node.outerHTML.replace('allowfullscreen=""', 'allowfullscreen');
|
||||
return '\n\n' + html + '\n\n';
|
||||
}
|
||||
});
|
||||
|
||||
return turndownService;
|
||||
}
|
||||
|
||||
function getPostContent(post, turndownService, config) {
|
||||
let content = post.encoded[0];
|
||||
|
||||
// insert an empty div element between double line breaks
|
||||
// this nifty trick causes turndown to keep adjacent paragraphs separated
|
||||
// without mucking up content inside of other elemnts (like <code> blocks)
|
||||
content = content.replace(/(\r?\n){2}/g, '\n<div></div>\n');
|
||||
|
||||
if (config.saveScrapedImages) {
|
||||
// writeImageFile() will save all content images to a relative /images
|
||||
// folder so update references in post content to match
|
||||
content = content.replace(/(<img[^>]*src=").*?([^/"]+\.(?:gif|jpe?g|png))("[^>]*>)/gi, '$1images/$2$3');
|
||||
}
|
||||
|
||||
// this is a hack to make <iframe> nodes non-empty by inserting a "." which
|
||||
// allows the iframe rule declared in initTurndownService() to take effect
|
||||
// (using turndown's blankRule() and keep() solution did not work for me)
|
||||
content = content.replace(/(<\/iframe>)/gi, '.$1');
|
||||
|
||||
// use turndown to convert HTML to Markdown
|
||||
content = turndownService.turndown(content);
|
||||
|
||||
// clean up extra spaces in list items
|
||||
content = content.replace(/(-|\d+\.) +/g, '$1 ');
|
||||
|
||||
// clean up the "." from the iframe hack above
|
||||
content = content.replace(/\.(<\/iframe>)/gi, '$1');
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
exports.initTurndownService = initTurndownService;
|
||||
exports.getPostContent = getPostContent;
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
const camelcase = require('camelcase');
|
||||
const commander = require('commander');
|
||||
const fs = require('fs');
|
||||
const inquirer = require('inquirer');
|
||||
const path = require('path');
|
||||
|
||||
const package = require('../package.json');
|
||||
|
||||
// all user options for command line and wizard are declard here
|
||||
const options = [
|
||||
// wizard must always be first
|
||||
{
|
||||
name: 'wizard',
|
||||
type: 'boolean',
|
||||
description: 'Use wizard',
|
||||
default: true
|
||||
},
|
||||
{
|
||||
name: 'input',
|
||||
type: 'file',
|
||||
description: 'Path to WordPress export file',
|
||||
default: 'export.xml'
|
||||
},
|
||||
{
|
||||
name: 'output',
|
||||
type: 'folder',
|
||||
description: 'Path to output folder',
|
||||
default: 'output'
|
||||
},
|
||||
{
|
||||
name: 'year-folders',
|
||||
type: 'boolean',
|
||||
description: 'Create year folders',
|
||||
default: false
|
||||
},
|
||||
{
|
||||
name: 'month-folders',
|
||||
type: 'boolean',
|
||||
description: 'Create month folders',
|
||||
default: false
|
||||
},
|
||||
{
|
||||
name: 'post-folders',
|
||||
type: 'boolean',
|
||||
description: 'Create a folder for each post',
|
||||
default: true
|
||||
},
|
||||
{
|
||||
name: 'prefix-date',
|
||||
type: 'boolean',
|
||||
description: 'Prefix post folders/files with date',
|
||||
default: false
|
||||
},
|
||||
{
|
||||
name: 'save-attached-images',
|
||||
type: 'boolean',
|
||||
description: 'Save images attached to posts',
|
||||
default: true
|
||||
},
|
||||
{
|
||||
name: 'save-scraped-images',
|
||||
type: 'boolean',
|
||||
description: 'Save images scraped from post body content',
|
||||
default: true
|
||||
}
|
||||
];
|
||||
|
||||
async function getConfig(argv) {
|
||||
extendOptionsData();
|
||||
const program = parseCommandLine(argv);
|
||||
|
||||
let answers;
|
||||
if (program.wizard) {
|
||||
console.log('\nStarting wizard...');
|
||||
const questions = options.map(option => ({
|
||||
when: option.name !== 'wizard' && !option.isProvided,
|
||||
name: camelcase(option.name),
|
||||
type: option.prompt,
|
||||
message: option.description + '?',
|
||||
default: option.default,
|
||||
|
||||
// these are not used for all option types and that's fine
|
||||
filter: option.coerce,
|
||||
validate: option.validate
|
||||
}));
|
||||
answers = await inquirer.prompt(questions);
|
||||
} else {
|
||||
console.log('\nSkipping wizard...');
|
||||
answers = {};
|
||||
}
|
||||
|
||||
const config = { ...program.opts(), ...answers };
|
||||
return config;
|
||||
}
|
||||
|
||||
function extendOptionsData() {
|
||||
// add more data to each option based on its type
|
||||
const map = {
|
||||
boolean: {
|
||||
prompt: 'confirm',
|
||||
coerce: coerceBoolean,
|
||||
},
|
||||
file: {
|
||||
prompt: 'input',
|
||||
coerce: coercePath,
|
||||
validate: validateFile
|
||||
},
|
||||
folder: {
|
||||
prompt: 'input',
|
||||
coerce: coercePath
|
||||
}
|
||||
};
|
||||
|
||||
options.forEach(option => {
|
||||
Object.assign(option, map[option.type]);
|
||||
});
|
||||
}
|
||||
|
||||
function parseCommandLine(argv) {
|
||||
// setup for help output
|
||||
commander
|
||||
.name('node index.js')
|
||||
.version('v' + package.version, '-v, --version', 'Display version number')
|
||||
.helpOption('-h, --help', 'See the thing you\'re looking at right now')
|
||||
.on('--help', () => {
|
||||
console.log('\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown');
|
||||
});
|
||||
|
||||
options.forEach(input => {
|
||||
const flag = '--' + input.name + ' <' + input.type + '>';
|
||||
const coerce = (value) => {
|
||||
// commander only calls coerce when an input is provided on the command line, which
|
||||
// makes for an easy way to flag (for later) if it should be excluded from the wizard
|
||||
input.isProvided = true;
|
||||
return input.coerce(value);
|
||||
};
|
||||
commander.option(flag, input.description, coerce, input.default);
|
||||
});
|
||||
|
||||
return commander.parse(argv);
|
||||
}
|
||||
|
||||
function coerceBoolean(value) {
|
||||
return !['false', 'no', '0'].includes(value.toLowerCase());
|
||||
}
|
||||
|
||||
function coercePath(value) {
|
||||
return path.normalize(value);
|
||||
}
|
||||
|
||||
function validateFile(value) {
|
||||
let isValid;
|
||||
try {
|
||||
isValid = fs.existsSync(value) && fs.statSync(value).isFile();
|
||||
} catch (ex) {
|
||||
isValid = false;
|
||||
}
|
||||
|
||||
return isValid ? true : 'Unable to find file: ' + path.resolve(value);
|
||||
}
|
||||
|
||||
exports.getConfig = getConfig;
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
const chalk = require('chalk');
|
||||
const fs = require('fs');
|
||||
const luxon = require('luxon');
|
||||
const path = require('path');
|
||||
const requestPromiseNative = require('request-promise-native');
|
||||
|
||||
const shared = require('./shared');
|
||||
|
||||
async function writeFilesPromise(posts, config) {
|
||||
await writeMarkdownFilesPromise(posts, config);
|
||||
await writeImageFilesPromise(posts, config);
|
||||
}
|
||||
|
||||
async function processPayloadsPromise(payloads, loadFunc, config) {
|
||||
const promises = payloads.map(payload => new Promise((resolve, reject) => {
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
const data = await loadFunc(payload.item, config);
|
||||
await writeFile(payload.destinationPath, data);
|
||||
console.log(chalk.green('[OK]') + ' ' + payload.name);
|
||||
resolve();
|
||||
} catch (ex) {
|
||||
console.error(chalk.red('[FAILED]') + ' ' + payload.name + ' ' + chalk.red('(' + ex.toString() + ')'));
|
||||
reject();
|
||||
}
|
||||
}, payload.delay);
|
||||
}));
|
||||
|
||||
const results = await Promise.allSettled(promises);
|
||||
const failedCount = results.filter(result => result.status === 'rejected').length;
|
||||
if (failedCount === 0) {
|
||||
console.log('Done, got them all!');
|
||||
} else {
|
||||
console.log('Done, but with ' + chalk.red(failedCount + ' failed') + '.');
|
||||
}
|
||||
}
|
||||
|
||||
async function writeFile(destinationPath, data) {
|
||||
await fs.promises.mkdir(path.dirname(destinationPath), { recursive: true });
|
||||
await fs.promises.writeFile(destinationPath, data);
|
||||
}
|
||||
|
||||
async function writeMarkdownFilesPromise(posts, config ) {
|
||||
// package up posts into payloads
|
||||
const payloads = posts.map((post, index) => ({
|
||||
item: post,
|
||||
name: post.meta.slug,
|
||||
destinationPath: getPostPath(post, config),
|
||||
delay: index * 25
|
||||
}));
|
||||
|
||||
console.log('\nSaving posts...');
|
||||
await processPayloadsPromise(payloads, loadMarkdownFilePromise, config);
|
||||
}
|
||||
|
||||
async function loadMarkdownFilePromise(post) {
|
||||
let output = '---\n';
|
||||
Object.entries(post.frontmatter).forEach(pair => {
|
||||
const key = pair[0];
|
||||
const value = pair[1].replace(/"/g, '\\"');
|
||||
output += key + ': "' + value + '"\n';
|
||||
});
|
||||
output += '---\n\n' + post.content + '\n';
|
||||
return output;
|
||||
}
|
||||
|
||||
async function writeImageFilesPromise(posts, config) {
|
||||
// collect image data from all posts into a single flattened array of payloads
|
||||
let delay = 0;
|
||||
const payloads = posts.flatMap(post => {
|
||||
const postPath = getPostPath(post, config);
|
||||
const imagesDir = path.join(path.dirname(postPath), 'images');
|
||||
return post.meta.imageUrls.map(imageUrl => {
|
||||
const filename = shared.getFilenameFromUrl(imageUrl);
|
||||
const payload = {
|
||||
item: imageUrl,
|
||||
name: filename,
|
||||
destinationPath: path.join(imagesDir, filename),
|
||||
delay
|
||||
};
|
||||
delay += 100;
|
||||
return payload;
|
||||
});
|
||||
});
|
||||
|
||||
if (payloads.length > 0) {
|
||||
console.log('\nDownloading and saving images...');
|
||||
await processPayloadsPromise(payloads, loadImageFilePromise);
|
||||
} else {
|
||||
console.log('\nNo images to download and save...');
|
||||
}
|
||||
}
|
||||
|
||||
async function loadImageFilePromise(imageUrl) {
|
||||
let buffer;
|
||||
try {
|
||||
buffer = await requestPromiseNative.get({
|
||||
url: imageUrl,
|
||||
encoding: null // preserves binary encoding
|
||||
});
|
||||
} catch (ex) {
|
||||
if (ex.name === 'StatusCodeError') {
|
||||
// these errors contain a lot of noise, simplify to just the status code
|
||||
ex.message = ex.statusCode;
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
function getPostPath(post, config) {
|
||||
const dt = luxon.DateTime.fromISO(post.frontmatter.date);
|
||||
|
||||
// start with base output dir
|
||||
const pathSegments = [config.output];
|
||||
|
||||
if (config.yearFolders) {
|
||||
pathSegments.push(dt.toFormat('yyyy'));
|
||||
}
|
||||
|
||||
if (config.monthFolders) {
|
||||
pathSegments.push(dt.toFormat('LL'));
|
||||
}
|
||||
|
||||
// create slug fragment, possibly date prefixed
|
||||
let slugFragment = post.meta.slug;
|
||||
if (config.prefixDate) {
|
||||
slugFragment = dt.toFormat('yyyy-LL-dd') + '-' + slugFragment;
|
||||
}
|
||||
|
||||
// use slug fragment as folder or filename as specified
|
||||
if (config.postFolders) {
|
||||
pathSegments.push(slugFragment, 'index.md');
|
||||
} else {
|
||||
pathSegments.push(slugFragment + '.md');
|
||||
}
|
||||
|
||||
return path.join(...pathSegments);
|
||||
}
|
||||
|
||||
exports.writeFilesPromise = writeFilesPromise;
|
||||
Reference in New Issue
Block a user