diff --git a/README.md b/README.md
index a03556a..13eb9ef 100644
--- a/README.md
+++ b/README.md
@@ -153,7 +153,15 @@ Whether or not to download and save images attached to posts. Generally speaking
- Type: `boolean`
- Default: `true`
-Whether or not to download and save images scraped from <img> tags in post body content. Images are saved into `/images`. The <img> tags are updated to point to where the images are saved.
+Whether or not to download and save images scraped from `![]()
` tags in post body content. Images are saved into `/images`. The `![]()
` tags are updated to point to where the images are saved.
+
+### Include custom post types and pages?
+
+- Argument: `--include-other-types`
+- Type: `boolean`
+- Default: `false`
+
+Some WordPress sites make use of a `"page"` post type and/or custom post types. Set this to `true` to include these post types in the results. Posts will be organized into post type folders.
## Advanced Settings
diff --git a/package-lock.json b/package-lock.json
index f867195..3a92806 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -82,24 +82,11 @@
}
}
},
- "abab": {
- "version": "2.0.5",
- "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.5.tgz",
- "integrity": "sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q=="
- },
"acorn": {
"version": "7.4.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
- "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="
- },
- "acorn-globals": {
- "version": "6.0.0",
- "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz",
- "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==",
- "requires": {
- "acorn": "^7.1.1",
- "acorn-walk": "^7.1.1"
- }
+ "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==",
+ "dev": true
},
"acorn-jsx": {
"version": "5.3.1",
@@ -107,11 +94,6 @@
"integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==",
"dev": true
},
- "acorn-walk": {
- "version": "7.2.0",
- "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz",
- "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA=="
- },
"ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@@ -211,11 +193,6 @@
"concat-map": "0.0.1"
}
},
- "browser-process-hrtime": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
- "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow=="
- },
"callsites": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -317,26 +294,6 @@
}
}
},
- "cssom": {
- "version": "0.4.4",
- "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz",
- "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw=="
- },
- "cssstyle": {
- "version": "2.3.0",
- "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz",
- "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==",
- "requires": {
- "cssom": "~0.3.6"
- },
- "dependencies": {
- "cssom": {
- "version": "0.3.8",
- "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz",
- "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg=="
- }
- }
- },
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -345,16 +302,6 @@
"assert-plus": "^1.0.0"
}
},
- "data-urls": {
- "version": "2.0.0",
- "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz",
- "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==",
- "requires": {
- "abab": "^2.0.3",
- "whatwg-mimetype": "^2.3.0",
- "whatwg-url": "^8.0.0"
- }
- },
"debug": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
@@ -364,15 +311,11 @@
"ms": "2.1.2"
}
},
- "decimal.js": {
- "version": "10.2.1",
- "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz",
- "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw=="
- },
"deep-is": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
- "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ="
+ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+ "dev": true
},
"delayed-stream": {
"version": "1.0.0",
@@ -388,20 +331,10 @@
"esutils": "^2.0.2"
}
},
- "domexception": {
- "version": "2.0.1",
- "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz",
- "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==",
- "requires": {
- "webidl-conversions": "^5.0.0"
- },
- "dependencies": {
- "webidl-conversions": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz",
- "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA=="
- }
- }
+ "domino": {
+ "version": "2.1.6",
+ "resolved": "https://registry.npmjs.org/domino/-/domino-2.1.6.tgz",
+ "integrity": "sha512-3VdM/SXBZX2omc9JF9nOPCtDaYQ67BGp5CoLpIQlO2KCAPETs8TcDHacF26jXadGbvUteZzRTeos2fhID5+ucQ=="
},
"ecc-jsbn": {
"version": "0.1.2",
@@ -422,18 +355,6 @@
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
},
- "escodegen": {
- "version": "1.14.3",
- "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz",
- "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==",
- "requires": {
- "esprima": "^4.0.1",
- "estraverse": "^4.2.0",
- "esutils": "^2.0.2",
- "optionator": "^0.8.1",
- "source-map": "~0.6.1"
- }
- },
"eslint": {
"version": "6.8.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-6.8.0.tgz",
@@ -585,7 +506,8 @@
"esprima": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
- "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="
+ "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+ "dev": true
},
"esquery": {
"version": "1.3.1",
@@ -624,12 +546,14 @@
"estraverse": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
- "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw=="
+ "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+ "dev": true
},
"esutils": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
- "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g=="
+ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
+ "dev": true
},
"extend": {
"version": "3.0.2",
@@ -664,7 +588,8 @@
"fast-levenshtein": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
- "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc="
+ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
+ "dev": true
},
"figures": {
"version": "3.2.0",
@@ -794,14 +719,6 @@
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
},
- "html-encoding-sniffer": {
- "version": "2.0.1",
- "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz",
- "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==",
- "requires": {
- "whatwg-encoding": "^1.0.5"
- }
- },
"http-signature": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -878,11 +795,6 @@
"through": "^2.3.6"
}
},
- "ip-regex": {
- "version": "2.1.0",
- "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
- "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk="
- },
"is-extglob": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
@@ -903,11 +815,6 @@
"is-extglob": "^2.1.1"
}
},
- "is-potential-custom-element-name": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.0.tgz",
- "integrity": "sha1-DFLlS8yjkbssSUsh6GJtczbG45c="
- },
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@@ -945,51 +852,6 @@
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
},
- "jsdom": {
- "version": "16.4.0",
- "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.4.0.tgz",
- "integrity": "sha512-lYMm3wYdgPhrl7pDcRmvzPhhrGVBeVhPIqeHjzeiHN3DFmD1RBpbExbi8vU7BJdH8VAZYovR8DMt0PNNDM7k8w==",
- "requires": {
- "abab": "^2.0.3",
- "acorn": "^7.1.1",
- "acorn-globals": "^6.0.0",
- "cssom": "^0.4.4",
- "cssstyle": "^2.2.0",
- "data-urls": "^2.0.0",
- "decimal.js": "^10.2.0",
- "domexception": "^2.0.1",
- "escodegen": "^1.14.1",
- "html-encoding-sniffer": "^2.0.1",
- "is-potential-custom-element-name": "^1.0.0",
- "nwsapi": "^2.2.0",
- "parse5": "5.1.1",
- "request": "^2.88.2",
- "request-promise-native": "^1.0.8",
- "saxes": "^5.0.0",
- "symbol-tree": "^3.2.4",
- "tough-cookie": "^3.0.1",
- "w3c-hr-time": "^1.0.2",
- "w3c-xmlserializer": "^2.0.0",
- "webidl-conversions": "^6.1.0",
- "whatwg-encoding": "^1.0.5",
- "whatwg-mimetype": "^2.3.0",
- "whatwg-url": "^8.0.0",
- "ws": "^7.2.3",
- "xml-name-validator": "^3.0.0"
- },
- "dependencies": {
- "tough-cookie": {
- "version": "3.0.1",
- "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz",
- "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==",
- "requires": {
- "ip-regex": "^2.1.0",
- "psl": "^1.1.28",
- "punycode": "^2.1.1"
- }
- }
- }
- },
"json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
@@ -1026,6 +888,7 @@
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
"integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
+ "dev": true,
"requires": {
"prelude-ls": "~1.1.2",
"type-check": "~0.3.2"
@@ -1036,11 +899,6 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
- "lodash.sortby": {
- "version": "4.7.0",
- "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
- "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg="
- },
"luxon": {
"version": "1.25.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.25.0.tgz",
@@ -1111,11 +969,6 @@
"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
"dev": true
},
- "nwsapi": {
- "version": "2.2.0",
- "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz",
- "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ=="
- },
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -1142,6 +995,7 @@
"version": "0.8.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
"integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==",
+ "dev": true,
"requires": {
"deep-is": "~0.1.3",
"fast-levenshtein": "~2.0.6",
@@ -1165,11 +1019,6 @@
"callsites": "^3.0.0"
}
},
- "parse5": {
- "version": "5.1.1",
- "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz",
- "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug=="
- },
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
@@ -1190,7 +1039,8 @@
"prelude-ls": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
- "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
+ "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=",
+ "dev": true
},
"progress": {
"version": "2.0.3",
@@ -1316,14 +1166,6 @@
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
- "saxes": {
- "version": "5.0.1",
- "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz",
- "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==",
- "requires": {
- "xmlchars": "^2.2.0"
- }
- },
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@@ -1393,12 +1235,6 @@
}
}
},
- "source-map": {
- "version": "0.6.1",
- "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
- "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
- "optional": true
- },
"sprintf-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@@ -1458,11 +1294,6 @@
"has-flag": "^4.0.0"
}
},
- "symbol-tree": {
- "version": "3.2.4",
- "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
- "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
- },
"table": {
"version": "5.4.6",
"resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz",
@@ -1543,14 +1374,6 @@
"punycode": "^2.1.1"
}
},
- "tr46": {
- "version": "2.0.2",
- "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz",
- "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==",
- "requires": {
- "punycode": "^2.1.1"
- }
- },
"tslib": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
@@ -1565,11 +1388,11 @@
}
},
"turndown": {
- "version": "6.0.0",
- "resolved": "https://registry.npmjs.org/turndown/-/turndown-6.0.0.tgz",
- "integrity": "sha512-UVJBhSyRHCpNKtQ00mNWlYUM/i+tcipkb++F0PrOpt0L7EhNd0AX9mWEpL2dRFBu7LWXMp4HgAMA4OeKKnN7og==",
+ "version": "7.0.0",
+ "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.0.0.tgz",
+ "integrity": "sha512-G1FfxfR0mUNMeGjszLYl3kxtopC4O9DRRiMlMDDVHvU1jaBkGFg4qxIyjIk2aiKLHyDyZvZyu4qBO2guuYBy3Q==",
"requires": {
- "jsdom": "^16.2.0"
+ "domino": "^2.1.6"
}
},
"turndown-plugin-gfm": {
@@ -1586,6 +1409,7 @@
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
"integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
+ "dev": true,
"requires": {
"prelude-ls": "~1.1.2"
}
@@ -1624,50 +1448,6 @@
"extsprintf": "^1.2.0"
}
},
- "w3c-hr-time": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",
- "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==",
- "requires": {
- "browser-process-hrtime": "^1.0.0"
- }
- },
- "w3c-xmlserializer": {
- "version": "2.0.0",
- "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz",
- "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==",
- "requires": {
- "xml-name-validator": "^3.0.0"
- }
- },
- "webidl-conversions": {
- "version": "6.1.0",
- "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.1.0.tgz",
- "integrity": "sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w=="
- },
- "whatwg-encoding": {
- "version": "1.0.5",
- "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz",
- "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==",
- "requires": {
- "iconv-lite": "0.4.24"
- }
- },
- "whatwg-mimetype": {
- "version": "2.3.0",
- "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz",
- "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g=="
- },
- "whatwg-url": {
- "version": "8.4.0",
- "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.4.0.tgz",
- "integrity": "sha512-vwTUFf6V4zhcPkWp/4CQPr1TW9Ml6SF4lVyaIMBdJw5i6qUUJ1QWM4Z6YYVkfka0OUIzVo/0aNtGVGk256IKWw==",
- "requires": {
- "lodash.sortby": "^4.7.0",
- "tr46": "^2.0.2",
- "webidl-conversions": "^6.1.0"
- }
- },
"which": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
@@ -1680,7 +1460,8 @@
"word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
- "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ=="
+ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
+ "dev": true
},
"wrappy": {
"version": "1.0.2",
@@ -1697,16 +1478,6 @@
"mkdirp": "^0.5.1"
}
},
- "ws": {
- "version": "7.4.1",
- "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.1.tgz",
- "integrity": "sha512-pTsP8UAfhy3sk1lSk/O/s4tjD0CRwvMnzvwr4OKGX7ZvqZtUyx4KIJB5JWbkykPoc55tixMGgTNoh3k4FkNGFQ=="
- },
- "xml-name-validator": {
- "version": "3.0.0",
- "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz",
- "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw=="
- },
"xml2js": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
@@ -1720,11 +1491,6 @@
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
- },
- "xmlchars": {
- "version": "2.2.0",
- "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
- "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
}
}
}
diff --git a/package.json b/package.json
index e229f3a..6fc58e6 100644
--- a/package.json
+++ b/package.json
@@ -27,7 +27,7 @@
"luxon": "^1.23.0",
"request": "^2.88.2",
"request-promise-native": "^1.0.8",
- "turndown": "^6.0.0",
+ "turndown": "^7.0.0",
"turndown-plugin-gfm": "^1.0.2",
"xml2js": "^0.4.23"
},
diff --git a/src/parser.js b/src/parser.js
index 47b44e7..cf3e5d4 100644
--- a/src/parser.js
+++ b/src/parser.js
@@ -29,6 +29,20 @@ async function parseFilePromise(config) {
return posts;
}
+function getItemTypes(data, config) {
+ if (config.includeOtherTypes) {
+ // search export file for all post types minus some default types we don't want
+ // effectively this will be 'post', 'page', and custom post types
+ const types = data.rss.channel[0].item
+ .map(item => item.post_type[0])
+ .filter(type => !['attachment', 'revision', 'nav_menu_item', 'custom_css', 'customize_changeset'].includes(type));
+ return [...new Set(types)]; // remove duplicates
+ } else {
+ // just plain old vanilla "post" posts
+ return ['post'];
+ }
+}
+
function getItemsOfType(data, type) {
return data.rss.channel[0].item.filter(item => item.post_type[0] === type);
}
@@ -37,27 +51,40 @@ function collectPosts(data, config) {
// this is passed into getPostContent() for the markdown conversion
const turndownService = translator.initTurndownService();
- const posts = getItemsOfType(data, 'post')
- .filter(post => post.status[0] !== 'trash' && post.status[0] !== 'draft')
- .map(post => ({
- // meta data isn't written to file, but is used to help with other things
- meta: {
- id: getPostId(post),
- slug: getPostSlug(post),
- coverImageId: getPostCoverImageId(post),
- imageUrls: []
- },
- frontmatter: {
- title: getPostTitle(post),
- date: getPostDate(post),
- categories: getCategories(post),
- tags: getTags(post)
- },
- content: translator.getPostContent(post, turndownService, config)
- }));
+ const types = getItemTypes(data, config);
+ let allPosts = [];
+ types.forEach(postType => {
+ const postsForType = getItemsOfType(data, postType)
+ .filter(post => post.status[0] !== 'trash' && post.status[0] !== 'draft')
+ .map(post => ({
+ // meta data isn't written to file, but is used to help with other things
+ meta: {
+ id: getPostId(post),
+ slug: getPostSlug(post),
+ coverImageId: getPostCoverImageId(post),
+ type: postType,
+ imageUrls: []
+ },
+ frontmatter: {
+ title: getPostTitle(post),
+ date: getPostDate(post),
+ categories: getCategories(post),
+ tags: getTags(post)
+ },
+ content: translator.getPostContent(post, turndownService, config)
+ }));
- console.log(posts.length + ' posts found.');
- return posts;
+ if (types.length > 1) {
+ console.log(`${postsForType.length} "${postType}" posts found.`);
+ }
+
+ allPosts.push(...postsForType);
+ });
+
+ if (types.length === 1) {
+ console.log(allPosts.length + ' posts found.');
+ }
+ return allPosts;
}
function getPostId(post) {
@@ -65,7 +92,7 @@ function getPostId(post) {
}
function getPostSlug(post) {
- return decodeURI(post.post_name[0]);
+ return decodeURIComponent(post.post_name[0]);
}
function getPostCoverImageId(post) {
@@ -110,7 +137,7 @@ function processCategoryTags(post, domain) {
return post.category
.filter(category => category.$.domain === domain)
- .map(({ $: attributes }) => decodeURI(attributes.nicename));
+ .map(({ $: attributes }) => decodeURIComponent(attributes.nicename));
}
function collectAttachedImages(data) {
@@ -141,7 +168,7 @@ function collectScrapedImages(data) {
images.push({
id: -1,
postId: postId,
- url: decodeURI(url)
+ url
});
});
});
diff --git a/src/shared.js b/src/shared.js
index b3e6165..5175848 100644
--- a/src/shared.js
+++ b/src/shared.js
@@ -1,5 +1,5 @@
function getFilenameFromUrl(url) {
- return url.split('/').slice(-1)[0];
+ return decodeURIComponent(url.split('/').slice(-1)[0]);
}
exports.getFilenameFromUrl = getFilenameFromUrl;
diff --git a/src/translator.js b/src/translator.js
index 194cd43..41c110b 100644
--- a/src/translator.js
+++ b/src/translator.js
@@ -66,7 +66,7 @@ function getPostContent(post, turndownService, config) {
if (config.saveScrapedImages) {
// writeImageFile() will save all content images to a relative /images
- // folder so update references in post content to match
+ // folder so update references in post content to match
content = content.replace(/(]*src=").*?([^/"]+\.(?:gif|jpe?g|png))("[^>]*>)/gi, '$1images/$2$3');
}
diff --git a/src/wizard.js b/src/wizard.js
index e3e87dc..5285b8d 100644
--- a/src/wizard.js
+++ b/src/wizard.js
@@ -3,11 +3,10 @@ const commander = require('commander');
const fs = require('fs');
const inquirer = require('inquirer');
const path = require('path');
-const process = require('process');
const package = require('../package.json');
-// all user options for command line and wizard are declard here
+// all user options for command line and wizard are declared here
const options = [
// wizard must always be first
{
@@ -69,6 +68,12 @@ const options = [
type: 'boolean',
description: 'Save images scraped from post body content',
default: true
+ },
+ {
+ name: 'include-other-types',
+ type: 'boolean',
+ description: 'Include custom post types and pages',
+ default: false
}
];
diff --git a/src/writer.js b/src/writer.js
index 3f23955..7b04f78 100644
--- a/src/writer.js
+++ b/src/writer.js
@@ -12,11 +12,11 @@ async function writeFilesPromise(posts, config) {
await writeImageFilesPromise(posts, config);
}
-async function processPayloadsPromise(payloads, loadFunc, config) {
+async function processPayloadsPromise(payloads, loadFunc) {
const promises = payloads.map(payload => new Promise((resolve, reject) => {
setTimeout(async () => {
try {
- const data = await loadFunc(payload.item, config);
+ const data = await loadFunc(payload.item);
await writeFile(payload.destinationPath, data);
console.log(chalk.green('[OK]') + ' ' + payload.name);
resolve();
@@ -43,15 +43,33 @@ async function writeFile(destinationPath, data) {
async function writeMarkdownFilesPromise(posts, config ) {
// package up posts into payloads
- const payloads = posts.map((post, index) => ({
- item: post,
- name: post.meta.slug,
- destinationPath: getPostPath(post, config),
- delay: index * settings.markdown_file_write_delay
- }));
+ let skipCount = 0;
+ let delay = 0;
+ const payloads = posts.flatMap(post => {
+ const destinationPath = getPostPath(post, config);
+ if (checkFile(destinationPath)) {
+ // already exists, don't need to save again
+ skipCount++;
+ return [];
+ } else {
+ const payload = {
+ item: post,
+ name: (config.includeOtherTypes ? post.meta.type + ' - ' : '') + post.meta.slug,
+ destinationPath,
+ delay
+ };
+ delay += settings.markdown_file_write_delay;
+ return [payload];
+ }
+ });
- console.log('\nSaving posts...');
- await processPayloadsPromise(payloads, loadMarkdownFilePromise, config);
+ const remainingCount = payloads.length;
+ if (remainingCount + skipCount === 0) {
+ console.log('\nNo posts to save...');
+ } else {
+ console.log(`\nSaving ${remainingCount} posts (${skipCount} already exist)...`);
+ await processPayloadsPromise(payloads, loadMarkdownFilePromise);
+ }
}
async function loadMarkdownFilePromise(post) {
@@ -81,36 +99,48 @@ async function loadMarkdownFilePromise(post) {
async function writeImageFilesPromise(posts, config) {
// collect image data from all posts into a single flattened array of payloads
+ let skipCount = 0;
let delay = 0;
const payloads = posts.flatMap(post => {
const postPath = getPostPath(post, config);
const imagesDir = path.join(path.dirname(postPath), 'images');
- return post.meta.imageUrls.map(imageUrl => {
+ return post.meta.imageUrls.flatMap(imageUrl => {
const filename = shared.getFilenameFromUrl(imageUrl);
- const payload = {
- item: imageUrl,
- name: filename,
- destinationPath: path.join(imagesDir, filename),
- delay
- };
- delay += settings.image_file_request_delay;
- return payload;
+ const destinationPath = path.join(imagesDir, filename);
+ if (checkFile(destinationPath)) {
+ // already exists, don't need to save again
+ skipCount++;
+ return [];
+ } else {
+ const payload = {
+ item: imageUrl,
+ name: filename,
+ destinationPath,
+ delay
+ };
+ delay += settings.image_file_request_delay;
+ return [payload];
+ }
});
});
- if (payloads.length > 0) {
- console.log('\nDownloading and saving images...');
- await processPayloadsPromise(payloads, loadImageFilePromise);
- } else {
+ const remainingCount = payloads.length;
+ if (remainingCount + skipCount === 0) {
console.log('\nNo images to download and save...');
+ } else {
+ console.log(`\nDownloading and saving ${remainingCount} images (${skipCount} already exist)...`);
+ await processPayloadsPromise(payloads, loadImageFilePromise);
}
}
async function loadImageFilePromise(imageUrl) {
+ // only encode the URL if it doesn't already have encoded characters
+ const url = (/%[\da-f]{2}/i).test(imageUrl) ? imageUrl : encodeURI(imageUrl);
+
let buffer;
try {
buffer = await requestPromiseNative.get({
- url: encodeURI(imageUrl),
+ url,
encoding: null, // preserves binary encoding
headers: {
'User-Agent': 'wordpress-export-to-markdown'
@@ -132,6 +162,11 @@ function getPostPath(post, config) {
// start with base output dir
const pathSegments = [config.output];
+ // create segment for post type if we're dealing with more than just "post"
+ if (config.includeOtherTypes) {
+ pathSegments.push(post.meta.type);
+ }
+
if (config.yearFolders) {
pathSegments.push(dt.toFormat('yyyy'));
}
@@ -156,4 +191,8 @@ function getPostPath(post, config) {
return path.join(...pathSegments);
}
+function checkFile(path) {
+ return fs.existsSync(path);
+}
+
exports.writeFilesPromise = writeFilesPromise;