Initial commit

diff --git a/.gitignore b/.gitignore
new file mode 100644 (file)
index 0000000..b9832f9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,66 @@
+# Linux
+# backup files
+*~
+
+# Windows
+# thumbnails
+Thumbs.db
+
+# Mac OS X
+# metadata
+.DS_Store
+# thumbnails
+._*
+
+# Visual Studio PHP
+*.sln
+*.phpproj
+*.puo
+*.suo
+*.cache
+
+# Netbeans
+nbproject/
+catalog.xml
+nbactions.xml
+
+# Eclipse
+.settings/
+.buildpath
+.classpath
+.project
+
+# SVN
+# svn folders
+.svn/
+
+# PHPStorm
+.idea/
+.nameencodings
+.xmlmisc
+.xmlmodules
+.xmlprojectCodeStyle
+.xmlvcs.xml
+*.imlworkspace
+.xml
+
+# Sublime Text 2
+*.sublime-*
+
+# Textmate
+*.tmproj
+
+# Visual Studio Code
+.vscode/
+
+# WoltLab Suite
+# Ignore packages build directly in the workspace. They can however be added manually via git add, if wanted.
+*.tar
+*.tar.gz
+
+# Node.js
+dist/*
+node_modules
+
+# Used for internal testing of the generated files.
+public/*

diff --git a/package-lock.json b/package-lock.json
new file mode 100644 (file)
index 0000000..a162893
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,1111 @@
+{
+  "name": "blacklist",
+  "version": "5.2.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "@types/adm-zip": {
+      "version": "0.4.32",
+      "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.4.32.tgz",
+      "integrity": "sha512-hv1O7ySn+XvP5OeDQcJFWwVb2v+GFGO1A9aMTQ5B/bzxb7WW21O8iRhVdsKKr8QwuiagzGmPP+gsUAYZ6bRddQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "@types/csv-parse": {
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/@types/csv-parse/-/csv-parse-1.1.12.tgz",
+      "integrity": "sha512-p6uZznjJOcFaymduLYf45ik28IYzChnkt+ofJOWa16bb2JRCHdxs/ai03q6raizCc5JuunVsbgtlDxfu9y2Nag==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "@types/minimist": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.0.tgz",
+      "integrity": "sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY=",
+      "dev": true
+    },
+    "@types/node": {
+      "version": "11.9.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-11.9.5.tgz",
+      "integrity": "sha512-vVjM0SVzgaOUpflq4GYBvCpozes8OgIIS5gVXVka+OfK3hvnkC1i93U8WiY2OtNE4XUWyyy/86Kf6e0IHTQw1Q==",
+      "dev": true
+    },
+    "@types/node-fetch": {
+      "version": "2.1.6",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.1.6.tgz",
+      "integrity": "sha512-Hv1jgh3pfpUEl2F2mqUd1AfLSk1YbUCeBJFaP36t7esAO617dErqdxWb5cdG2NfJGOofkmBW36fdx0dVewxDRg==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "@types/sqlite3": {
+      "version": "3.1.4",
+      "resolved": "https://registry.npmjs.org/@types/sqlite3/-/sqlite3-3.1.4.tgz",
+      "integrity": "sha512-EZ/fWzLSKclqFyvMQEw2Ii/NYRTbIWzkdbm56Lgnq/GKNX0o3rkAvNvggXgWkUzgC28nC7yYVgfhWUHvUPd/Fg==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
+    "abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q=="
+    },
+    "adm-zip": {
+      "version": "0.4.13",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.13.tgz",
+      "integrity": "sha512-fERNJX8sOXfel6qCBCMPvZLzENBEhZTzKqg6vrOW5pvoEaQuJhRU4ndTAh6lHOxn1I6jnz2NHra56ZODM751uw=="
+    },
+    "ajv": {
+      "version": "6.9.2",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz",
+      "integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==",
+      "requires": {
+        "fast-deep-equal": "^2.0.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      }
+    },
+    "ansi-regex": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
+    },
+    "ansi-styles": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+      "dev": true
+    },
+    "aproba": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
+      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
+    },
+    "are-we-there-yet": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
+      "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
+      "requires": {
+        "delegates": "^1.0.0",
+        "readable-stream": "^2.0.6"
+      }
+    },
+    "argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "dev": true,
+      "requires": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
+    "asn1": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
+      "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
+      "requires": {
+        "safer-buffer": "~2.1.0"
+      }
+    },
+    "assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+    },
+    "asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
+    },
+    "aws-sign2": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
+      "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
+    },
+    "aws4": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz",
+      "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ=="
+    },
+    "babel-code-frame": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz",
+      "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=",
+      "dev": true,
+      "requires": {
+        "chalk": "^1.1.3",
+        "esutils": "^2.0.2",
+        "js-tokens": "^3.0.2"
+      },
+      "dependencies": {
+        "chalk": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^2.2.1",
+            "escape-string-regexp": "^1.0.2",
+            "has-ansi": "^2.0.0",
+            "strip-ansi": "^3.0.0",
+            "supports-color": "^2.0.0"
+          }
+        }
+      }
+    },
+    "balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
+    },
+    "bcrypt-pbkdf": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
+      "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
+      "requires": {
+        "tweetnacl": "^0.14.3"
+      }
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "requires": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "builtin-modules": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
+      "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=",
+      "dev": true
+    },
+    "caseless": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
+      "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
+    },
+    "chalk": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
+      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+      "dev": true,
+      "requires": {
+        "ansi-styles": "^3.2.1",
+        "escape-string-regexp": "^1.0.5",
+        "supports-color": "^5.3.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "3.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
+          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^1.9.0"
+          }
+        },
+        "supports-color": {
+          "version": "5.5.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+          "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "chownr": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
+      "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g=="
+    },
+    "code-point-at": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
+      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
+    },
+    "color-convert": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
+      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
+      "dev": true,
+      "requires": {
+        "color-name": "1.1.3"
+      }
+    },
+    "color-name": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
+      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
+      "dev": true
+    },
+    "combined-stream": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
+      "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
+      "requires": {
+        "delayed-stream": "~1.0.0"
+      }
+    },
+    "commander": {
+      "version": "2.19.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz",
+      "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==",
+      "dev": true
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+    },
+    "console-control-strings": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+    },
+    "csv-parse": {
+      "version": "4.3.3",
+      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.3.3.tgz",
+      "integrity": "sha512-bZ+AZjm2LlWEp5+TKeFeXDldduCUUaxEif+KUv+zvAwmCvCKTqeSHVEyxztGCQ6OE+87ObRq4NsCmg91SuJbhQ=="
+    },
+    "dashdash": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+      "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
+      "requires": {
+        "assert-plus": "^1.0.0"
+      }
+    },
+    "debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+      "requires": {
+        "ms": "2.0.0"
+      }
+    },
+    "deep-extend": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
+      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
+    },
+    "delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
+    },
+    "delegates": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
+    },
+    "detect-libc": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
+      "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
+    },
+    "diff": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
+      "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
+      "dev": true
+    },
+    "ecc-jsbn": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
+      "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
+      "requires": {
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.1.0"
+      }
+    },
+    "escape-string-regexp": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
+      "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
+      "dev": true
+    },
+    "esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+      "dev": true
+    },
+    "esutils": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz",
+      "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=",
+      "dev": true
+    },
+    "extend": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
+    },
+    "extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
+    },
+    "fast-deep-equal": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
+      "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk="
+    },
+    "fast-json-stable-stringify": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
+      "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
+    },
+    "forever-agent": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
+      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
+    },
+    "form-data": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+      "requires": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.6",
+        "mime-types": "^2.1.12"
+      }
+    },
+    "fs-minipass": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz",
+      "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==",
+      "requires": {
+        "minipass": "^2.2.1"
+      }
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
+    },
+    "gauge": {
+      "version": "2.7.4",
+      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
+      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
+      "requires": {
+        "aproba": "^1.0.3",
+        "console-control-strings": "^1.0.0",
+        "has-unicode": "^2.0.0",
+        "object-assign": "^4.1.0",
+        "signal-exit": "^3.0.0",
+        "string-width": "^1.0.1",
+        "strip-ansi": "^3.0.1",
+        "wide-align": "^1.1.0"
+      }
+    },
+    "getpass": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+      "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
+      "requires": {
+        "assert-plus": "^1.0.0"
+      }
+    },
+    "glob": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
+      "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
+      "requires": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.0.4",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
+    "har-schema": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
+      "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
+    },
+    "har-validator": {
+      "version": "5.1.3",
+      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz",
+      "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==",
+      "requires": {
+        "ajv": "^6.5.5",
+        "har-schema": "^2.0.0"
+      }
+    },
+    "has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "dev": true,
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
+    "has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+      "dev": true
+    },
+    "has-unicode": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
+    },
+    "http-signature": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
+      "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
+      "requires": {
+        "assert-plus": "^1.0.0",
+        "jsprim": "^1.2.2",
+        "sshpk": "^1.7.0"
+      }
+    },
+    "iconv-lite": {
+      "version": "0.4.24",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "requires": {
+        "safer-buffer": ">= 2.1.2 < 3"
+      }
+    },
+    "ignore-walk": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz",
+      "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==",
+      "requires": {
+        "minimatch": "^3.0.4"
+      }
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "requires": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
+    },
+    "ini": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
+    },
+    "is-fullwidth-code-point": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
+      "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+      "requires": {
+        "number-is-nan": "^1.0.0"
+      }
+    },
+    "is-typedarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
+      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
+    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+    },
+    "isstream": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
+    },
+    "js-tokens": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",
+      "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=",
+      "dev": true
+    },
+    "js-yaml": {
+      "version": "3.12.2",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.2.tgz",
+      "integrity": "sha512-QHn/Lh/7HhZ/Twc7vJYQTkjuCa0kaCcDcjK5Zlk2rvnUpy7DxMJ23+Jc2dcyvltwQVg1nygAVlB2oRDFHoRS5Q==",
+      "dev": true,
+      "requires": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      }
+    },
+    "jsbn": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+      "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
+    },
+    "json-schema": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
+      "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
+    },
+    "json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+    },
+    "json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
+    },
+    "jsprim": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
+      "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
+      "requires": {
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.2.3",
+        "verror": "1.10.0"
+      }
+    },
+    "mime-db": {
+      "version": "1.38.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz",
+      "integrity": "sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg=="
+    },
+    "mime-types": {
+      "version": "2.1.22",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz",
+      "integrity": "sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog==",
+      "requires": {
+        "mime-db": "~1.38.0"
+      }
+    },
+    "minimatch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "requires": {
+        "brace-expansion": "^1.1.7"
+      }
+    },
+    "minimist": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+      "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
+    },
+    "minipass": {
+      "version": "2.3.5",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz",
+      "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==",
+      "requires": {
+        "safe-buffer": "^5.1.2",
+        "yallist": "^3.0.0"
+      }
+    },
+    "minizlib": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz",
+      "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==",
+      "requires": {
+        "minipass": "^2.2.1"
+      }
+    },
+    "mkdirp": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "requires": {
+        "minimist": "0.0.8"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "0.0.8",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
+          "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+        }
+      }
+    },
+    "ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
+    },
+    "nan": {
+      "version": "2.10.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
+      "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA=="
+    },
+    "needle": {
+      "version": "2.2.4",
+      "resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz",
+      "integrity": "sha512-HyoqEb4wr/rsoaIDfTH2aVL9nWtQqba2/HvMv+++m8u0dz808MaagKILxtfeSN7QU7nvbQ79zk3vYOJp9zsNEA==",
+      "requires": {
+        "debug": "^2.1.2",
+        "iconv-lite": "^0.4.4",
+        "sax": "^1.2.4"
+      }
+    },
+    "node-fetch": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz",
+      "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA=="
+    },
+    "node-pre-gyp": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
+      "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
+      "requires": {
+        "detect-libc": "^1.0.2",
+        "mkdirp": "^0.5.1",
+        "needle": "^2.2.1",
+        "nopt": "^4.0.1",
+        "npm-packlist": "^1.1.6",
+        "npmlog": "^4.0.2",
+        "rc": "^1.2.7",
+        "rimraf": "^2.6.1",
+        "semver": "^5.3.0",
+        "tar": "^4"
+      }
+    },
+    "nopt": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz",
+      "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=",
+      "requires": {
+        "abbrev": "1",
+        "osenv": "^0.1.4"
+      }
+    },
+    "npm-bundled": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz",
+      "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g=="
+    },
+    "npm-packlist": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz",
+      "integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==",
+      "requires": {
+        "ignore-walk": "^3.0.1",
+        "npm-bundled": "^1.0.1"
+      }
+    },
+    "npmlog": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
+      "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
+      "requires": {
+        "are-we-there-yet": "~1.1.2",
+        "console-control-strings": "~1.1.0",
+        "gauge": "~2.7.3",
+        "set-blocking": "~2.0.0"
+      }
+    },
+    "number-is-nan": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
+      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
+    },
+    "oauth-sign": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
+      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
+    },
+    "object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "os-homedir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
+      "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
+    },
+    "os-tmpdir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
+      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
+    },
+    "osenv": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
+      "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
+      "requires": {
+        "os-homedir": "^1.0.0",
+        "os-tmpdir": "^1.0.0"
+      }
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
+    },
+    "path-parse": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz",
+      "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
+      "dev": true
+    },
+    "performance-now": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
+      "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
+    },
+    "process-nextick-args": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
+      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+    },
+    "psl": {
+      "version": "1.1.31",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz",
+      "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw=="
+    },
+    "punycode": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
+    },
+    "qs": {
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
+      "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
+    },
+    "rc": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "requires": {
+        "deep-extend": "^0.6.0",
+        "ini": "~1.3.0",
+        "minimist": "^1.2.0",
+        "strip-json-comments": "~2.0.1"
+      }
+    },
+    "readable-stream": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
+      "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
+      "requires": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "reflect-metadata": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
+      "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg=="
+    },
+    "request": {
+      "version": "2.88.0",
+      "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
+      "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==",
+      "requires": {
+        "aws-sign2": "~0.7.0",
+        "aws4": "^1.8.0",
+        "caseless": "~0.12.0",
+        "combined-stream": "~1.0.6",
+        "extend": "~3.0.2",
+        "forever-agent": "~0.6.1",
+        "form-data": "~2.3.2",
+        "har-validator": "~5.1.0",
+        "http-signature": "~1.2.0",
+        "is-typedarray": "~1.0.0",
+        "isstream": "~0.1.2",
+        "json-stringify-safe": "~5.0.1",
+        "mime-types": "~2.1.19",
+        "oauth-sign": "~0.9.0",
+        "performance-now": "^2.1.0",
+        "qs": "~6.5.2",
+        "safe-buffer": "^5.1.2",
+        "tough-cookie": "~2.4.3",
+        "tunnel-agent": "^0.6.0",
+        "uuid": "^3.3.2"
+      }
+    },
+    "resolve": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz",
+      "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==",
+      "dev": true,
+      "requires": {
+        "path-parse": "^1.0.6"
+      }
+    },
+    "rimraf": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
+      "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
+      "requires": {
+        "glob": "^7.1.3"
+      }
+    },
+    "safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
+    "safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+    },
+    "sax": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+    },
+    "semver": {
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
+      "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="
+    },
+    "set-blocking": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
+    },
+    "signal-exit": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
+      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
+    },
+    "sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+      "dev": true
+    },
+    "sqlite3": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.0.6.tgz",
+      "integrity": "sha512-EqBXxHdKiwvNMRCgml86VTL5TK1i0IKiumnfxykX0gh6H6jaKijAXvE9O1N7+omfNSawR2fOmIyJZcfe8HYWpw==",
+      "requires": {
+        "nan": "~2.10.0",
+        "node-pre-gyp": "^0.11.0",
+        "request": "^2.87.0"
+      }
+    },
+    "sshpk": {
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
+      "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
+      "requires": {
+        "asn1": "~0.2.3",
+        "assert-plus": "^1.0.0",
+        "bcrypt-pbkdf": "^1.0.0",
+        "dashdash": "^1.12.0",
+        "ecc-jsbn": "~0.1.1",
+        "getpass": "^0.1.1",
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.0.2",
+        "tweetnacl": "~0.14.0"
+      }
+    },
+    "string-width": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
+      "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
+      "requires": {
+        "code-point-at": "^1.0.0",
+        "is-fullwidth-code-point": "^1.0.0",
+        "strip-ansi": "^3.0.0"
+      }
+    },
+    "string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "requires": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
+    "strip-ansi": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
+    "strip-json-comments": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
+      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
+    },
+    "supports-color": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+      "dev": true
+    },
+    "tar": {
+      "version": "4.4.8",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz",
+      "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==",
+      "requires": {
+        "chownr": "^1.1.1",
+        "fs-minipass": "^1.2.5",
+        "minipass": "^2.3.4",
+        "minizlib": "^1.1.1",
+        "mkdirp": "^0.5.0",
+        "safe-buffer": "^5.1.2",
+        "yallist": "^3.0.2"
+      }
+    },
+    "tough-cookie": {
+      "version": "2.4.3",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz",
+      "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==",
+      "requires": {
+        "psl": "^1.1.24",
+        "punycode": "^1.4.1"
+      },
+      "dependencies": {
+        "punycode": {
+          "version": "1.4.1",
+          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
+          "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
+        }
+      }
+    },
+    "tslib": {
+      "version": "1.9.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
+      "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==",
+      "dev": true
+    },
+    "tslint": {
+      "version": "5.13.0",
+      "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.13.0.tgz",
+      "integrity": "sha512-ECOOQRxXCYnUUePG5h/+Z1Zouobk3KFpIHA9aKBB/nnMxs97S1JJPDGt5J4cGm1y9U9VmVlfboOxA8n1kSNzGw==",
+      "dev": true,
+      "requires": {
+        "babel-code-frame": "^6.22.0",
+        "builtin-modules": "^1.1.1",
+        "chalk": "^2.3.0",
+        "commander": "^2.12.1",
+        "diff": "^3.2.0",
+        "glob": "^7.1.1",
+        "js-yaml": "^3.7.0",
+        "minimatch": "^3.0.4",
+        "mkdirp": "^0.5.1",
+        "resolve": "^1.3.2",
+        "semver": "^5.3.0",
+        "tslib": "^1.8.0",
+        "tsutils": "^2.27.2"
+      }
+    },
+    "tsutils": {
+      "version": "2.29.0",
+      "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz",
+      "integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==",
+      "dev": true,
+      "requires": {
+        "tslib": "^1.8.1"
+      }
+    },
+    "tunnel-agent": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+      "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
+      "requires": {
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "tweetnacl": {
+      "version": "0.14.5",
+      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
+      "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
+    },
+    "typedi": {
+      "version": "0.8.0",
+      "resolved": "https://registry.npmjs.org/typedi/-/typedi-0.8.0.tgz",
+      "integrity": "sha512-/c7Bxnm6eh5kXx2I+mTuO+2OvoWni5+rXA3PhXwVWCtJRYmz3hMok5s1AKLzoDvNAZqj/Q/acGstN0ri5aQoOA=="
+    },
+    "typescript": {
+      "version": "3.3.3333",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.3.3333.tgz",
+      "integrity": "sha512-JjSKsAfuHBE/fB2oZ8NxtRTk5iGcg6hkYXMnZ3Wc+b2RSqejEqTaem11mHASMnFilHrax3sLK0GDzcJrekZYLw==",
+      "dev": true
+    },
+    "uri-js": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
+      "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
+      "requires": {
+        "punycode": "^2.1.0"
+      }
+    },
+    "util-deprecate": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+    },
+    "uuid": {
+      "version": "3.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
+      "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
+    },
+    "verror": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
+      "requires": {
+        "assert-plus": "^1.0.0",
+        "core-util-is": "1.0.2",
+        "extsprintf": "^1.2.0"
+      }
+    },
+    "wide-align": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
+      "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
+      "requires": {
+        "string-width": "^1.0.2 || 2"
+      }
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+    },
+    "yallist": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
+      "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A=="
+    }
+  }
+}

diff --git a/package.json b/package.json
new file mode 100644 (file)
index 0000000..f899745
--- /dev/null
+++ b/package.json
@@ -0,0 +1,31 @@
+{
+  "name": "blacklist",
+  "version": "5.2.0",
+  "description": "Reads and processes the data from stopforumspam.com, optimizing it for efficient delta upgrades.",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc -w",
+    "lint": "tslint -c tslint.json -p tsconfig.json",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Alexander Ebert",
+  "license": "LGPL-2.1-or-later",
+  "dependencies": {
+    "adm-zip": "^0.4.13",
+    "csv-parse": "^4.3.3",
+    "minimist": "^1.2.0",
+    "node-fetch": "^2.3.0",
+    "reflect-metadata": "^0.1.13",
+    "sqlite3": "^4.0.6",
+    "typedi": "^0.8.0"
+  },
+  "devDependencies": {
+    "@types/adm-zip": "^0.4.32",
+    "@types/csv-parse": "^1.1.12",
+    "@types/minimist": "^1.2.0",
+    "@types/node-fetch": "^2.1.6",
+    "@types/sqlite3": "^3.1.4",
+    "tslint": "^5.13.0",
+    "typescript": "^3.3.3333"
+  }
+}

diff --git a/src/archive.ts b/src/archive.ts
new file mode 100644 (file)
index 0000000..ccce32b
--- /dev/null
+++ b/src/archive.ts
@@ -0,0 +1,50 @@
+import * as AdmZip from 'adm-zip';
+import fetch from 'node-fetch';
+
+export const sources = new Map<string, string>([
+  ['ipv4', 'https://www.stopforumspam.com/downloads/listed_ip_1_all.zip'],
+  ['ipv6', 'https://www.stopforumspam.com/downloads/listed_ip_1_ipv6_all.zip'],
+  ['email', 'https://www.stopforumspam.com/downloads/listed_email_1_all.zip'],
+  [
+    'username',
+    'https://www.stopforumspam.com/downloads/listed_username_1_all.zip',
+  ],
+]);
+
+export class Archive {
+  public get filename(): string {
+    return `tmp.${this.type}.csv`;
+  }
+
+  public static getAll(): Archive[] {
+    const archives: Archive[] = [];
+    sources.forEach(
+      (url: string, type: string): void => {
+        archives.push(new Archive(type));
+      },
+    );
+
+    return archives;
+  }
+  constructor(public readonly type: string) {
+    if (!sources.has(this.type)) {
+      throw new Error(`The type '${this.type}' is not known.`);
+    }
+  }
+
+  public async download(): Promise<Buffer | undefined> {
+    const response = await fetch(sources.get(this.type));
+    if (response.ok) {
+      const entries: AdmZip.IZipEntry[] = new AdmZip(
+        await response.buffer(),
+      ).getEntries();
+      for (let i = 0, length = entries.length; i < length; i++) {
+        if (/^listed_.+_all\.txt$/.exec(entries[i].name)) {
+          return entries[i].getData();
+        }
+      }
+    }
+
+    return undefined;
+  }
+}

diff --git a/src/blacklist-index.ts b/src/blacklist-index.ts
new file mode 100644 (file)
index 0000000..5c233da
--- /dev/null
+++ b/src/blacklist-index.ts
@@ -0,0 +1,121 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { promisify } from 'util';
+
+const fsReaddir = promisify(fs.readdir);
+const fsRmdir = promisify(fs.rmdir);
+const fsStat = promisify(fs.stat);
+const fsUnlink = promisify(fs.unlink);
+const fsWriteFile = promisify(fs.writeFile);
+
+export class BlacklistIndex {
+  constructor(
+    protected readonly now: Date,
+    protected readonly outDir: string,
+  ) {}
+
+  public async rebuild(): Promise<void> {
+    const validDates: string[] = [];
+    const day = new Date(this.now.getTime());
+    // Keeping "today" plus two weeks of historical data.
+    for (let i = 0; i < 15; i++) {
+      validDates.push(day.toISOString().substr(0, 10));
+      day.setDate(day.getDate() - 1);
+    }
+
+    let data: IBlacklistIndexEntry[] = [];
+    const removeDirectories: string[] = [];
+
+    await Promise.all(
+      (await fsReaddir(this.outDir)).map(
+        async (directory: string): Promise<void> => {
+          if (
+            /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/.exec(directory) &&
+            (await fsStat(path.join(this.outDir, directory))).isDirectory()
+          ) {
+            if (validDates.indexOf(directory) === -1) {
+              removeDirectories.push(directory);
+
+              return;
+            }
+
+            const files: string[] = [];
+            const dirPath = path.join(this.outDir, directory);
+            await Promise.all(
+              (await fsReaddir(dirPath)).map(
+                async (file: string): Promise<void> => {
+                  if (file === 'full.json' || /^delta\-[1-4]\.json$/) {
+                    files.push(file);
+                  }
+                },
+              ),
+            );
+
+            if (files.length) {
+              data.push({
+                date: directory,
+                files: {
+                  delta1: files.indexOf('delta1.json') !== -1,
+                  delta2: files.indexOf('delta2.json') !== -1,
+                  delta3: files.indexOf('delta3.json') !== -1,
+                  delta4: files.indexOf('delta4.json') !== -1,
+                  full: files.indexOf('full.json') !== -1,
+                },
+              });
+            }
+          }
+        },
+      ),
+    );
+
+    data = data.sort(
+      (a: IBlacklistIndexEntry, b: IBlacklistIndexEntry): number => {
+        const aTime = new Date(a.date).getTime();
+        const bTime = new Date(b.date).getTime();
+
+        if (aTime === bTime) {
+          return 0;
+        } else {
+          return aTime > bTime ? -1 : 1;
+        }
+      },
+    );
+
+    await fsWriteFile(
+      path.join(this.outDir, 'index.json'),
+      JSON.stringify(data, null, 2),
+    );
+
+    await this.cleanup(removeDirectories);
+  }
+
+  protected async cleanup(directories: string[]): Promise<void> {
+    await Promise.all(
+      directories.map(
+        async (directory: string): Promise<void> => {
+          const dirPath = path.join(this.outDir, directory);
+          await Promise.all(
+            (await fsReaddir(directory)).map(
+              async (file: string): Promise<void> => {
+                await fsUnlink(path.join(dirPath, file));
+              },
+            ),
+          );
+
+          await fsRmdir(dirPath);
+        },
+      ),
+    );
+  }
+}
+
+interface IBlacklistIndexEntry {
+  date: string;
+  files: {
+    delta1: boolean;
+    delta2: boolean;
+    delta3: boolean;
+    delta4: boolean;
+    full: boolean;
+  };
+}

diff --git a/src/blacklist.ts b/src/blacklist.ts
new file mode 100644 (file)
index 0000000..54f1a06
--- /dev/null
+++ b/src/blacklist.ts
@@ -0,0 +1,83 @@
+import { Container, Service } from 'typedi';
+
+import { CsvParser, ICsvRow } from './csv-parser';
+import { Database } from './database';
+
+@Service()
+export class Blacklist {
+  protected get db(): Database {
+    return Container.get(Database);
+  }
+
+  protected get parser(): CsvParser {
+    return Container.get(CsvParser);
+  }
+
+  constructor(public readonly type: string) {}
+
+  public async setup(): Promise<void> {
+    await this.db.exec(`CREATE TABLE IF NOT EXISTS ${this.tableName} (
+      hash TEXT NOT NULL,
+      occurrences INTEGER NOT NULL,
+      lastSeen INTEGER NOT NULL
+    );`);
+
+    await this.db.exec(
+      `CREATE UNIQUE INDEX IF NOT EXISTS hash_${this.type} ON ${
+        this.tableName
+      } (hash);`,
+    );
+  }
+
+  public async upsert(buffer: Buffer): Promise<void> {
+    const items: ICsvRow[] = await this.parser.parse(buffer);
+    if (items.length === 0) {
+      throw new Error(`Received a zero-length response for '${this.type}'`);
+    }
+
+    const statement = await this.db.prepare(
+      `INSERT INTO ${
+        this.tableName
+      } (hash, occurrences, lastSeen) VALUES (?, ?, ?)
+      ON CONFLICT(hash) DO UPDATE SET occurrences = excluded.occurrences, lastSeen = excluded.lastSeen;`,
+    );
+    items.forEach(
+      (item: ICsvRow): void => {
+        statement.run(item.hash, item.occurrences, item.lastSeen);
+      },
+    );
+
+    return statement.finalize();
+  }
+
+  public async getRows(
+    timestampStart: number,
+    timestampEnd: number,
+  ): Promise<any> {
+    const data: any = {};
+    await this.db.each(
+      `SELECT * FROM ${
+        this.tableName
+      } WHERE lastSeen BETWEEN ${timestampStart} AND ${timestampEnd}`,
+      (err: Error, row: IBlacklistItem): void => {
+        // Discard the value for 'lastSeen', the client can implicitly use the timestamp
+        // when it has fetched them. The loss of resolution is very well acceptable and
+        // the reduction of data decreases both the transfer size and the time required to
+        // parse the file on the client side.
+        data[row.hash] = row.occurrences;
+      },
+    );
+
+    return data;
+  }
+
+  protected get tableName(): string {
+    return `blacklist_${this.type}`;
+  }
+}
+
+export interface IBlacklistItem {
+  hash: string;
+  lastSeen: number;
+  occurrences: number;
+}

diff --git a/src/csv-parser.ts b/src/csv-parser.ts
new file mode 100644 (file)
index 0000000..dd2e408
--- /dev/null
+++ b/src/csv-parser.ts
@@ -0,0 +1,53 @@
+import { createHash, Hash } from 'crypto';
+import * as csvParse from 'csv-parse';
+import { Service } from 'typedi';
+
+@Service()
+export class CsvParser {
+  public async parse(buffer: Buffer): Promise<ICsvRow[]> {
+    const records = await this.parseCsv(buffer);
+
+    return records.map(
+      (columns: string[]): ICsvRow => {
+        return {
+          hash: this.getHash(columns[0]),
+          lastSeen: new Date(`${columns[2]}Z`).getTime(),
+          occurrences: parseInt(columns[1], 10),
+        };
+      },
+    );
+  }
+
+  protected async parseCsv(buffer: Buffer): Promise<string[][]> {
+    return new Promise<string[][]>(
+      (
+        resolve: (rows: string[][]) => void,
+        reject: (err: Error) => void,
+      ): void => {
+        csvParse(
+          buffer.toString(),
+          { escape: '\\' },
+          (err: Error | undefined, records: string[][] | undefined) => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve(records);
+            }
+          },
+        );
+      },
+    );
+  }
+
+  protected getHash(input: string): string {
+    return createHash('sha256')
+      .update(input)
+      .digest('hex');
+  }
+}
+
+export interface ICsvRow {
+  hash: string;
+  lastSeen: number;
+  occurrences: number;
+}

diff --git a/src/database-statement.ts b/src/database-statement.ts
new file mode 100644 (file)
index 0000000..7c233ea
--- /dev/null
+++ b/src/database-statement.ts
@@ -0,0 +1,36 @@
+import { RunResult, Statement } from 'sqlite3';
+
+export class DatabaseStatement {
+  constructor(protected statement: Statement) {}
+
+  public async run(...params: any[]): Promise<RunResult> {
+    return new Promise<RunResult>(
+      (
+        resolve: (result: RunResult) => void,
+        reject: (err: Error) => void,
+      ): void => {
+        this.statement.run(params, function(err?: Error): void {
+          if (err) {
+            reject(err);
+          } else {
+            resolve(this);
+          }
+        });
+      },
+    );
+  }
+
+  public async finalize(): Promise<void> {
+    return new Promise<void>(
+      (resolve: () => void, reject: (err: Error) => void): void => {
+        this.statement.finalize((err?: Error) => {
+          if (err) {
+            reject(err);
+          } else {
+            resolve();
+          }
+        });
+      },
+    );
+  }
+}

diff --git a/src/database.ts b/src/database.ts
new file mode 100644 (file)
index 0000000..4a11a43
--- /dev/null
+++ b/src/database.ts
@@ -0,0 +1,127 @@
+import * as path from 'path';
+import { Database as SqliteDatabase, RunResult } from 'sqlite3';
+import { Service } from 'typedi';
+
+import { DatabaseStatement } from './database-statement';
+
+@Service()
+export class Database {
+  protected db: SqliteDatabase;
+
+  constructor() {
+    this.db = new SqliteDatabase(path.join(__dirname, 'data.db'));
+  }
+
+  public async setup(): Promise<void> {
+    this.db.exec('PRAGMA synchronous = OFF');
+    this.db.exec('PRAGMA journal_mode = MEMORY');
+  }
+
+  public async shutdown(): Promise<void> {
+    return new Promise<void>(
+      (resolve: () => void, reject: (err: Error) => void): void => {
+        this.db.close(
+          (err?: Error): void => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve();
+            }
+          },
+        );
+      },
+    );
+  }
+
+  public async all(sql: string): Promise<any[]> {
+    return new Promise<any[]>(
+      (resolve: (rows: any[]) => void, reject: (err: Error) => void): void => {
+        this.db.all(
+          sql,
+          (err: Error | undefined, rows: any[]): void => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve(rows);
+            }
+          },
+        );
+      },
+    );
+  }
+
+  public async each(
+    sql: string,
+    callback: (err: Error | undefined, row: any) => void,
+  ): Promise<number> {
+    return new Promise<number>(
+      (
+        resolve: (count: number) => void,
+        reject: (err: Error) => void,
+      ): void => {
+        this.db.each(
+          sql,
+          callback,
+          (err: Error | undefined, count: number | undefined): void => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve(count);
+            }
+          },
+        );
+      },
+    );
+  }
+
+  public async exec(sql: string): Promise<void> {
+    return new Promise<void>(
+      (resolve: () => void, reject: (err: Error) => void): void => {
+        this.db.exec(
+          sql,
+          (err: Error | null): void => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve();
+            }
+          },
+        );
+      },
+    );
+  }
+
+  public async prepare(sql: string): Promise<DatabaseStatement> {
+    return new Promise<DatabaseStatement>(
+      (
+        resolve: (statement: DatabaseStatement) => void,
+        reject: (err: Error) => void,
+      ): void => {
+        this.db.prepare(sql, function(err: Error | null): void {
+          if (err) {
+            reject(err);
+          } else {
+            resolve(new DatabaseStatement(this));
+          }
+        });
+      },
+    );
+  }
+
+  public async run(sql: string): Promise<RunResult> {
+    return new Promise<RunResult>(
+      (
+        resolve: (result: RunResult) => void,
+        reject: (err: Error) => void,
+      ): void => {
+        this.db.run(sql, function(err?: Error): void {
+          if (err) {
+            reject(err);
+          } else {
+            resolve(this);
+          }
+        });
+      },
+    );
+  }
+}

diff --git a/src/index.ts b/src/index.ts
new file mode 100644 (file)
index 0000000..253d126
--- /dev/null
+++ b/src/index.ts
@@ -0,0 +1,11 @@
+import 'reflect-metadata';
+import { Container } from 'typedi';
+
+import { Manager } from './manager';
+
+(async (): Promise<void> => {
+  const manager = Container.get(Manager);
+  await manager.setup();
+  await manager.update();
+  await manager.rebuildIndex();
+})();

diff --git a/src/manager.ts b/src/manager.ts
new file mode 100644 (file)
index 0000000..252577c
--- /dev/null
+++ b/src/manager.ts
@@ -0,0 +1,213 @@
+import * as fs from 'fs';
+import * as minimist from 'minimist';
+import * as path from 'path';
+import { Inject, Service } from 'typedi';
+import { promisify } from 'util';
+
+import { Archive } from './archive';
+import { Blacklist, IBlacklistItem } from './blacklist';
+import { BlacklistIndex } from './blacklist-index';
+import { Database } from './database';
+
+const fsExists = promisify(fs.exists);
+const fsMkdir = promisify(fs.mkdir);
+const fsWriteFile = promisify(fs.writeFile);
+
+@Service()
+export class Manager {
+  protected readonly delta: number;
+  protected readonly deltaDate: string;
+  protected readonly deltaEnd: Date;
+  protected readonly deltaStart: Date;
+  protected readonly filenameDaily: string;
+  protected readonly filenameSixHours: string;
+  protected readonly now: Date;
+  protected readonly outDir: string;
+  protected readonly yesterday: Date;
+
+  @Inject()
+  protected db: Database;
+
+  constructor() {
+    const argv = minimist(process.argv.slice(2), {
+      string: ['out-dir'],
+      unknown: (): boolean => false,
+    });
+
+    this.outDir = argv['out-dir'];
+    if (!this.outDir) {
+      throw new Error("The '--out-dir' argument is missing.");
+    }
+
+    this.now = new Date();
+    this.yesterday = new Date(this.now.getTime());
+    this.yesterday.setDate(this.yesterday.getDate() - 1);
+
+    this.filenameDaily = path.join(
+      this.outDir,
+      this.yesterday.toISOString().substr(0, 10),
+      'full.json',
+    );
+
+    // The 6 hours delta only considers past segments in order to avoid incomplete data sets.
+    const utcHours = this.now.getUTCHours();
+    if (utcHours < 6) {
+      this.delta = 4;
+
+      this.deltaDate = this.yesterday.toISOString().substr(0, 10);
+      this.deltaStart = new Date(`${this.deltaDate}T18:00:00Z`);
+      this.deltaEnd = new Date(`${this.deltaDate}T23:59:59Z`);
+    } else {
+      this.delta = Math.floor(utcHours / 6);
+
+      this.deltaDate = this.now.toISOString().substr(0, 10);
+      this.deltaStart = new Date(
+        this.deltaDate +
+          'T' +
+          ((this.delta - 1) * 6).toString().padStart(2, '0') +
+          ':00:00Z',
+      );
+      this.deltaEnd = new Date(
+        this.deltaDate +
+          'T' +
+          ((this.delta - 1) * 6 + 5).toString().padStart(2, '0') +
+          ':59:59Z',
+      );
+    }
+
+    this.filenameSixHours = path.join(
+      this.outDir,
+      this.deltaDate,
+      `delta${this.delta}.json`,
+    );
+  }
+
+  public async setup(): Promise<void> {
+    await this.db.setup();
+  }
+
+  public async update(): Promise<void> {
+    const rebuildDaily = await this.pendingDailyUpdate();
+    const rebuildSixHours = await this.pendingSixHoursUpdate();
+
+    if (rebuildDaily || rebuildSixHours) {
+      const blacklists: Blacklist[] = [];
+
+      await Promise.all(
+        Archive.getAll().map(
+          async (archive: Archive): Promise<void> => {
+            const blacklist = new Blacklist(archive.type);
+            blacklists.push(blacklist);
+
+            await blacklist.setup();
+
+            const buffer: Buffer | undefined = await archive.download();
+            if (buffer) {
+              await this.updateBlacklist(blacklist, buffer);
+            } else {
+              console.error(
+                `Failed to download the file for '${archive.type}'.`,
+              );
+            }
+          },
+        ),
+      );
+
+      if (rebuildDaily) {
+        await this.rebuildDaily(blacklists);
+      }
+
+      if (rebuildSixHours) {
+        await this.rebuildSixHours(blacklists);
+      }
+    }
+  }
+
+  public async rebuildIndex(): Promise<void> {
+    await new BlacklistIndex(this.now, this.outDir).rebuild();
+  }
+
+  protected async updateBlacklist(
+    blacklist: Blacklist,
+    buffer: Buffer,
+  ): Promise<Blacklist> {
+    await blacklist.setup();
+
+    try {
+      await blacklist.upsert(buffer);
+    } catch (e) {
+      console.error(
+        `Failed to upsert the data for '${blacklist.type}': ${e.message}`,
+        e.stack,
+      );
+    }
+
+    return blacklist;
+  }
+
+  protected async rebuildDaily(blacklists: Blacklist[]): Promise<void> {
+    const date = this.yesterday.toISOString().substr(0, 10);
+    const start = date + 'T00:00:00Z';
+    const end = date + 'T23:59:59Z';
+
+    return this.rebuild(blacklists, this.filenameDaily, start, end, {
+      date,
+      end,
+      start,
+      type: 'day',
+    });
+  }
+
+  protected async rebuildSixHours(blacklists: Blacklist[]): Promise<void> {
+    return this.rebuild(
+      blacklists,
+      this.filenameSixHours,
+      this.deltaStart.toISOString(),
+      this.deltaEnd.toISOString(),
+      {
+        date: this.deltaDate,
+        end: this.deltaEnd.toISOString(),
+        start: this.deltaStart.toISOString(),
+        type: `delta${this.delta}`,
+      },
+    );
+  }
+
+  protected async rebuild(
+    blacklists: Blacklist[],
+    filename: string,
+    start: string,
+    end: string,
+    meta: any,
+  ): Promise<void> {
+    const data: any = {
+      meta,
+    };
+
+    await Promise.all(
+      blacklists.map(
+        async (blacklist: Blacklist): Promise<void> => {
+          data[blacklist.type] = await blacklist.getRows(
+            new Date(start).getTime(),
+            new Date(end).getTime(),
+          );
+        },
+      ),
+    );
+
+    const directory = path.dirname(filename);
+    if (!(await fsExists(directory))) {
+      await fsMkdir(directory);
+    }
+
+    await fsWriteFile(filename, JSON.stringify(data, null, 2));
+  }
+
+  protected async pendingDailyUpdate(): Promise<boolean> {
+    return !(await fsExists(this.filenameDaily));
+  }
+
+  protected async pendingSixHoursUpdate(): Promise<boolean> {
+    return !(await fsExists(this.filenameSixHours));
+  }
+}

diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644 (file)
index 0000000..e123d28
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "noImplicitAny": true,
+    "removeComments": true,
+    "preserveConstEnums": true,
+    "lib": [
+      "es2017"
+    ],
+    "target": "es2017",
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "outDir": "dist/",
+    "alwaysStrict": true
+  },
+  "include": [
+    "src/**/*.ts",
+  ],
+  "exclude": [
+    "node_modules",
+  ],
+}
\ No newline at end of file

diff --git a/tslint.json b/tslint.json
new file mode 100644 (file)
index 0000000..bfd3d3a
--- /dev/null
+++ b/tslint.json
@@ -0,0 +1,17 @@
+{
+  "extends": ["tslint:recommended"],
+  "rules": {
+    "arrow-return-shorthand": true,
+    "no-return-await": true,
+    "quotemark": [true, "single", "avoid-escape", "avoid-template"],
+    "return-undefined": true,
+    "typedef": [
+      true,
+      "call-signature",
+      "arrow-call-signature",
+      "parameter",
+      "arrow-parameter",
+      "member-variable-declaration"
+    ]
+  }
+}