From d148e55046f14101bec9d1a549455ead9d0cac0e Mon Sep 17 00:00:00 2001 From: Alexander Ebert Date: Thu, 7 Mar 2019 22:21:31 +0100 Subject: [PATCH] Initial commit --- .gitignore | 66 +++ package-lock.json | 1111 +++++++++++++++++++++++++++++++++++++ package.json | 31 ++ src/archive.ts | 50 ++ src/blacklist-index.ts | 121 ++++ src/blacklist.ts | 83 +++ src/csv-parser.ts | 53 ++ src/database-statement.ts | 36 ++ src/database.ts | 127 +++++ src/index.ts | 11 + src/manager.ts | 213 +++++++ tsconfig.json | 22 + tslint.json | 17 + 13 files changed, 1941 insertions(+) create mode 100644 .gitignore create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 src/archive.ts create mode 100644 src/blacklist-index.ts create mode 100644 src/blacklist.ts create mode 100644 src/csv-parser.ts create mode 100644 src/database-statement.ts create mode 100644 src/database.ts create mode 100644 src/index.ts create mode 100644 src/manager.ts create mode 100644 tsconfig.json create mode 100644 tslint.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b9832f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,66 @@ +# Linux +# backup files +*~ + +# Windows +# thumbnails +Thumbs.db + +# Mac OS X +# metadata +.DS_Store +# thumbnails +._* + +# Visual Studio PHP +*.sln +*.phpproj +*.puo +*.suo +*.cache + +# Netbeans +nbproject/ +catalog.xml +nbactions.xml + +# Eclipse +.settings/ +.buildpath +.classpath +.project + +# SVN +# svn folders +.svn/ + +# PHPStorm +.idea/ +.nameencodings +.xmlmisc +.xmlmodules +.xmlprojectCodeStyle +.xmlvcs.xml +*.imlworkspace +.xml + +# Sublime Text 2 +*.sublime-* + +# Textmate +*.tmproj + +# Visual Studio Code +.vscode/ + +# WoltLab Suite +# Ignore packages build directly in the workspace. They can however be added manually via git add, if wanted. +*.tar +*.tar.gz + +# Node.js +dist/* +node_modules + +# Used for internal testing of the generated files. +public/* diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..a162893 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1111 @@ +{ + "name": "blacklist", + "version": "5.2.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/adm-zip": { + "version": "0.4.32", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.4.32.tgz", + "integrity": "sha512-hv1O7ySn+XvP5OeDQcJFWwVb2v+GFGO1A9aMTQ5B/bzxb7WW21O8iRhVdsKKr8QwuiagzGmPP+gsUAYZ6bRddQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/csv-parse": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/@types/csv-parse/-/csv-parse-1.1.12.tgz", + "integrity": "sha512-p6uZznjJOcFaymduLYf45ik28IYzChnkt+ofJOWa16bb2JRCHdxs/ai03q6raizCc5JuunVsbgtlDxfu9y2Nag==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY=", + "dev": true + }, + "@types/node": { + "version": "11.9.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-11.9.5.tgz", + "integrity": "sha512-vVjM0SVzgaOUpflq4GYBvCpozes8OgIIS5gVXVka+OfK3hvnkC1i93U8WiY2OtNE4XUWyyy/86Kf6e0IHTQw1Q==", + "dev": true + }, + "@types/node-fetch": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.1.6.tgz", + "integrity": "sha512-Hv1jgh3pfpUEl2F2mqUd1AfLSk1YbUCeBJFaP36t7esAO617dErqdxWb5cdG2NfJGOofkmBW36fdx0dVewxDRg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/sqlite3": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@types/sqlite3/-/sqlite3-3.1.4.tgz", + "integrity": "sha512-EZ/fWzLSKclqFyvMQEw2Ii/NYRTbIWzkdbm56Lgnq/GKNX0o3rkAvNvggXgWkUzgC28nC7yYVgfhWUHvUPd/Fg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, + "adm-zip": { + "version": "0.4.13", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.13.tgz", + "integrity": "sha512-fERNJX8sOXfel6qCBCMPvZLzENBEhZTzKqg6vrOW5pvoEaQuJhRU4ndTAh6lHOxn1I6jnz2NHra56ZODM751uw==" + }, + "ajv": { + "version": "6.9.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz", + "integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true + }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "babel-code-frame": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", + "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "esutils": "^2.0.2", + "js-tokens": "^3.0.2" + }, + "dependencies": { + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + } + } + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "builtin-modules": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", + "dev": true + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "chownr": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==" + }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "combined-stream": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "commander": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", + "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "csv-parse": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.3.3.tgz", + "integrity": "sha512-bZ+AZjm2LlWEp5+TKeFeXDldduCUUaxEif+KUv+zvAwmCvCKTqeSHVEyxztGCQ6OE+87ObRq4NsCmg91SuJbhQ==" + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" + }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + }, + "diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "dev": true + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "fs-minipass": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz", + "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", + "requires": { + "minipass": "^2.2.1" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "ignore-walk": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz", + "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", + "requires": { + "minimatch": "^3.0.4" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "js-tokens": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", + "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=", + "dev": true + }, + "js-yaml": { + "version": "3.12.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.2.tgz", + "integrity": "sha512-QHn/Lh/7HhZ/Twc7vJYQTkjuCa0kaCcDcjK5Zlk2rvnUpy7DxMJ23+Jc2dcyvltwQVg1nygAVlB2oRDFHoRS5Q==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "mime-db": { + "version": "1.38.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", + "integrity": "sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg==" + }, + "mime-types": { + "version": "2.1.22", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz", + "integrity": "sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog==", + "requires": { + "mime-db": "~1.38.0" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "minipass": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "minizlib": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", + "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", + "requires": { + "minipass": "^2.2.1" + } + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + } + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "nan": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" + }, + "needle": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz", + "integrity": "sha512-HyoqEb4wr/rsoaIDfTH2aVL9nWtQqba2/HvMv+++m8u0dz808MaagKILxtfeSN7QU7nvbQ79zk3vYOJp9zsNEA==", + "requires": { + "debug": "^2.1.2", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + } + }, + "node-fetch": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz", + "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==" + }, + "node-pre-gyp": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", + "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + } + }, + "nopt": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", + "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, + "npm-bundled": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz", + "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==" + }, + "npm-packlist": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz", + "integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==", + "requires": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1" + } + }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, + "psl": { + "version": "1.1.31", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", + "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "reflect-metadata": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", + "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "resolve": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz", + "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "sqlite3": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.0.6.tgz", + "integrity": "sha512-EqBXxHdKiwvNMRCgml86VTL5TK1i0IKiumnfxykX0gh6H6jaKijAXvE9O1N7+omfNSawR2fOmIyJZcfe8HYWpw==", + "requires": { + "nan": "~2.10.0", + "node-pre-gyp": "^0.11.0", + "request": "^2.87.0" + } + }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true + }, + "tar": { + "version": "4.4.8", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz", + "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.3.4", + "minizlib": "^1.1.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.2" + } + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, + "tslib": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz", + "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==", + "dev": true + }, + "tslint": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.13.0.tgz", + "integrity": "sha512-ECOOQRxXCYnUUePG5h/+Z1Zouobk3KFpIHA9aKBB/nnMxs97S1JJPDGt5J4cGm1y9U9VmVlfboOxA8n1kSNzGw==", + "dev": true, + "requires": { + "babel-code-frame": "^6.22.0", + "builtin-modules": "^1.1.1", + "chalk": "^2.3.0", + "commander": "^2.12.1", + "diff": "^3.2.0", + "glob": "^7.1.1", + "js-yaml": "^3.7.0", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "resolve": "^1.3.2", + "semver": "^5.3.0", + "tslib": "^1.8.0", + "tsutils": "^2.27.2" + } + }, + "tsutils": { + "version": "2.29.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", + "integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==", + "dev": true, + "requires": { + "tslib": "^1.8.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "typedi": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/typedi/-/typedi-0.8.0.tgz", + "integrity": "sha512-/c7Bxnm6eh5kXx2I+mTuO+2OvoWni5+rXA3PhXwVWCtJRYmz3hMok5s1AKLzoDvNAZqj/Q/acGstN0ri5aQoOA==" + }, + "typescript": { + "version": "3.3.3333", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.3.3333.tgz", + "integrity": "sha512-JjSKsAfuHBE/fB2oZ8NxtRTk5iGcg6hkYXMnZ3Wc+b2RSqejEqTaem11mHASMnFilHrax3sLK0GDzcJrekZYLw==", + "dev": true + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "requires": { + "string-width": "^1.0.2 || 2" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..f899745 --- /dev/null +++ b/package.json @@ -0,0 +1,31 @@ +{ + "name": "blacklist", + "version": "5.2.0", + "description": "Reads and processes the data from stopforumspam.com, optimizing it for efficient delta upgrades.", + "main": "dist/index.js", + "scripts": { + "build": "tsc -w", + "lint": "tslint -c tslint.json -p tsconfig.json", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Alexander Ebert", + "license": "LGPL-2.1-or-later", + "dependencies": { + "adm-zip": "^0.4.13", + "csv-parse": "^4.3.3", + "minimist": "^1.2.0", + "node-fetch": "^2.3.0", + "reflect-metadata": "^0.1.13", + "sqlite3": "^4.0.6", + "typedi": "^0.8.0" + }, + "devDependencies": { + "@types/adm-zip": "^0.4.32", + "@types/csv-parse": "^1.1.12", + "@types/minimist": "^1.2.0", + "@types/node-fetch": "^2.1.6", + "@types/sqlite3": "^3.1.4", + "tslint": "^5.13.0", + "typescript": "^3.3.3333" + } +} diff --git a/src/archive.ts b/src/archive.ts new file mode 100644 index 0000000..ccce32b --- /dev/null +++ b/src/archive.ts @@ -0,0 +1,50 @@ +import * as AdmZip from 'adm-zip'; +import fetch from 'node-fetch'; + +export const sources = new Map([ + ['ipv4', 'https://www.stopforumspam.com/downloads/listed_ip_1_all.zip'], + ['ipv6', 'https://www.stopforumspam.com/downloads/listed_ip_1_ipv6_all.zip'], + ['email', 'https://www.stopforumspam.com/downloads/listed_email_1_all.zip'], + [ + 'username', + 'https://www.stopforumspam.com/downloads/listed_username_1_all.zip', + ], +]); + +export class Archive { + public get filename(): string { + return `tmp.${this.type}.csv`; + } + + public static getAll(): Archive[] { + const archives: Archive[] = []; + sources.forEach( + (url: string, type: string): void => { + archives.push(new Archive(type)); + }, + ); + + return archives; + } + constructor(public readonly type: string) { + if (!sources.has(this.type)) { + throw new Error(`The type '${this.type}' is not known.`); + } + } + + public async download(): Promise { + const response = await fetch(sources.get(this.type)); + if (response.ok) { + const entries: AdmZip.IZipEntry[] = new AdmZip( + await response.buffer(), + ).getEntries(); + for (let i = 0, length = entries.length; i < length; i++) { + if (/^listed_.+_all\.txt$/.exec(entries[i].name)) { + return entries[i].getData(); + } + } + } + + return undefined; + } +} diff --git a/src/blacklist-index.ts b/src/blacklist-index.ts new file mode 100644 index 0000000..5c233da --- /dev/null +++ b/src/blacklist-index.ts @@ -0,0 +1,121 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { promisify } from 'util'; + +const fsReaddir = promisify(fs.readdir); +const fsRmdir = promisify(fs.rmdir); +const fsStat = promisify(fs.stat); +const fsUnlink = promisify(fs.unlink); +const fsWriteFile = promisify(fs.writeFile); + +export class BlacklistIndex { + constructor( + protected readonly now: Date, + protected readonly outDir: string, + ) {} + + public async rebuild(): Promise { + const validDates: string[] = []; + const day = new Date(this.now.getTime()); + // Keeping "today" plus two weeks of historical data. + for (let i = 0; i < 15; i++) { + validDates.push(day.toISOString().substr(0, 10)); + day.setDate(day.getDate() - 1); + } + + let data: IBlacklistIndexEntry[] = []; + const removeDirectories: string[] = []; + + await Promise.all( + (await fsReaddir(this.outDir)).map( + async (directory: string): Promise => { + if ( + /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/.exec(directory) && + (await fsStat(path.join(this.outDir, directory))).isDirectory() + ) { + if (validDates.indexOf(directory) === -1) { + removeDirectories.push(directory); + + return; + } + + const files: string[] = []; + const dirPath = path.join(this.outDir, directory); + await Promise.all( + (await fsReaddir(dirPath)).map( + async (file: string): Promise => { + if (file === 'full.json' || /^delta\-[1-4]\.json$/) { + files.push(file); + } + }, + ), + ); + + if (files.length) { + data.push({ + date: directory, + files: { + delta1: files.indexOf('delta1.json') !== -1, + delta2: files.indexOf('delta2.json') !== -1, + delta3: files.indexOf('delta3.json') !== -1, + delta4: files.indexOf('delta4.json') !== -1, + full: files.indexOf('full.json') !== -1, + }, + }); + } + } + }, + ), + ); + + data = data.sort( + (a: IBlacklistIndexEntry, b: IBlacklistIndexEntry): number => { + const aTime = new Date(a.date).getTime(); + const bTime = new Date(b.date).getTime(); + + if (aTime === bTime) { + return 0; + } else { + return aTime > bTime ? -1 : 1; + } + }, + ); + + await fsWriteFile( + path.join(this.outDir, 'index.json'), + JSON.stringify(data, null, 2), + ); + + await this.cleanup(removeDirectories); + } + + protected async cleanup(directories: string[]): Promise { + await Promise.all( + directories.map( + async (directory: string): Promise => { + const dirPath = path.join(this.outDir, directory); + await Promise.all( + (await fsReaddir(directory)).map( + async (file: string): Promise => { + await fsUnlink(path.join(dirPath, file)); + }, + ), + ); + + await fsRmdir(dirPath); + }, + ), + ); + } +} + +interface IBlacklistIndexEntry { + date: string; + files: { + delta1: boolean; + delta2: boolean; + delta3: boolean; + delta4: boolean; + full: boolean; + }; +} diff --git a/src/blacklist.ts b/src/blacklist.ts new file mode 100644 index 0000000..54f1a06 --- /dev/null +++ b/src/blacklist.ts @@ -0,0 +1,83 @@ +import { Container, Service } from 'typedi'; + +import { CsvParser, ICsvRow } from './csv-parser'; +import { Database } from './database'; + +@Service() +export class Blacklist { + protected get db(): Database { + return Container.get(Database); + } + + protected get parser(): CsvParser { + return Container.get(CsvParser); + } + + constructor(public readonly type: string) {} + + public async setup(): Promise { + await this.db.exec(`CREATE TABLE IF NOT EXISTS ${this.tableName} ( + hash TEXT NOT NULL, + occurrences INTEGER NOT NULL, + lastSeen INTEGER NOT NULL + );`); + + await this.db.exec( + `CREATE UNIQUE INDEX IF NOT EXISTS hash_${this.type} ON ${ + this.tableName + } (hash);`, + ); + } + + public async upsert(buffer: Buffer): Promise { + const items: ICsvRow[] = await this.parser.parse(buffer); + if (items.length === 0) { + throw new Error(`Received a zero-length response for '${this.type}'`); + } + + const statement = await this.db.prepare( + `INSERT INTO ${ + this.tableName + } (hash, occurrences, lastSeen) VALUES (?, ?, ?) + ON CONFLICT(hash) DO UPDATE SET occurrences = excluded.occurrences, lastSeen = excluded.lastSeen;`, + ); + items.forEach( + (item: ICsvRow): void => { + statement.run(item.hash, item.occurrences, item.lastSeen); + }, + ); + + return statement.finalize(); + } + + public async getRows( + timestampStart: number, + timestampEnd: number, + ): Promise { + const data: any = {}; + await this.db.each( + `SELECT * FROM ${ + this.tableName + } WHERE lastSeen BETWEEN ${timestampStart} AND ${timestampEnd}`, + (err: Error, row: IBlacklistItem): void => { + // Discard the value for 'lastSeen', the client can implicitly use the timestamp + // when it has fetched them. The loss of resolution is very well acceptable and + // the reduction of data decreases both the transfer size and the time required to + // parse the file on the client side. + data[row.hash] = row.occurrences; + }, + ); + + return data; + } + + protected get tableName(): string { + return `blacklist_${this.type}`; + } +} + +export interface IBlacklistItem { + hash: string; + lastSeen: number; + occurrences: number; +} diff --git a/src/csv-parser.ts b/src/csv-parser.ts new file mode 100644 index 0000000..dd2e408 --- /dev/null +++ b/src/csv-parser.ts @@ -0,0 +1,53 @@ +import { createHash, Hash } from 'crypto'; +import * as csvParse from 'csv-parse'; +import { Service } from 'typedi'; + +@Service() +export class CsvParser { + public async parse(buffer: Buffer): Promise { + const records = await this.parseCsv(buffer); + + return records.map( + (columns: string[]): ICsvRow => { + return { + hash: this.getHash(columns[0]), + lastSeen: new Date(`${columns[2]}Z`).getTime(), + occurrences: parseInt(columns[1], 10), + }; + }, + ); + } + + protected async parseCsv(buffer: Buffer): Promise { + return new Promise( + ( + resolve: (rows: string[][]) => void, + reject: (err: Error) => void, + ): void => { + csvParse( + buffer.toString(), + { escape: '\\' }, + (err: Error | undefined, records: string[][] | undefined) => { + if (err) { + reject(err); + } else { + resolve(records); + } + }, + ); + }, + ); + } + + protected getHash(input: string): string { + return createHash('sha256') + .update(input) + .digest('hex'); + } +} + +export interface ICsvRow { + hash: string; + lastSeen: number; + occurrences: number; +} diff --git a/src/database-statement.ts b/src/database-statement.ts new file mode 100644 index 0000000..7c233ea --- /dev/null +++ b/src/database-statement.ts @@ -0,0 +1,36 @@ +import { RunResult, Statement } from 'sqlite3'; + +export class DatabaseStatement { + constructor(protected statement: Statement) {} + + public async run(...params: any[]): Promise { + return new Promise( + ( + resolve: (result: RunResult) => void, + reject: (err: Error) => void, + ): void => { + this.statement.run(params, function(err?: Error): void { + if (err) { + reject(err); + } else { + resolve(this); + } + }); + }, + ); + } + + public async finalize(): Promise { + return new Promise( + (resolve: () => void, reject: (err: Error) => void): void => { + this.statement.finalize((err?: Error) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }, + ); + } +} diff --git a/src/database.ts b/src/database.ts new file mode 100644 index 0000000..4a11a43 --- /dev/null +++ b/src/database.ts @@ -0,0 +1,127 @@ +import * as path from 'path'; +import { Database as SqliteDatabase, RunResult } from 'sqlite3'; +import { Service } from 'typedi'; + +import { DatabaseStatement } from './database-statement'; + +@Service() +export class Database { + protected db: SqliteDatabase; + + constructor() { + this.db = new SqliteDatabase(path.join(__dirname, 'data.db')); + } + + public async setup(): Promise { + this.db.exec('PRAGMA synchronous = OFF'); + this.db.exec('PRAGMA journal_mode = MEMORY'); + } + + public async shutdown(): Promise { + return new Promise( + (resolve: () => void, reject: (err: Error) => void): void => { + this.db.close( + (err?: Error): void => { + if (err) { + reject(err); + } else { + resolve(); + } + }, + ); + }, + ); + } + + public async all(sql: string): Promise { + return new Promise( + (resolve: (rows: any[]) => void, reject: (err: Error) => void): void => { + this.db.all( + sql, + (err: Error | undefined, rows: any[]): void => { + if (err) { + reject(err); + } else { + resolve(rows); + } + }, + ); + }, + ); + } + + public async each( + sql: string, + callback: (err: Error | undefined, row: any) => void, + ): Promise { + return new Promise( + ( + resolve: (count: number) => void, + reject: (err: Error) => void, + ): void => { + this.db.each( + sql, + callback, + (err: Error | undefined, count: number | undefined): void => { + if (err) { + reject(err); + } else { + resolve(count); + } + }, + ); + }, + ); + } + + public async exec(sql: string): Promise { + return new Promise( + (resolve: () => void, reject: (err: Error) => void): void => { + this.db.exec( + sql, + (err: Error | null): void => { + if (err) { + reject(err); + } else { + resolve(); + } + }, + ); + }, + ); + } + + public async prepare(sql: string): Promise { + return new Promise( + ( + resolve: (statement: DatabaseStatement) => void, + reject: (err: Error) => void, + ): void => { + this.db.prepare(sql, function(err: Error | null): void { + if (err) { + reject(err); + } else { + resolve(new DatabaseStatement(this)); + } + }); + }, + ); + } + + public async run(sql: string): Promise { + return new Promise( + ( + resolve: (result: RunResult) => void, + reject: (err: Error) => void, + ): void => { + this.db.run(sql, function(err?: Error): void { + if (err) { + reject(err); + } else { + resolve(this); + } + }); + }, + ); + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..253d126 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,11 @@ +import 'reflect-metadata'; +import { Container } from 'typedi'; + +import { Manager } from './manager'; + +(async (): Promise => { + const manager = Container.get(Manager); + await manager.setup(); + await manager.update(); + await manager.rebuildIndex(); +})(); diff --git a/src/manager.ts b/src/manager.ts new file mode 100644 index 0000000..252577c --- /dev/null +++ b/src/manager.ts @@ -0,0 +1,213 @@ +import * as fs from 'fs'; +import * as minimist from 'minimist'; +import * as path from 'path'; +import { Inject, Service } from 'typedi'; +import { promisify } from 'util'; + +import { Archive } from './archive'; +import { Blacklist, IBlacklistItem } from './blacklist'; +import { BlacklistIndex } from './blacklist-index'; +import { Database } from './database'; + +const fsExists = promisify(fs.exists); +const fsMkdir = promisify(fs.mkdir); +const fsWriteFile = promisify(fs.writeFile); + +@Service() +export class Manager { + protected readonly delta: number; + protected readonly deltaDate: string; + protected readonly deltaEnd: Date; + protected readonly deltaStart: Date; + protected readonly filenameDaily: string; + protected readonly filenameSixHours: string; + protected readonly now: Date; + protected readonly outDir: string; + protected readonly yesterday: Date; + + @Inject() + protected db: Database; + + constructor() { + const argv = minimist(process.argv.slice(2), { + string: ['out-dir'], + unknown: (): boolean => false, + }); + + this.outDir = argv['out-dir']; + if (!this.outDir) { + throw new Error("The '--out-dir' argument is missing."); + } + + this.now = new Date(); + this.yesterday = new Date(this.now.getTime()); + this.yesterday.setDate(this.yesterday.getDate() - 1); + + this.filenameDaily = path.join( + this.outDir, + this.yesterday.toISOString().substr(0, 10), + 'full.json', + ); + + // The 6 hours delta only considers past segments in order to avoid incomplete data sets. + const utcHours = this.now.getUTCHours(); + if (utcHours < 6) { + this.delta = 4; + + this.deltaDate = this.yesterday.toISOString().substr(0, 10); + this.deltaStart = new Date(`${this.deltaDate}T18:00:00Z`); + this.deltaEnd = new Date(`${this.deltaDate}T23:59:59Z`); + } else { + this.delta = Math.floor(utcHours / 6); + + this.deltaDate = this.now.toISOString().substr(0, 10); + this.deltaStart = new Date( + this.deltaDate + + 'T' + + ((this.delta - 1) * 6).toString().padStart(2, '0') + + ':00:00Z', + ); + this.deltaEnd = new Date( + this.deltaDate + + 'T' + + ((this.delta - 1) * 6 + 5).toString().padStart(2, '0') + + ':59:59Z', + ); + } + + this.filenameSixHours = path.join( + this.outDir, + this.deltaDate, + `delta${this.delta}.json`, + ); + } + + public async setup(): Promise { + await this.db.setup(); + } + + public async update(): Promise { + const rebuildDaily = await this.pendingDailyUpdate(); + const rebuildSixHours = await this.pendingSixHoursUpdate(); + + if (rebuildDaily || rebuildSixHours) { + const blacklists: Blacklist[] = []; + + await Promise.all( + Archive.getAll().map( + async (archive: Archive): Promise => { + const blacklist = new Blacklist(archive.type); + blacklists.push(blacklist); + + await blacklist.setup(); + + const buffer: Buffer | undefined = await archive.download(); + if (buffer) { + await this.updateBlacklist(blacklist, buffer); + } else { + console.error( + `Failed to download the file for '${archive.type}'.`, + ); + } + }, + ), + ); + + if (rebuildDaily) { + await this.rebuildDaily(blacklists); + } + + if (rebuildSixHours) { + await this.rebuildSixHours(blacklists); + } + } + } + + public async rebuildIndex(): Promise { + await new BlacklistIndex(this.now, this.outDir).rebuild(); + } + + protected async updateBlacklist( + blacklist: Blacklist, + buffer: Buffer, + ): Promise { + await blacklist.setup(); + + try { + await blacklist.upsert(buffer); + } catch (e) { + console.error( + `Failed to upsert the data for '${blacklist.type}': ${e.message}`, + e.stack, + ); + } + + return blacklist; + } + + protected async rebuildDaily(blacklists: Blacklist[]): Promise { + const date = this.yesterday.toISOString().substr(0, 10); + const start = date + 'T00:00:00Z'; + const end = date + 'T23:59:59Z'; + + return this.rebuild(blacklists, this.filenameDaily, start, end, { + date, + end, + start, + type: 'day', + }); + } + + protected async rebuildSixHours(blacklists: Blacklist[]): Promise { + return this.rebuild( + blacklists, + this.filenameSixHours, + this.deltaStart.toISOString(), + this.deltaEnd.toISOString(), + { + date: this.deltaDate, + end: this.deltaEnd.toISOString(), + start: this.deltaStart.toISOString(), + type: `delta${this.delta}`, + }, + ); + } + + protected async rebuild( + blacklists: Blacklist[], + filename: string, + start: string, + end: string, + meta: any, + ): Promise { + const data: any = { + meta, + }; + + await Promise.all( + blacklists.map( + async (blacklist: Blacklist): Promise => { + data[blacklist.type] = await blacklist.getRows( + new Date(start).getTime(), + new Date(end).getTime(), + ); + }, + ), + ); + + const directory = path.dirname(filename); + if (!(await fsExists(directory))) { + await fsMkdir(directory); + } + + await fsWriteFile(filename, JSON.stringify(data, null, 2)); + } + + protected async pendingDailyUpdate(): Promise { + return !(await fsExists(this.filenameDaily)); + } + + protected async pendingSixHoursUpdate(): Promise { + return !(await fsExists(this.filenameSixHours)); + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..e123d28 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "module": "commonjs", + "noImplicitAny": true, + "removeComments": true, + "preserveConstEnums": true, + "lib": [ + "es2017" + ], + "target": "es2017", + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "outDir": "dist/", + "alwaysStrict": true + }, + "include": [ + "src/**/*.ts", + ], + "exclude": [ + "node_modules", + ], +} \ No newline at end of file diff --git a/tslint.json b/tslint.json new file mode 100644 index 0000000..bfd3d3a --- /dev/null +++ b/tslint.json @@ -0,0 +1,17 @@ +{ + "extends": ["tslint:recommended"], + "rules": { + "arrow-return-shorthand": true, + "no-return-await": true, + "quotemark": [true, "single", "avoid-escape", "avoid-template"], + "return-undefined": true, + "typedef": [ + true, + "call-signature", + "arrow-call-signature", + "parameter", + "arrow-parameter", + "member-variable-declaration" + ] + } +} -- 2.20.1