From 9f2f63ff18552ed27f058c6fe39e81e1184e0360 Mon Sep 17 00:00:00 2001 From: Al Straumann Date: Tue, 3 Sep 2019 21:45:45 -0500 Subject: [PATCH 1/2] Update python version. Readd regex because my life is a lie. --- Pipfile | 2 +- Pipfile.lock | 323 +++++++++++++++++++-------------------------------- src/robot.py | 77 ++++++------ 3 files changed, 162 insertions(+), 240 deletions(-) diff --git a/Pipfile b/Pipfile index eb84df8..adfbc40 100644 --- a/Pipfile +++ b/Pipfile @@ -28,4 +28,4 @@ tox = "*" pytest-runner = "*" [requires] -python_version = "3.6" +python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index a6218bd..a5067c3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "43be0ae0c9c856c1c9943c5dd35fd460aa3748059da188ef87c8811d164bece2" + "sha256": "f5ae334fb689433627ed112de1afac4a9fc669cd3a7c80af7c81342b23eadbf9" }, "pipfile-spec": 6, "requires": { - "python_version": "3.6" + "python_version": "3" }, "sources": [ { @@ -16,13 +16,6 @@ ] }, "default": { - "aiodns": { - "hashes": [ - "sha256:815fdef4607474295d68da46978a54481dd1e7be153c7d60f9e72773cd38d77d", - "sha256:aaa5ac584f40fe778013df0aa6544bf157799bd3f608364b451840ed2c8688de" - ], - "version": "==2.0.0" - }, "aiohttp": { "hashes": [ "sha256:00d198585474299c9c3b4f1d5de1a576cc230d562abc5e4a0e81d71a20a6ca55", @@ -88,11 +81,11 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", - "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", - "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" + "sha256:05668158c7b85b791c5abde53e50265e16f98ad601c402ba44d70f96c4159612", + "sha256:25288c9e176f354bf277c0a10aa96c782a6a18a17122dba2e8cec4a97e03343b", + "sha256:f040590be10520f2ea4c2ae8c3dae441c7cfff5308ec9d58a0ec0c1b8f81d469" ], - "version": "==4.7.1" + "version": "==4.8.0" }, "bs4": { "hashes": [ @@ -109,39 +102,6 @@ "index": "pypi", "version": "==2019.6.16" }, - "cffi": { - "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" - ], - "version": "==1.12.3" - }, "chardet": { "hashes": [ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", @@ -179,11 +139,11 @@ }, "django": { "hashes": [ - "sha256:4d23f61b26892bac785f07401bc38cbf8fa4cec993f400e9cd9ddf28fd51c0ea", - "sha256:6e974d4b57e3b29e4882b244d40171d6a75202ab8d2402b8e8adbd182e25cf0c" + "sha256:148a4a2d1a85b23883b0a4e99ab7718f518a83675e4485e44dc0c1d36988c5fa", + "sha256:deb70aa038e59b58593673b15e9a711d1e5ccd941b5973b30750d5d026abfd56" ], "index": "pypi", - "version": "==2.2.3" + "version": "==2.2.5" }, "dnspython": { "hashes": [ @@ -203,11 +163,11 @@ }, "docutils": { "hashes": [ - "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", - "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", - "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" ], - "version": "==0.14" + "version": "==0.15.2" }, "idna": { "hashes": [ @@ -216,13 +176,6 @@ ], "version": "==2.8" }, - "idna-ssl": { - "hashes": [ - "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" - ], - "markers": "python_version < '3.7'", - "version": "==1.1.0" - }, "imagesize": { "hashes": [ "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", @@ -272,10 +225,10 @@ }, "mattermostdriver": { "hashes": [ - "sha256:d4590f8f5bb6a3dd381cd2973ac0b0ea10a23e42ec0310092150bf7a1a6b0137" + "sha256:7ab6cd779d08b817577647b665e8f7512417573de9d8a010f6e931575ab6ecdc" ], "index": "pypi", - "version": "==6.1.3" + "version": "==6.2.0" }, "multidict": { "hashes": [ @@ -321,34 +274,10 @@ }, "packaging": { "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" - ], - "version": "==19.0" - }, - "pycares": { - "hashes": [ - "sha256:2ca080db265ea238dc45f997f94effb62b979a617569889e265c26a839ed6305", - "sha256:6f79c6afb6ce603009db2042fddc2e348ad093ece9784cbe2daa809499871a23", - "sha256:70918d06eb0603016d37092a5f2c0228509eb4e6c5a3faacb4184f6ab7be7650", - "sha256:755187d28d24a9ea63aa2b4c0638be31d65fbf7f0ce16d41261b9f8cb55a1b99", - "sha256:7baa4b1f2146eb8423ff8303ebde3a20fb444a60db761fba0430d104fe35ddbf", - "sha256:90b27d4df86395f465a171386bc341098d6d47b65944df46518814ae298f6cc6", - "sha256:9e090dd6b2afa65cb51c133883b2bf2240fd0f717b130b0048714b33fb0f47ce", - "sha256:a11b7d63c3718775f6e805d6464cb10943780395ab042c7e5a0a7a9f612735dd", - "sha256:b253f5dcaa0ac7076b79388a3ac80dd8f3bd979108f813baade40d3a9b8bf0bd", - "sha256:c7f4f65e44ba35e35ad3febc844270665bba21cfb0fb7d749434e705b556e087", - "sha256:cdb342e6a254f035bd976d95807a2184038fc088d957a5104dcaab8be602c093", - "sha256:cf08e164f8bfb83b9fe633feb56f2754fae6baefcea663593794fa0518f8f98c", - "sha256:df9bc694cf03673878ea8ce674082c5acd134991d64d6c306d4bd61c0c1df98f" + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" ], - "version": "==3.0.0" - }, - "pycparser": { - "hashes": [ - "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" - ], - "version": "==2.19" + "version": "==19.1" }, "pygments": { "hashes": [ @@ -359,17 +288,17 @@ }, "pyparsing": { "hashes": [ - "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", - "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "version": "==2.4.0" + "version": "==2.4.2" }, "pytz": { "hashes": [ - "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", - "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", + "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" ], - "version": "==2019.1" + "version": "==2019.2" }, "requests": { "hashes": [ @@ -381,10 +310,10 @@ }, "shodan": { "hashes": [ - "sha256:4aa8ea11448159147dbdf65b2aa3b10d47c05decd94992fdd016efdc7781e91b" + "sha256:7903d110376dac9d3b7c046a1a1bd285e0f63a73649a700e80bbca3b698bb5bf" ], "index": "pypi", - "version": "==1.13.0" + "version": "==1.15.0" }, "six": { "hashes": [ @@ -395,11 +324,11 @@ }, "slackclient": { "hashes": [ - "sha256:82cef9b4aa4180be440598807903f618f07aebc0f61afe3135b81870edb1b1ad", - "sha256:dccece1609fae52d23af5e9b7418c7b74a3a1040048cf8ffe416521a6f29b8a4" + "sha256:ed0f60dd5a646b4ccd31ec1f11503b8689850894681e6baae2bc06b941fc1c58", + "sha256:eeb0dad774f6661f7504d80d8a22deadadb941b840e7a0719b5b92d4d84f9015" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.1.0" }, "snowballstemmer": { "hashes": [ @@ -409,18 +338,18 @@ }, "soupsieve": { "hashes": [ - "sha256:72b5f1aea9101cf720a36bb2327ede866fd6f1a07b1e87c92a1cc18113cbc946", - "sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de" + "sha256:8662843366b8d8779dec4e2f921bebec9afd856a5ff2e82cd419acc5054a1a92", + "sha256:a5a6166b4767725fd52ae55fee8c8b6137d9a51e9f1edea461a062a759160118" ], - "version": "==1.9.2" + "version": "==1.9.3" }, "sphinx": { "hashes": [ - "sha256:22538e1bbe62b407cf5a8aabe1bb15848aa66bb79559f42f5202bbce6b757a69", - "sha256:f9a79e746b87921cabc3baa375199c6076d1270cee53915dbd24fdbeaaacc427" + "sha256:0d586b0f8c2fc3cc6559c5e8fd6124628110514fda0e5d7c82e682d749d2e845", + "sha256:839a3ed6f6b092bb60f492024489cc9e6991360fb9f52ed6361acd510d261069" ], "index": "pypi", - "version": "==2.1.2" + "version": "==2.2.0" }, "sphinx-rtd-theme": { "hashes": [ @@ -481,29 +410,20 @@ }, "tqdm": { "hashes": [ - "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61", - "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab" + "sha256:1be3e4e3198f2d0e47b928e9d9a8ec1b63525db29095cec1467f4c5a4ea8ebf9", + "sha256:7e39a30e3d34a7a6539378e39d7490326253b7ee354878a92255656dc4284457" ], "index": "pypi", - "version": "==4.32.2" + "version": "==4.35.0" }, "typing": { "hashes": [ - "sha256:38566c558a0a94d6531012c8e917b1b8518a41e418f7f15f00e129cc80162ad3", - "sha256:53765ec4f83a2b720214727e319607879fec4acde22c4fbb54fa2604e79e44ce", - "sha256:84698954b4e6719e912ef9a42a2431407fe3755590831699debda6fba92aac55" + "sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23", + "sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36", + "sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714" ], "index": "pypi", - "version": "==3.7.4" - }, - "typing-extensions": { - "hashes": [ - "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", - "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", - "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" - ], - "markers": "python_version < '3.7'", - "version": "==3.7.4" + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ @@ -521,36 +441,26 @@ }, "websockets": { "hashes": [ - "sha256:04b42a1b57096ffa5627d6a78ea1ff7fad3bc2c0331ffc17bc32a4024da7fea0", - "sha256:08e3c3e0535befa4f0c4443824496c03ecc25062debbcf895874f8a0b4c97c9f", - "sha256:10d89d4326045bf5e15e83e9867c85d686b612822e4d8f149cf4840aab5f46e0", - "sha256:232fac8a1978fc1dead4b1c2fa27c7756750fb393eb4ac52f6bc87ba7242b2fa", - "sha256:4bf4c8097440eff22bc78ec76fe2a865a6e658b6977a504679aaf08f02c121da", - "sha256:51642ea3a00772d1e48fb0c492f0d3ae3b6474f34d20eca005a83f8c9c06c561", - "sha256:55d86102282a636e195dad68aaaf85b81d0bef449d7e2ef2ff79ac450bb25d53", - "sha256:564d2675682bd497b59907d2205031acbf7d3fadf8c763b689b9ede20300b215", - "sha256:5d13bf5197a92149dc0badcc2b699267ff65a867029f465accfca8abab95f412", - "sha256:5eda665f6789edb9b57b57a159b9c55482cbe5b046d7db458948370554b16439", - "sha256:5edb2524d4032be4564c65dc4f9d01e79fe8fad5f966e5b552f4e5164fef0885", - "sha256:79691794288bc51e2a3b8de2bc0272ca8355d0b8503077ea57c0716e840ebaef", - "sha256:7fcc8681e9981b9b511cdee7c580d5b005f3bb86b65bde2188e04a29f1d63317", - "sha256:8e447e05ec88b1b408a4c9cde85aa6f4b04f06aa874b9f0b8e8319faf51b1fee", - "sha256:90ea6b3e7787620bb295a4ae050d2811c807d65b1486749414f78cfd6fb61489", - "sha256:9e13239952694b8b831088431d15f771beace10edfcf9ef230cefea14f18508f", - "sha256:d40f081187f7b54d7a99d8a5c782eaa4edc335a057aa54c85059272ed826dc09", - "sha256:e1df1a58ed2468c7b7ce9a2f9752a32ad08eac2bcd56318625c3647c2cd2da6f", - "sha256:e98d0cec437097f09c7834a11c69d79fe6241729b23f656cfc227e93294fc242", - "sha256:f8d59627702d2ff27cb495ca1abdea8bd8d581de425c56e93bff6517134e0a9b", - "sha256:fc30cdf2e949a2225b012a7911d1d031df3d23e99b7eda7dfc982dc4a860dae9" + "sha256:049e694abe33f8a1d99969fee7bfc0ae6761f7fd5f297c58ea933b27dd6805f2", + "sha256:73ce69217e4655783ec72ce11c151053fcbd5b837cc39de7999e19605182e28a", + "sha256:83e63aa73331b9ca21af61df8f115fb5fbcba3f281bee650a4ad16a40cd1ef15", + "sha256:882a7266fa867a2ebb2c0baaa0f9159cabf131cf18c1b4270d79ad42f9208dc5", + "sha256:8c77f7d182a6ea2a9d09c2612059f3ad859a90243e899617137ee3f6b7f2b584", + "sha256:8d7a20a2f97f1e98c765651d9fb9437201a9ccc2c70e94b0270f1c5ef29667a3", + "sha256:a7affaeffbc5d55681934c16bb6b8fc82bb75b175e7fd4dcca798c938bde8dda", + "sha256:c82e286555f839846ef4f0fdd6910769a577952e1e26aa8ee7a6f45f040e3c2b", + "sha256:e906128532a14b9d264a43eb48f9b3080d53a9bda819ab45bf56b8039dc606ac", + "sha256:e9102043a81cdc8b7c8032ff4bce39f6229e4ac39cb2010946c912eeb84e2cb6", + "sha256:f5cb2683367e32da6a256b60929a3af9c29c212b5091cf5bace9358d03011bf5" ], - "version": "==7.0" + "version": "==8.0.2" }, "xlsxwriter": { "hashes": [ - "sha256:5ec6aa71f6ae4b6298376d8b6a56ca9cdcb8b80323a444212226447aed4fa10f", - "sha256:ec51d99c0cc5d95ec8d8e9c8de7c8fbbf461988bec01a8c86b5155a6716b0a5a" + "sha256:22904d1284b82426ed43b2ada0ff60f99c4736634acb7fc1584caf07789e55ac", + "sha256:e60be95ee38e81916d7c67f658902f9e51322400be20cf0563837f69a63ad3c5" ], - "version": "==1.1.8" + "version": "==1.2.0" }, "yarl": { "hashes": [ @@ -586,39 +496,40 @@ }, "coverage": { "hashes": [ - "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", - "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", - "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", - "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", - "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", - "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", - "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", - "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", - "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", - "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", - "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", - "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", - "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", - "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", - "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", - "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", - "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", - "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", - "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", - "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", - "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", - "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", - "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", - "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", - "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", - "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", - "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", - "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", - "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", - "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", - "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" - ], - "version": "==4.5.3" + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + ], + "version": "==4.5.4" }, "filelock": { "hashes": [ @@ -629,24 +540,25 @@ }, "importlib-metadata": { "hashes": [ - "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", - "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + "sha256:9ff1b1c5a354142de080b8a4e9803e5d0d59283c93aed808617c787d16768375", + "sha256:b7143592e374e50584564794fcb8aaf00a23025f9db866627f89a21491847a8d" ], - "version": "==0.18" + "markers": "python_version < '3.8'", + "version": "==0.20" }, "more-itertools": { "hashes": [ - "sha256:3ad685ff8512bf6dc5a8b82ebf73543999b657eded8c11803d9ba6b648986f4d", - "sha256:8bb43d1f51ecef60d81854af61a3a880555a14643691cc4b64a6ee269c78f09a" + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" ], - "version": "==7.1.0" + "version": "==7.2.0" }, "packaging": { "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" ], - "version": "==19.0" + "version": "==19.1" }, "pluggy": { "hashes": [ @@ -664,18 +576,18 @@ }, "pyparsing": { "hashes": [ - "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", - "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "version": "==2.4.0" + "version": "==2.4.2" }, "pytest": { "hashes": [ - "sha256:2878de8ae1c79a62c012da6186b88ff0562ea96ce29c4208d2a9b11d9f607df1", - "sha256:95b700cf21ed5b7e91bce7a6b5a573b2e3ef7b3643d00f681d8f9c4672f9fbdf" + "sha256:95d13143cc14174ca1a01ec68e84d76ba5d9d493ac02716fd9706c949a505210", + "sha256:b78fe2881323bd44fd9bd76e5317173d4316577e7b1cddebae9136a4495ec865" ], "index": "pypi", - "version": "==5.0.0" + "version": "==5.1.2" }, "pytest-cov": { "hashes": [ @@ -703,24 +615,25 @@ "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", + "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" ], "version": "==0.10.0" }, "tox": { "hashes": [ - "sha256:45a265e953368fb372cca3eac33b69fdb1b1453e5b114be231c84fc3dfadceed", - "sha256:f5cb0b5b8d14f2100982b0981c750d840228180a348e6bad355aa38e949fbc3f" + "sha256:0bc216b6a2e6afe764476b4a07edf2c1dab99ed82bb146a1130b2e828f5bff5e", + "sha256:c4f6b319c20ba4913dbfe71ebfd14ff95d1853c4231493608182f66e566ecfe1" ], "index": "pypi", - "version": "==3.13.1" + "version": "==3.14.0" }, "virtualenv": { "hashes": [ - "sha256:b7335cddd9260a3dd214b73a2521ffc09647bde3e9457fcca31dc3be3999d04a", - "sha256:d28ca64c0f3f125f59cabf13e0a150e1c68e5eea60983cc4395d88c584495783" + "sha256:680af46846662bb38c5504b78bad9ed9e4f3ba2d54f54ba42494fdf94337fe30", + "sha256:f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2" ], - "version": "==16.6.1" + "version": "==16.7.5" }, "wcwidth": { "hashes": [ @@ -731,10 +644,10 @@ }, "zipp": { "hashes": [ - "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", - "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "version": "==0.5.1" + "version": "==0.6.0" } } } diff --git a/src/robot.py b/src/robot.py index fffffc6..a501bfe 100644 --- a/src/robot.py +++ b/src/robot.py @@ -8,6 +8,7 @@ import threading import requests import glob +import mmap from urllib.parse import urlparse from requests.packages.urllib3.exceptions import InsecureRequestWarning from concurrent.futures import ThreadPoolExecutor @@ -445,11 +446,14 @@ def build_db(ips, cursor): """ cursor.execute('BEGIN TRANSACTION') domain = self.domain.replace(".","_") - for host, ip in ips: - cursor.execute("""INSERT OR IGNORE INTO data - (ip, hostname, http_headers, https_headers, domain) - VALUES (?,?, NULL, NULL, ?);""", - (ip, host, domain)) + try: + for host, ip in ips: + cursor.execute("""INSERT INTO data + (ip, hostname, http_headers, https_headers, domain) + VALUES (?,?, NULL, NULL, ?);""", + (ip, host, domain)) + except: + print(f"Issue with the following data: {ip} {host} {domain}") cursor.execute('COMMIT') @@ -519,14 +523,15 @@ def read_file(filename): all_ips =[] for filename in all_files: print(f"[*] Parsing file: {filename}") - for data in read_file(join_abs(filename)): - all_ips += self._reverse_ip_lookup(data) + all_ips += self._reverse_ip_lookup(filename) + # for data in read_file(join_abs(filename)): + # all_ips += self._reverse_ip_lookup(data) build_db(all_ips, dbcurs) dbconn.commit() finally: dbconn.close() - def _reverse_ip_lookup(self, data): + def _reverse_ip_lookup(self, filename): """ Args: @@ -537,33 +542,37 @@ def _reverse_ip_lookup(self, data): Returns: (List) of Tuples (hostname, ip) """ - + print("Extracting ips and hostnames from text") + ip_reg = re.compile(r"(?:(?:1\d\d|2[0-5][0-5]|2[0-4]\d|0?[1-9]\d|0?0?\d)\.){3}(?:1\d\d|2[0-5][0-5]|2[0-4]\d|0?[1-9]\d|0?0?\d)") + #hostname_reg = re.compile(r"([A-Za-z0-9\-]*\.?)*\." + self.domain) + hostname_reg = re.compile(r"([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*?\."+self.domain) results = [] - for item in tqdm(data): - hostname = None - ip = None - try: - socket.inet_pton(socket.AF_INET, item.rstrip()) - ip = item.rstrip() - except socket.error: - hostname = urlparse(item.rstrip()).netloc - - if hostname is None and ip is None: - continue - - try: - if not hostname and ip: - hostname = socket.gethostbyaddr(ip) - hostname = hostname[0] - if not ip and hostname: - ip = socket.gethostbyname(hostname) - - except socket.herror as er: - logger.exception(f"{ip} {hostname}: Host cannot be resolved, not adding") - except Exception as er: - logger.exception(f"{ip} {hostname}: Error with socket") - finally: - results += [(hostname, ip)] + with open(filename, "r") as f: + for line in tqdm(f.readlines()): + host = hostname_reg.match(line) + if host: + host = host.group() + ip = ip_reg.match(line) + if ip: + ip = ip.group() + try: + if host is not None and ip is None: + ip = socket.gethostbyname(host) + if ip is not None and host is None: + host = socket.gethostbyaddr(ip) + except: + pass + if host or ip: + results += [(host, ip)] + + # print("Extracting ips") + # for ip in tqdm(ips): + # hostname = None + # try: + # hostname = socket.gethostbyaddr(ip) + # hostname = hostname[0] + # except: + # results += [(hostname, ip)] return results From 3e25e32892e9ef95e9bdccfd7997a776d934fdb8 Mon Sep 17 00:00:00 2001 From: Al Straumann Date: Tue, 3 Sep 2019 22:16:31 -0500 Subject: [PATCH 2/2] Unique key for hostname and ip. Update Amass to reflect most recent changes --- docker_buildfiles/Dockerfile.Amass.tmp | 20 +++++++++++++++----- src/robot.py | 3 ++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docker_buildfiles/Dockerfile.Amass.tmp b/docker_buildfiles/Dockerfile.Amass.tmp index aff418d..87f504e 100644 --- a/docker_buildfiles/Dockerfile.Amass.tmp +++ b/docker_buildfiles/Dockerfile.Amass.tmp @@ -1,16 +1,26 @@ -FROM golang:alpine as build +FROM golang:1.12.6-alpine3.10 as build +RUN apk --no-cache add git +RUN go get github.com/OWASP/Amass; exit 0 + +ENV GO111MODULE on + +WORKDIR /go/src/github.com/OWASP/Amass + +RUN go install ./... + +FROM alpine:latest +COPY --from=build /go/bin/amass /bin/amass +COPY --from=build /go/src/github.com/OWASP/Amass/wordlists/ /wordlists/ ENV http_proxy $proxy ENV https_proxy $proxy ENV DNS $dns +ENV HOME / RUN mkdir -p $output -RUN apk --no-cache add git \ - && go get -u github.com/OWASP/Amass/... - ENV TARGET $target ENV OUTPUT $output/amass.txt ENV INFILE $infile -ENTRYPOINT amass -d "$target" --passive -o $output +ENTRYPOINT amass enum --passive -d "$target" -o $$OUTPUT diff --git a/src/robot.py b/src/robot.py index a501bfe..ba3b955 100644 --- a/src/robot.py +++ b/src/robot.py @@ -497,7 +497,8 @@ def read_file(filename): http_headers TEXT, https_headers TEXT, domain VARCHAR, - FOREIGN KEY(domain) REFERENCES domains(domain) + FOREIGN KEY(domain) REFERENCES domains(domain), + UNIQUE(ip, hostname) ) """) # Quickly create entry in domains table.