diff --git a/Pipfile b/Pipfile index 5cd21a0d..a43b9c3c 100644 --- a/Pipfile +++ b/Pipfile @@ -76,6 +76,8 @@ python-dateutil = "*" geoip2 = "*" iso3166 = "*" django-logentry-admin = "*" +pandas = "*" +rich = "*" [pipenv] # Needed for `black`. See https://github.com/microsoft/vscode-python/pull/5967. diff --git a/Pipfile.lock b/Pipfile.lock index 342dcd3e..92ab8ca5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "01b663466247d744bc35c06abaec815b56696bcda8af2397389b584cb61cca96" + "sha256": "612fda4ff92d3308f840b5005d3acb36a5daaaf2020283d0d5e5c0df109d9739" }, "pipfile-spec": 6, "requires": {}, @@ -126,35 +126,36 @@ }, "bcrypt": { "hashes": [ - "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521", - "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb", - "sha256:4e029cef560967fb0cf4a802bcf4d562d3d6b4b1bf81de5ec1abbe0f1adb027e", - "sha256:61bae49580dce88095d669226d5076d0b9d927754cedbdf76c6c9f5099ad6f26", - "sha256:6d2cb9d969bfca5bc08e45864137276e4c3d3d7de2b162171def3d188bf9d34a", - "sha256:7180d98a96f00b1050e93f5b0f556e658605dd9f524d0b0e68ae7944673f525e", - "sha256:7d9ba2e41e330d2af4af6b1b6ec9e6128e91343d0b4afb9282e54e5508f31baa", - "sha256:7ff2069240c6bbe49109fe84ca80508773a904f5a8cb960e02a977f7f519b129", - "sha256:88273d806ab3a50d06bc6a2fc7c87d737dd669b76ad955f449c43095389bc8fb", - "sha256:a2c46100e315c3a5b90fdc53e429c006c5f962529bc27e1dfd656292c20ccc40", - "sha256:cd43303d6b8a165c29ec6756afd169faba9396a9472cdff753fe9f19b96ce2fa" - ], - "version": "==3.2.2" + "sha256:0b0f0c7141622a31e9734b7f649451147c04ebb5122327ac0bd23744df84be90", + "sha256:1c3334446fac200499e8bc04a530ce3cf0b3d7151e0e4ac5c0dddd3d95e97843", + "sha256:2d0dd19aad87e4ab882ef1d12df505f4c52b28b69666ce83c528f42c07379227", + "sha256:594780b364fb45f2634c46ec8d3e61c1c0f1811c4f2da60e8eb15594ecbf93ed", + "sha256:7c7dd6c1f05bf89e65261d97ac3a6520f34c2acb369afb57e3ea4449be6ff8fd", + "sha256:845b1daf4df2dd94d2fdbc9454953ca9dd0e12970a0bfc9f3dcc6faea3fa96e4", + "sha256:8780e69f9deec9d60f947b169507d2c9816e4f11548f1f7ebee2af38b9b22ae4", + "sha256:bf413f2a9b0a2950fc750998899013f2e718d20fa4a58b85ca50b6df5ed1bbf9", + "sha256:bfb67f6a6c72dfb0a02f3df51550aa1862708e55128b22543e2b42c74f3620d7", + "sha256:c59c170fc9225faad04dde1ba61d85b413946e8ce2e5f5f5ff30dfd67283f319", + "sha256:dc6ec3dc19b1c193b2f7cf279d3e32e7caf447532fbcb7af0906fe4398900c33", + "sha256:ede0f506554571c8eda80db22b83c139303ec6b595b8f60c4c8157bdd0bdee36" + ], + "version": "==4.0.0" }, "boto3": { "hashes": [ - "sha256:44026e44549148dbc5b261ead5f6b339e785680c350ef621bf85f7e2fca05b49", - "sha256:b2d9d55f123a9a91eea2fd8e379d90abf37634420fbb45c22d67e10b324ec71b" + "sha256:818a40b82e4f66b4bdd4fa38fcc3ed0fb26542f7d8c4d15279d4ba1d4762cd95", + "sha256:84b962f18506ad495dc973aeb5168697ee882497c01430dddeba5e8339db7932" ], "index": "pypi", - "version": "==1.24.46" + "version": "==1.24.62" }, "botocore": { "hashes": [ - "sha256:747b7e94aef41498f063fc0be79c5af102d940beea713965179e1ead89c7e9ec", - "sha256:f66d8305d1f59d83334df9b11b6512bb1e14698ec4d5d6d42f833f39f3304ca7" + "sha256:69682c874dc8ed1856bffd203786c9591fb76a1946d3dcc516bda1ee6a6989f3", + "sha256:8563c7d8b80e8041667cf35b397f1c399537f6c887e1c501f0064bfd7ae541ba" ], "markers": "python_version >= '3.7'", - "version": "==1.27.46" + "version": "==1.27.62" }, "brotli": { "hashes": [ @@ -231,82 +232,13 @@ "markers": "python_version >= '3.6'", "version": "==2022.6.15" }, - "cffi": { - "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "version": "==1.15.1" - }, "charset-normalizer": { "hashes": [ - "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", - "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", + "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], "markers": "python_version >= '3.6'", - "version": "==2.1.0" + "version": "==2.1.1" }, "click": { "hashes": [ @@ -323,6 +255,13 @@ ], "version": "==1.2.2" }, + "commonmark": { + "hashes": [ + "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", + "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9" + ], + "version": "==0.9.1" + }, "confusable-homoglyphs": { "hashes": [ "sha256:3b4a0d9fa510669498820c91a0bfc0c327568cecec90648cf3819d4a6fc6a751", @@ -471,11 +410,11 @@ }, "django-mysql": { "hashes": [ - "sha256:13d640413680b57c0ea3a49c4a0a9d1ac2b4a6e679baa5ac299dd45f3880a583", - "sha256:26ba087d9163b411d404d44bc53dfbb4b321b4692eee3061a373040e9482000e" + "sha256:ec809868113c652930874c055dd59baf93b3ba8c6d7e90bd891e28db15a4c840", + "sha256:f66b02fabd48f858b9fd70e57f4d66f1c6c807047a2eec8b7dce2b0aa1dcd4c1" ], "index": "pypi", - "version": "==4.7.0" + "version": "==4.7.1" }, "django-redis": { "hashes": [ @@ -495,11 +434,11 @@ }, "django-storages": { "hashes": [ - "sha256:204a99f218b747c46edbfeeb1310d357f83f90fa6a6024d8d0a3f422570cee84", - "sha256:a475edb2f0f04c4f7e548919a751ecd50117270833956ed5bd585c0575d2a5e7" + "sha256:3540b45618b04be2c867c0982e8d2bd8e34f84dae922267fcebe4691fb93daf0", + "sha256:b3d98ecc09f1b1627c2b2cf430964322ce4e08617dbf9b4236c16a32875a1e0b" ], "index": "pypi", - "version": "==1.12.3" + "version": "==1.13.1" }, "django-taggit": { "hashes": [ @@ -542,11 +481,11 @@ }, "django-waffle": { "hashes": [ - "sha256:150c0867d0f30c650a5c1e59a56b164780b62ab3bd48c4c65b6ae3481cf9dcfa", - "sha256:b0fef2cab15e739cadaffaf5b71206e813604bad093dc547959f1e3ee76c556d" + "sha256:3f4bd5e265c27ff3346b8379f5165a43e0791c515c8c08e7a94faf6281c758db", + "sha256:da1b76d4ca42b5c59c798ffec022a6209eb3dd2eb8fe0ce30da6dd9d9a5a5e24" ], "index": "pypi", - "version": "==2.6.0" + "version": "==2.7.0" }, "django-widget-tweaks": { "hashes": [ @@ -949,6 +888,40 @@ "index": "pypi", "version": "==2.1.1" }, + "numpy": { + "hashes": [ + "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0", + "sha256:2bd879d3ca4b6f39b7770829f73278b7c5e248c91d538aab1e506c628353e47f", + "sha256:4f41f5bf20d9a521f8cab3a34557cd77b6f205ab2116651f12959714494268b0", + "sha256:5593f67e66dea4e237f5af998d31a43e447786b2154ba1ad833676c788f37cde", + "sha256:5e28cd64624dc2354a349152599e55308eb6ca95a13ce6a7d5679ebff2962913", + "sha256:633679a472934b1c20a12ed0c9a6c9eb167fbb4cb89031939bfd03dd9dbc62b8", + "sha256:806970e69106556d1dd200e26647e9bee5e2b3f1814f9da104a943e8d548ca38", + "sha256:806cc25d5c43e240db709875e947076b2826f47c2c340a5a2f36da5bb10c58d6", + "sha256:8247f01c4721479e482cc2f9f7d973f3f47810cbc8c65e38fd1bbd3141cc9842", + "sha256:8ebf7e194b89bc66b78475bd3624d92980fca4e5bb86dda08d677d786fefc414", + "sha256:8ecb818231afe5f0f568c81f12ce50f2b828ff2b27487520d85eb44c71313b9e", + "sha256:8f9d84a24889ebb4c641a9b99e54adb8cab50972f0166a3abc14c3b93163f074", + "sha256:909c56c4d4341ec8315291a105169d8aae732cfb4c250fbc375a1efb7a844f8f", + "sha256:9b83d48e464f393d46e8dd8171687394d39bc5abfe2978896b77dc2604e8635d", + "sha256:ac987b35df8c2a2eab495ee206658117e9ce867acf3ccb376a19e83070e69418", + "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01", + "sha256:b8b97a8a87cadcd3f94659b4ef6ec056261fa1e1c3317f4193ac231d4df70215", + "sha256:bd5b7ccae24e3d8501ee5563e82febc1771e73bd268eef82a1e8d2b4d556ae66", + "sha256:bdc02c0235b261925102b1bd586579b7158e9d0d07ecb61148a1799214a4afd5", + "sha256:be6b350dfbc7f708d9d853663772a9310783ea58f6035eec649fb9c4371b5389", + "sha256:c403c81bb8ffb1c993d0165a11493fd4bf1353d258f6997b3ee288b0a48fce77", + "sha256:cf8c6aed12a935abf2e290860af8e77b26a042eb7f2582ff83dc7ed5f963340c", + "sha256:d98addfd3c8728ee8b2c49126f3c44c703e2b005d4a95998e2167af176a9e722", + "sha256:dc76bca1ca98f4b122114435f83f1fcf3c0fe48e4e6f660e07996abf2f53903c", + "sha256:dec198619b7dbd6db58603cd256e092bcadef22a796f778bf87f8592b468441d", + "sha256:df28dda02c9328e122661f399f7655cdcbcf22ea42daa3650a26bce08a187450", + "sha256:e603ca1fb47b913942f3e660a15e55a9ebca906857edfea476ae5f0fe9b457d5", + "sha256:ecfdd68d334a6b97472ed032b5b37a30d8217c097acfff15e8452c710e775524" + ], + "markers": "python_version < '3.10'", + "version": "==1.23.2" + }, "packaging": { "hashes": [ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", @@ -957,6 +930,33 @@ "markers": "python_version >= '3.6'", "version": "==21.3" }, + "pandas": { + "hashes": [ + "sha256:0329e22df7e85fd4cd4df5d186fad200fa879abc62583a2b0fa5919032b3e7d5", + "sha256:0a3cfd8777f3eb9fa6df12ea6f4aea338a9205507eb12d8dca3b3f00cf4ffe17", + "sha256:1002500cd99fbd00b4421f252ee7f8be5b6aabfeff29d5a744ec73ba3a917beb", + "sha256:1db16e716aac37e0d77278526c59027bdd02701994586912d83f4bf10ef80a06", + "sha256:1e2b9186b4d960043c47f62aeb489d88f74bfa574b486d4d2fd52bf496f94c23", + "sha256:4517b8f426b152620143ada0059e3a4142802f9582372885eb56a509283ca216", + "sha256:53400419ae6d9cec1cc8df719c0cbf7f76da9afd1725a1428993b112adb7f268", + "sha256:61fbe9e0e638a606ad47e362af5d8c71e9d33e4e5824e5b8d3811cdb45aabc89", + "sha256:6b842469bbdf1e7733c70b4e9bceee90ff8f421ad871cefb09d7e4d326557112", + "sha256:774c07952f1da5c8813ca082744161797dfd05c9bd16f0afae8c9774e82e06e6", + "sha256:7c0bd042e4d43c185f21a5616b98eaa0f5370ac816e633949a2c3ce922d35c0f", + "sha256:a9a8b27f08c7fe9b9bdb8631e663af140cee0deeac102dee84c867633b460e2a", + "sha256:b0cb459061105bc0927ebba493cb4cb745e43a18532b92e49000c4f34ea30eb9", + "sha256:bc626256a6d430e96b6e47208c0e5f9ad0403558a2237873e68ec37ce321bf70", + "sha256:bf99d2ba08dd476c3adf2baaa6e6ed16f78a35832146555260c08915e0633e3f", + "sha256:c0df2515125d83489521e1edfebe7cab2276c9b2984a21dcc5e8ef563e4b8e0d", + "sha256:c8867fc7722fc5f10c61ee7e8b76295c41d20d0f26b1f7341837476efca66601", + "sha256:cba004d0d33a28722c89b41512d90398f8041d0c24492f794863db1ddcc7932f", + "sha256:dac53ad9e6c45e1dae2263a6789aab79aa3b24b8974760ce22febe05b0b63865", + "sha256:f972af3232d41f6c755682f3f11ba0e5f0cf546a7eb6b3beba36677c5fcb2ca0", + "sha256:f9f763d6a88c21690a5cad79c6f3074c3b71e04f8d949edf1970d27f0d746cf5" + ], + "index": "pypi", + "version": "==1.5.0rc0" + }, "phpserialize": { "hashes": [ "sha256:bf672d312d203d09a84c26366fab8f438a3ffb355c407e69974b7ef2d39a0fa7" @@ -1043,12 +1043,13 @@ "markers": "python_version >= '3.6'", "version": "==0.14.1" }, - "pycparser": { + "pygments": { "hashes": [ - "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", - "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1", + "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42" ], - "version": "==2.21" + "markers": "python_version >= '3.6'", + "version": "==2.13.0" }, "pyparsing": { "hashes": [ @@ -1068,10 +1069,10 @@ }, "pytz": { "hashes": [ - "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", - "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197", + "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5" ], - "version": "==2022.1" + "version": "==2022.2.1" }, "pyyaml": { "hashes": [ @@ -1128,6 +1129,14 @@ "index": "pypi", "version": "==2.28.1" }, + "rich": { + "hashes": [ + "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb", + "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca" + ], + "index": "pypi", + "version": "==12.5.1" + }, "ruamel.yaml": { "hashes": [ "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7", @@ -1138,13 +1147,17 @@ }, "ruamel.yaml.clib": { "hashes": [ + "sha256:066f886bc90cc2ce44df8b5f7acfc6a7e2b2e672713f027136464492b0c34d7c", "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd", "sha256:1070ba9dd7f9370d0513d649420c3b362ac2d687fe78c6e888f5b12bf8bc7bee", "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0", + "sha256:1b4139a6ffbca8ef60fdaf9b33dec05143ba746a6f0ae0f9d11d38239211d335", + "sha256:210c8fcfeff90514b7133010bf14e3bad652c8efde6b20e00c43854bf94fa5a6", "sha256:221eca6f35076c6ae472a531afa1c223b9c29377e62936f61bc8e6e8bdc5f9e7", "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277", "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104", "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd", + "sha256:61bc5e5ca632d95925907c569daa559ea194a4d16084ba86084be98ab1cec1c6", "sha256:6e7be2c5bcb297f5b82fee9c665eb2eb7001d1050deaba8471842979293a80b0", "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78", "sha256:77df077d32921ad46f34816a9a16e6356d8100374579bc35e15bab5d4e9377de", @@ -1160,6 +1173,7 @@ "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5", "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe", "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751", + "sha256:d3c620a54748a3d4cf0bcfe623e388407c8e85a4b06b8188e126302bcab93ea8", "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502", "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed", "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c" @@ -1177,11 +1191,11 @@ }, "sentry-sdk": { "hashes": [ - "sha256:60b13757d6344a94bf0ccb3c0a006c4de77daab09871b30fbbd05d5ec24e54fb", - "sha256:f185c53496d79b280fe5d9d21e6572aee1ab802d3354eb12314d216cfbaa8d30" + "sha256:2d7ec7bc88ebbdf2c4b6b2650b3257893d386325a96c9b723adcd31033469b63", + "sha256:b4b41f90951ed83e7b4c176eef021b19ecba39da5b73aca106c97a9b7e279a90" ], "index": "pypi", - "version": "==1.9.0" + "version": "==1.9.5" }, "six": { "hashes": [ @@ -1193,45 +1207,45 @@ }, "sqlalchemy": { "hashes": [ - "sha256:047ef5ccd8860f6147b8ac6c45a4bc573d4e030267b45d9a1c47b55962ff0e6f", - "sha256:05a05771617bfa723ba4cef58d5b25ac028b0d68f28f403edebed5b8243b3a87", - "sha256:0ec54460475f0c42512895c99c63d90dd2d9cbd0c13491a184182e85074b04c5", - "sha256:107df519eb33d7f8e0d0d052128af2f25066c1a0f6b648fd1a9612ab66800b86", - "sha256:14ea8ff2d33c48f8e6c3c472111d893b9e356284d1482102da9678195e5a8eac", - "sha256:1745987ada1890b0e7978abdb22c133eca2e89ab98dc17939042240063e1ef21", - "sha256:1962dfee37b7fb17d3d4889bf84c4ea08b1c36707194c578f61e6e06d12ab90f", - "sha256:20bf65bcce65c538e68d5df27402b39341fabeecf01de7e0e72b9d9836c13c52", - "sha256:26146c59576dfe9c546c9f45397a7c7c4a90c25679492ff610a7500afc7d03a6", - "sha256:365b75938049ae31cf2176efd3d598213ddb9eb883fbc82086efa019a5f649df", - "sha256:4770eb3ba69ec5fa41c681a75e53e0e342ac24c1f9220d883458b5596888e43a", - "sha256:50e7569637e2e02253295527ff34666706dbb2bc5f6c61a5a7f44b9610c9bb09", - "sha256:5c2d19bfb33262bf987ef0062345efd0f54c4189c2d95159c72995457bf4a359", - "sha256:621f050e72cc7dfd9ad4594ff0abeaad954d6e4a2891545e8f1a53dcdfbef445", - "sha256:6d81de54e45f1d756785405c9d06cd17918c2eecc2d4262dc2d276ca612c2f61", - "sha256:6f95706da857e6e79b54c33c1214f5467aab10600aa508ddd1239d5df271986e", - "sha256:752ef2e8dbaa3c5d419f322e3632f00ba6b1c3230f65bc97c2ff5c5c6c08f441", - "sha256:7b2785dd2a0c044a36836857ac27310dc7a99166253551ee8f5408930958cc60", - "sha256:7f13644b15665f7322f9e0635129e0ef2098409484df67fcd225d954c5861559", - "sha256:8194896038753b46b08a0b0ae89a5d80c897fb601dd51e243ed5720f1f155d27", - "sha256:864d4f89f054819cb95e93100b7d251e4d114d1c60bc7576db07b046432af280", - "sha256:8b773c9974c272aae0fa7e95b576d98d17ee65f69d8644f9b6ffc90ee96b4d19", - "sha256:8f901be74f00a13bf375241a778455ee864c2c21c79154aad196b7a994e1144f", - "sha256:91d2b89bb0c302f89e753bea008936acfa4e18c156fb264fe41eb6bbb2bbcdeb", - "sha256:b0538b66f959771c56ff996d828081908a6a52a47c5548faed4a3d0a027a5368", - "sha256:b30e70f1594ee3c8902978fd71900d7312453922827c4ce0012fa6a8278d6df4", - "sha256:b71be98ef6e180217d1797185c75507060a57ab9cd835653e0112db16a710f0d", - "sha256:c6d00cb9da8d0cbfaba18cad046e94b06de6d4d0ffd9d4095a3ad1838af22528", - "sha256:d1f665e50592caf4cad3caed3ed86f93227bffe0680218ccbb293bd5a6734ca8", - "sha256:e6e2c8581c6620136b9530137954a8376efffd57fe19802182c7561b0ab48b48", - "sha256:e7a7667d928ba6ee361a3176e1bef6847c1062b37726b33505cc84136f657e0d", - "sha256:ec3985c883d6d217cf2013028afc6e3c82b8907192ba6195d6e49885bfc4b19d", - "sha256:ede13a472caa85a13abe5095e71676af985d7690eaa8461aeac5c74f6600b6c0", - "sha256:f24d4d6ec301688c59b0c4bb1c1c94c5d0bff4ecad33bb8f5d9efdfb8d8bc925", - "sha256:f2a42acc01568b9701665e85562bbff78ec3e21981c7d51d56717c22e5d3d58b", - "sha256:fbc076f79d830ae4c9d49926180a1140b49fa675d0f0d555b44c9a15b29f4c80" + "sha256:00dd998b43b282c71de46b061627b5edb9332510eb1edfc5017b9e4356ed44ea", + "sha256:08b47c971327e733ffd6bae2d4f50a7b761793efe69d41067fcba86282819eea", + "sha256:0992f3cc640ec0f88f721e426da884c34ff0a60eb73d3d64172e23dfadfc8a0b", + "sha256:0c956a5d1adb49a35d78ef0fae26717afc48a36262359bb5b0cbd7a3a247c26f", + "sha256:1ab08141d93de83559f6a7d9a962830f918623a885b3759ec2b9d1a531ff28fe", + "sha256:1cf03d37819dc17a388d313919daf32058d19ba1e592efdf14ce8cbd997e6023", + "sha256:2026632051a93997cf8f6fda14360f99230be1725b7ab2ef15be205a4b8a5430", + "sha256:23b693876ac7963b6bc7b1a5f3a2642f38d2624af834faad5933913928089d1b", + "sha256:26ee4dbac5dd7abf18bf3cd8f04e51f72c339caf702f68172d308888cd26c6c9", + "sha256:28b1791a30d62fc104070965f1a2866699c45bbf5adc0be0cf5f22935edcac58", + "sha256:2b64955850a14b9d481c17becf0d3f62fb1bb31ac2c45c2caf5ad06d9e811187", + "sha256:2cf50611ef4221ad587fb7a1708e61ff72966f84330c6317642e08d6db4138fd", + "sha256:44a660506080cc975e1dfa5776fe5f6315ddc626a77b50bf0eee18b0389ea265", + "sha256:4ec440990ab00650d0c7ea2c75bc225087afdd7ddcb248e3d934def4dff62762", + "sha256:63ad778f4e80913fb171247e4fa82123d0068615ae1d51a9791fc4284cb81748", + "sha256:69deec3a94de10062080d91e1ba69595efeafeafe68b996426dec9720031fb25", + "sha256:6b70d02bbe1adbbf715d2249cacf9ac17c6f8d22dfcb3f1a4fbc5bf64364da8a", + "sha256:885e11638946472b4a0a7db8e6df604b2cf64d23dc40eedc3806d869fcb18fae", + "sha256:959bf4390766a8696aa01285016c766b4eb676f712878aac5fce956dd49695d9", + "sha256:9ced2450c9fd016f9232d976661623e54c450679eeefc7aa48a3d29924a63189", + "sha256:a0b9e3d81f86ba04007f0349e373a5b8c81ec2047aadb8d669caf8c54a092461", + "sha256:a62c0ecbb9976550f26f7bf75569f425e661e7249349487f1483115e5fc893a6", + "sha256:b07fc38e6392a65935dc8b486229679142b2ea33c94059366b4d8b56f1e35a97", + "sha256:b41b87b929118838bafc4bb18cf3c5cd1b3be4b61cd9042e75174df79e8ac7a2", + "sha256:b7ccdca6cd167611f4a62a8c2c0c4285c2535640d77108f782ce3f3cccb70f3a", + "sha256:b7ff0a8bf0aec1908b92b8dfa1246128bf4f94adbdd3da6730e9c542e112542d", + "sha256:bb342c0e25cc8f78a0e7c692da3b984f072666b316fbbec2a0e371cb4dfef5f0", + "sha256:bf073c619b5a7f7cd731507d0fdc7329bee14b247a63b0419929e4acd24afea8", + "sha256:c8d974c991eef0cd29418a5957ae544559dc326685a6f26b3a914c87759bf2f4", + "sha256:c9d0f1a9538cc5e75f2ea0cb6c3d70155a1b7f18092c052e0d84105622a41b63", + "sha256:cdee4d475e35684d210dc6b430ff8ca2ed0636378ac19b457e2f6f350d1f5acc", + "sha256:cfa8ab4ba0c97ab6bcae1f0948497d14c11b6c6ecd1b32b8a79546a0823d8211", + "sha256:d259fa08e4b3ed952c01711268bcf6cd2442b0c54866d64aece122f83da77c6d", + "sha256:f2aa85aebc0ef6b342d5d3542f969caa8c6a63c8d36cf5098769158a9fa2123c", + "sha256:fa9e0d7832b7511b3b3fd0e67fac85ff11fd752834c143ca2364c9b778c0485a", + "sha256:fb4edb6c354eac0fcc07cb91797e142f702532dbb16c1d62839d6eec35f814cf" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.4.39" + "version": "==1.4.40" }, "sqlite-fts4": { "hashes": [ @@ -1242,11 +1256,11 @@ }, "sqlite-utils": { "hashes": [ - "sha256:790b01f4be16c32756b9b5eae07b6b7c905b6613ca538d646877c50b05b0a53a", - "sha256:e8841bff7fcf29b0789c374a4ccca42107de1592836aa6899a04e3aebd304277" + "sha256:9e6b2bf7ae89e02e107557b0cff1121f1fbb06128a815122743332197693b272", + "sha256:d9ea1026a9c007a895cdd04abdcbe3cd2ac03515c2a2ebbad9233939aa111f5b" ], "index": "pypi", - "version": "==3.28" + "version": "==3.29" }, "sqlparse": { "hashes": [ @@ -1291,7 +1305,7 @@ "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6" ], - "markers": "python_version >= '3.7'", + "markers": "python_version < '3.9'", "version": "==4.3.0" }, "unicodecsv": { @@ -1310,11 +1324,11 @@ }, "urllib3": { "hashes": [ - "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", - "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" + "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", + "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", - "version": "==1.26.11" + "markers": "python_version >= '3.5'", + "version": "==1.26.12" }, "whitenoise": { "extras": [ @@ -1479,14 +1493,6 @@ ], "version": "==0.7.12" }, - "appnope": { - "hashes": [ - "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", - "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.3" - }, "asgiref": { "hashes": [ "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4", @@ -1497,18 +1503,18 @@ }, "astroid": { "hashes": [ - "sha256:86b0a340a512c65abf4368b80252754cda17c02cdbbd3f587dddf98112233e7b", - "sha256:bb24615c77f4837c707669d16907331374ae8a964650a66999da3f5ca68dc946" + "sha256:396c88d0a58d7f8daadf730b2ce90838bf338c6752558db719ec6f99c18ec20e", + "sha256:d612609242996c4365aeb0345e61edba34363eaaba55f1c0addf6a98f073bef6" ], - "markers": "python_full_version >= '3.6.2'", - "version": "==2.11.7" + "markers": "python_full_version >= '3.7.2'", + "version": "==2.12.5" }, "asttokens": { "hashes": [ - "sha256:0844691e88552595a6f4a4281a9f7f79b8dd45ca4ccea82e5e05b4bbdb76705c", - "sha256:9a54c114f02c7a9480d56550932546a3f1fe71d8a02f1bc7ccd0ee3ee35cf4d5" + "sha256:c61e16246ecfb2cde2958406b4c8ebc043c9e6d73aaa83c941673b35e5d3a76b", + "sha256:e3305297c744ae53ffa032c45dc347286165e4ffce6875dc662b205db0623d86" ], - "version": "==2.0.5" + "version": "==2.0.8" }, "attrs": { "hashes": [ @@ -1523,7 +1529,7 @@ "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51", "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.10.3" }, "backcall": { @@ -1533,12 +1539,34 @@ ], "version": "==0.2.0" }, + "backports.zoneinfo": { + "hashes": [ + "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf", + "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328", + "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546", + "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6", + "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570", + "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9", + "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7", + "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987", + "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722", + "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582", + "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc", + "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b", + "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1", + "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08", + "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac", + "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2" + ], + "markers": "python_version < '3.9'", + "version": "==0.2.1" + }, "beautifulsoup4": { "hashes": [ "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30", "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.11.1" }, "black": { @@ -1580,11 +1608,11 @@ }, "charset-normalizer": { "hashes": [ - "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", - "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", + "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], "markers": "python_version >= '3.6'", - "version": "==2.1.0" + "version": "==2.1.1" }, "click": { "hashes": [ @@ -1623,50 +1651,59 @@ "toml" ], "hashes": [ - "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32", - "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7", - "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996", - "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55", - "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46", - "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de", - "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039", - "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee", - "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1", - "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f", - "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63", - "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083", - "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe", - "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0", - "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6", - "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe", - "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933", - "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0", - "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c", - "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07", - "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8", - "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b", - "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e", - "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120", - "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f", - "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e", - "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd", - "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f", - "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386", - "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8", - "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae", - "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc", - "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783", - "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d", - "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c", - "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97", - "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978", - "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf", - "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29", - "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39", - "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452" - ], - "index": "pypi", - "version": "==6.4.2" + "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2", + "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820", + "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827", + "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3", + "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d", + "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145", + "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875", + "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2", + "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74", + "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f", + "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c", + "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973", + "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1", + "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782", + "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0", + "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760", + "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a", + "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3", + "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7", + "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a", + "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f", + "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e", + "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86", + "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa", + "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa", + "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796", + "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a", + "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928", + "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0", + "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac", + "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c", + "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685", + "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d", + "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e", + "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f", + "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558", + "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58", + "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781", + "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a", + "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa", + "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc", + "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892", + "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d", + "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817", + "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1", + "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c", + "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908", + "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19", + "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60", + "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b" + ], + "index": "pypi", + "version": "==6.4.4" }, "decorator": { "hashes": [ @@ -1697,11 +1734,11 @@ }, "django-debug-toolbar": { "hashes": [ - "sha256:89a52128309eb4da12738801ff0c202d2ff8730d1c3225fac6acf630c303e661", - "sha256:97965f2630692de316ea0c1ca5bfa81660d7ba13146dbc6be2059cf55b35d0e5" + "sha256:95fc2fd29c56cc86678aae9f6919ececefe892f2a78c4004b193a223a8380c3d", + "sha256:fe7fe3f21865218827e2162ecc06eba386dfe8cffe4f3501c49bb4359e06a0e6" ], "index": "pypi", - "version": "==3.5.0" + "version": "==3.6.0" }, "docutils": { "hashes": [ @@ -1713,11 +1750,11 @@ }, "exceptiongroup": { "hashes": [ - "sha256:6990c24f06b8d33c8065cfe43e5e8a4bfa384e0358be036af9cc60b6321bd11a", - "sha256:ab0a968e1ef769e55d9a596f4a89f7be9ffedbc9fdefdb77cc68cf5c33ce1035" + "sha256:2e3c3fc1538a094aab74fad52d6c33fc94de3dfee3ee01f187c0e0c72aec5337", + "sha256:9086a4a21ef9b31c72181c77c040a074ba0889ee56a7b289ff0afb0d97655f96" ], "markers": "python_version < '3.11'", - "version": "==1.0.0rc8" + "version": "==1.0.0rc9" }, "execnet": { "hashes": [ @@ -1729,10 +1766,10 @@ }, "executing": { "hashes": [ - "sha256:4ce4d6082d99361c0231fc31ac1a0f56979363cc6819de0b1410784f99e49105", - "sha256:ea278e2cf90cbbacd24f1080dd1f0ac25b71b2e21f50ab439b7ba45dd3195587" + "sha256:550d581b497228b572235e633599133eeee67073c65914ca346100ad56775349", + "sha256:98daefa9d1916a4f0d944880d5aeaf079e05585689bebd9ff9b32e31dd5e1017" ], - "version": "==0.9.1" + "version": "==1.0.0" }, "factory-boy": { "hashes": [ @@ -1744,11 +1781,11 @@ }, "faker": { "hashes": [ - "sha256:172e45220b7a46743f4fb58cf380adb306d5c3ab1c0b0d97062508474cec5ff8", - "sha256:7c3f8ee807d3916415568169a172bf0893ea9cc3371ab55e4e5f5170d2185bea" + "sha256:067a03f64e555261610e69277536072997b4576dbf84b113faef3c06d85b466b", + "sha256:0e00bfa1eadf1493f15662edb181222fea4847764cf3f9ff3e66ee0f95c9a644" ], - "markers": "python_version >= '3.6'", - "version": "==13.15.1" + "markers": "python_full_version >= '3.6.0'", + "version": "==14.1.0" }, "flake8": { "hashes": [ @@ -1768,11 +1805,11 @@ }, "hypothesis": { "hashes": [ - "sha256:b60b59d36e76626a38cf893d70d0c3c39c5fa02c6c5895966ff5fead8fb9ff0a", - "sha256:de63c34309181875e71d0f5d1c1051c9320a1fe0517ea6733af8cedf818191f4" + "sha256:469125235b4fdd6b6a4b6a6fa13e528d3e8de4dbd3bd9236b03323fefa3a9b32", + "sha256:ee42fe4d2ff96c49910085780d6b8f34cbcf4c44427616e22833869d451116bb" ], "index": "pypi", - "version": "==6.54.1" + "version": "==6.54.4" }, "idna": { "hashes": [ @@ -1833,7 +1870,7 @@ "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" ], - "markers": "python_version < '4' and python_full_version >= '3.6.1'", + "markers": "python_version < '4.0' and python_full_version >= '3.6.1'", "version": "==5.10.1" }, "jedi": { @@ -1841,7 +1878,7 @@ "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d", "sha256:74137626a64a99c8eb6ae5832d99b3bdd7d29a3850fe2aa80a4126b2a7d949ab" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==0.18.1" }, "jinja2": { @@ -1892,7 +1929,7 @@ "sha256:fccdf7c2c5821a8cbd0a9440a456f5050492f2270bd54e94360cac663398739b", "sha256:fd45683c3caddf83abbb1249b653a266e7069a09f486daa8863fb0e7496a9fdb" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.7.1" }, "livereload": { @@ -1957,18 +1994,18 @@ }, "matplotlib-inline": { "hashes": [ - "sha256:a04bfba22e0d1395479f866853ec1ee28eea1485c1d69a6faf00dc3e24ff34ee", - "sha256:aed605ba3b72462d64d475a21a9296f400a19c4f74a31b59103d2a99ffd5aa5c" + "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311", + "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304" ], "markers": "python_version >= '3.5'", - "version": "==0.1.3" + "version": "==0.1.6" }, "mccabe": { "hashes": [ "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==0.7.0" }, "mdit-py-plugins": { @@ -1981,11 +2018,11 @@ }, "mdurl": { "hashes": [ - "sha256:6a8f6804087b7128040b2fb2ebe242bdc2affaeaa034d5fc9feeed30b443651b", - "sha256:f79c9709944df218a4cdb0fcc0b0c7ead2f44594e3e84dc566606f04ad749c20" + "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", + "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" ], "markers": "python_version >= '3.7'", - "version": "==0.1.1" + "version": "==0.1.2" }, "mypy-extensions": { "hashes": [ @@ -2015,7 +2052,7 @@ "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0", "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==0.8.3" }, "pathspec": { @@ -2053,7 +2090,7 @@ "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.0.0" }, "prompt-toolkit": { @@ -2091,7 +2128,7 @@ "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785", "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.9.1" }, "pyflakes": { @@ -2099,24 +2136,24 @@ "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2", "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.5.0" }, "pygments": { "hashes": [ - "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb", - "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519" + "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1", + "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42" ], "markers": "python_version >= '3.6'", - "version": "==2.12.0" + "version": "==2.13.0" }, "pylint": { "hashes": [ - "sha256:487ce2192eee48211269a0e976421f334cf94de1806ca9d0a99449adcdf0285e", - "sha256:fabe30000de7d07636d2e82c9a518ad5ad7908590fe135ace169b44839c15f90" + "sha256:4b124affc198b7f7c9b5f9ab690d85db48282a025ef9333f51d2d7281b92a6c3", + "sha256:4f3f7e869646b0bd63b3dfb79f3c0f28fc3d2d923ea220d52620fd625aed92b0" ], "index": "pypi", - "version": "==2.14.5" + "version": "==2.15.0" }, "pylint-django": { "hashes": [ @@ -2179,7 +2216,7 @@ "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e", "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.4.0" }, "pytest-mock": { @@ -2208,10 +2245,10 @@ }, "pytz": { "hashes": [ - "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", - "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197", + "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5" ], - "version": "==2022.1" + "version": "==2022.2.1" }, "pyyaml": { "hashes": [ @@ -2295,7 +2332,7 @@ "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759", "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.3.2.post1" }, "sphinx": { @@ -2343,7 +2380,7 @@ "sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07", "sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.0.0" }, "sphinxcontrib-jsmath": { @@ -2380,10 +2417,10 @@ }, "stack-data": { "hashes": [ - "sha256:77bec1402dcd0987e9022326473fdbcc767304892a533ed8c29888dacb7dddbc", - "sha256:aa1d52d14d09c7a9a12bb740e6bdfffe0f5e8f4f9218d85e7c73a8c37f7ae38d" + "sha256:66d2ebd3d7f29047612ead465b6cae5371006a71f45037c7e2507d01367bce3b", + "sha256:715c8855fbf5c43587b141e46cc9d9339cc0d1f8d6e0f98ed0d01c6cb974e29f" ], - "version": "==0.3.0" + "version": "==0.5.0" }, "toml": { "hashes": [ @@ -2402,11 +2439,11 @@ }, "tomlkit": { "hashes": [ - "sha256:1c5bebdf19d5051e2e1de6cf70adfc5948d47221f097fcff7a3ffc91e953eaf5", - "sha256:61901f81ff4017951119cd0d1ed9b7af31c821d6845c8c477587bbdcd5e5854e" + "sha256:25d4e2e446c453be6360c67ddfb88838cfc42026322770ba13d1fbd403a93a5c", + "sha256:3235a9010fae54323e727c3ac06fb720752fe6635b3426e379daec60fbd44a83" ], - "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==0.11.1" + "markers": "python_version < '4.0' and python_full_version >= '3.6.0'", + "version": "==0.11.4" }, "tornado": { "hashes": [ @@ -2438,16 +2475,16 @@ "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6" ], - "markers": "python_version >= '3.7'", + "markers": "python_version < '3.9'", "version": "==4.3.0" }, "urllib3": { "hashes": [ - "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", - "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" + "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", + "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", - "version": "==1.26.11" + "markers": "python_version >= '3.5'", + "version": "==1.26.12" }, "wcwidth": { "hashes": [ diff --git a/apps/accounts/admin.py b/apps/accounts/admin.py index b7bfee2a..1e51ad0d 100644 --- a/apps/accounts/admin.py +++ b/apps/accounts/admin.py @@ -5,6 +5,7 @@ from django.contrib import admin from django.urls import reverse from django.contrib.auth.admin import UserAdmin, GroupAdmin, Group +import django.forms as dj_forms from django.contrib.auth.forms import UserCreationForm from django.utils.safestring import mark_safe from django.shortcuts import redirect, render @@ -24,7 +25,7 @@ ActionListFilter, UserListFilter, ) - +import rich from taggit.models import Tag import logging @@ -41,6 +42,8 @@ from apps.greencheck.models import GreencheckASNapprove from apps.greencheck.choices import StatusApproval +from apps.greencheck.forms import ImporterCSVForm + from .utils import get_admin_name, reverse_admin_name from .admin_site import greenweb_admin @@ -290,7 +293,7 @@ class HostingAdmin(admin.ModelAdmin): ] # these are not really fields, but buttons # see the corresponding methods - readonly_fields = ["preview_email_button"] + readonly_fields = ["preview_email_button", "start_csv_import_button"] ordering = ("name",) # Factories @@ -357,6 +360,79 @@ def preview_email(self, request, *args, **kwargs): return render(request, "preview_email.html", context) + def start_import_from_csv(self, request, *args, **kwargs): + """ + Show the form, and preview required formate for the importer + for the given hosting provider. + """ + + # get our provider + provider = Hostingprovider.objects.get(pk=kwargs["provider"]) + + # get our document + data = {"provider": provider.id} + form = ImporterCSVForm(data) + form.fields["provider"].widget = dj_forms.widgets.HiddenInput() + + return render( + request, + "import_csv_start.html", + {"form": form, "ip_ranges": [], "provider": provider}, + ) + + def save_import_from_csv(self, request, *args, **kwargs): + """ + Process the contents of the uploaded file, and either + show a preview of the IP ranges that would be created, or + create them, based on submitted form value + """ + provider = Hostingprovider.objects.get(pk=kwargs["provider"]) + + if request.method == "POST": + # get our provider + data = {"provider": provider.id} + + # try to get our document + form = ImporterCSVForm(request.POST, request.FILES) + form.fields["provider"].widget = dj_forms.widgets.HiddenInput() + + valid = form.is_valid() + skip_preview = form.cleaned_data["skip_preview"] + + if valid and skip_preview: + # not doing preview. Run the import + completed_importer = form.save() + + context = { + "ip_ranges": completed_importer, + "provider": provider, + } + return render(request, "import_csv_results.html", context,) + + if valid: + # the save default we don't save the contents + # just showing what would happen + ip_ranges = form.get_ip_ranges() + context = { + "form": form, + "ip_ranges": ip_ranges, + "provider": provider, + } + return render(request, "import_csv_preview.html", context,) + + # otherwise fallback to showing the form with errors, + # ready for another attempted submission + + context = { + "form": form, + "ip_ranges": None, + "provider": provider, + } + + return render(request, "import_csv_preview.html", context,) + + return redirect("greenweb_admin:accounts_hostingprovider_change", provider.id) + def send_email(self, request, *args, **kwargs): """ Send the given email, log the outbound request in the admin, and @@ -537,6 +613,16 @@ def get_urls(self): self.send_email, name=get_admin_name(self.model, "send_email"), ), + path( + "/start_import_from_csv", + self.start_import_from_csv, + name=get_admin_name(self.model, "start_import_from_csv"), + ), + path( + "/save_import_from_csv", + self.save_import_from_csv, + name=get_admin_name(self.model, "save_import_from_csv"), + ), path( "/preview_email", self.preview_email, @@ -565,7 +651,11 @@ def get_fieldsets(self, request, obj=None): fieldset = [ ( "Hostingprovider info", - {"fields": (("name", "website",), "country", "services")}, + { + "fields": ( + ("name", "website",), "country", "services", + ) + }, ) ] @@ -577,6 +667,7 @@ def get_fieldsets(self, request, obj=None): ("partner", "model"), ("staff_labels",), ("email_template", "preview_email_button"), + ("start_csv_import_button"), ) }, ) @@ -658,6 +749,20 @@ def preview_email_button(self, obj): preview_email_button.short_description = "Support Messages" + @mark_safe + def start_csv_import_button(self, obj): + """ + Create clickable link to begin process of bulk import + of IP ranges. + """ + url = reverse_admin_name( + Hostingprovider, name="start_import_from_csv", kwargs={"provider": obj.pk}, + ) + link = f'Import IP Ranges from CSV' + return link + + send_button.short_description = "Import IP Ranges from a CSV file" + @mark_safe def html_website(self, obj): html = f'{obj.website}' diff --git a/apps/accounts/admin_site.py b/apps/accounts/admin_site.py index d13d9cd7..de54e0ea 100644 --- a/apps/accounts/admin_site.py +++ b/apps/accounts/admin_site.py @@ -139,6 +139,7 @@ def get_urls(self): patterns = [ path("try_out/", CheckUrlView.as_view(), name="check_url"), path("green-urls", GreenUrlsView.as_view(), name="green_urls"), + path("import-ip-ranges", GreenUrlsView.as_view(), name="import_ip_ranges"), ] return patterns + urls diff --git a/apps/accounts/templates/import_csv_preview.html b/apps/accounts/templates/import_csv_preview.html new file mode 100644 index 00000000..929f9b64 --- /dev/null +++ b/apps/accounts/templates/import_csv_preview.html @@ -0,0 +1,85 @@ +{% extends "admin/base_site.html" %} +{% load i18n static %} + +{% block pretitle %} +

Import IP Ranges from a CSV file for {{ provider }}

+ + +{% endblock %} + +{% block content %} + +{% if ip_ranges %} + +
+ +

IP Range Import Preview

+ +

The following IP ranges would be imported for {{ provider }}:

+ + + + +{% for ip in ip_ranges.green_ips %} + + + + + + + + +{% endfor %} +
IP Range startIp Range EndCreated / UpdatedLength
{{ ip.ip_start }}{{ ip.ip_end }} + {% if ip.id %} + Updated + {% else %} + Created + {% endif %} + + {{ ip.ip_range_length }} +
+ +

AS Import Preview

+ +

The following AS Numbers ranges would be imported for {{ provider }}:

+ + + + +{% for as in ip_ranges.green_asns %} + + + + + + +{% endfor %} +
AS numberCreated / Updated
{{ as.asn }} + {% if as.id %} + Updated + {% else %} + Created + {% endif %} +
+ +{% endif %} + +
+ +
+ {% csrf_token %} + {{ form.as_p }} + + + + + Back to import start + + Back to hosting provider +
+ + + +{% endblock content %} diff --git a/apps/accounts/templates/import_csv_results.html b/apps/accounts/templates/import_csv_results.html new file mode 100644 index 00000000..ec259fde --- /dev/null +++ b/apps/accounts/templates/import_csv_results.html @@ -0,0 +1,86 @@ +{% extends "admin/base_site.html" %} +{% load i18n static %} + +{% block pretitle %} +

CSV Import Results for {{ provider }}

+{% endblock %} + +{% block content %} + + +{% if ip_ranges %} + +
+ +

IP Range Import

+ +

The following IP ranges were imported for {{ provider }}:

+ + + + +{% for ip in ip_ranges.green_ips %} + + + + + + + + +{% endfor %} +
IP Range startIp Range EndCreated / UpdatedLength
{{ ip.ip_start }}{{ ip.ip_end }} + {% if ip.id %} + Updated + {% else %} + Created + {% endif %} + + {{ ip.ip_range_length }} +
+ +

AS Import

+ +

The following AS Numbers ranges were imported for {{ provider }}:

+ + + + +{% for as in ip_ranges.green_asns %} + + + + + + +{% endfor %} +
AS numberCreated / Updated
{{ as.asn }} + {% if as.id %} + Updated + {% else %} + Created + {% endif %} +
+ +{% endif %} + +
+ + + +
+ Back to hosting provider + + | + + Make another CSV Import + + +{% endblock content %} diff --git a/apps/accounts/templates/import_csv_start.html b/apps/accounts/templates/import_csv_start.html new file mode 100644 index 00000000..e5052f9b --- /dev/null +++ b/apps/accounts/templates/import_csv_start.html @@ -0,0 +1,118 @@ +{% extends "admin/base_site.html" %} +{% load i18n static %} + +{% block pretitle %} +

Import IP Ranges from a CSV file for {{ provider }}

+ +
+Formatting notes for uploading CSVs + +
+

If you have a large number of CSV files, you can import them in bulk.

+ +

Make sure the form follows the sample format, without the header row

+ + + + + + + + + + + + + + + + + +
IP Range startIp Range End
104.21.2.197104.21.2.199
104.21.2.0/24
AS12345
+ +

If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.

+

So, uploading a CSV file with a single column IP addresses would be the equivalent to:

+ + + + + + + + + +
IP Range startIp Range End
104.21.2.197104.21.2.197
+ +

If you have an an IP range to import, add it all in the first column. It will be expanded into a full IP range

+ + + + + + + +
IP Range startIp Range End
104.21.2.0/24
+ +

Likewise with AS numbers. Do not as a space between 'AS' and the number itself.

+ + + + + + + +
IP Range startIp Range End
AS12345
+
+ +
+ + +{% endblock %} + + +{% block content %} + +
+ +
+ {% csrf_token %} + {{ form.as_p }} + + Back to hosting provider + +
+ + + +{% if ip_ranges %} + +
+ +

IP Range Import Preview

+ +

The following IP ranges would be imported for {{ provider }}:

+ + + + +{% for ip in ip_ranges %} + + + + + + + +{% endfor %} +
IP Range startIp Range EndCreated / Updated
{{ ip.ip_start }}{{ ip.ip_end }} + {% if ip.id %} + Updated + {% else %} + Created + {% endif %} +
+ +{% endif %} + +{% endblock content %} diff --git a/apps/accounts/templates/import_ip_ranges.html b/apps/accounts/templates/import_ip_ranges.html new file mode 100644 index 00000000..1290fb6a --- /dev/null +++ b/apps/accounts/templates/import_ip_ranges.html @@ -0,0 +1,86 @@ +{% extends "admin/base_site.html" %} +{% load i18n static %} + +{% block title %} + The Green Web Foundation Member portal: import ip ranges {{ domain }} +{% endblock %} + + +{% block pretitle %} +

Try out the greencheck with a URL of your choice

+{% endblock %} + diff --git a/apps/greencheck/fixtures/test_dataset_csv.csv b/apps/greencheck/fixtures/test_dataset_csv.csv new file mode 100644 index 00000000..e45d4af7 --- /dev/null +++ b/apps/greencheck/fixtures/test_dataset_csv.csv @@ -0,0 +1,6 @@ +104.21.2.197, 104.21.2.199 +104.21.2.0/24 +104.21.2.192/24 +AS234 +AS + diff --git a/apps/greencheck/forms.py b/apps/greencheck/forms.py index 864718a7..1d759275 100644 --- a/apps/greencheck/forms.py +++ b/apps/greencheck/forms.py @@ -1,4 +1,9 @@ +from django.db.models.fields import CharField +from django.forms.fields import BooleanField, FileField, IntegerField +from apps.accounts.models.hosting import ProviderLabel import logging +import rich +import io from django import forms from django.forms import ModelForm @@ -10,6 +15,8 @@ from .models import GreencheckIp from .models import GreencheckIpApprove from .models import GreencheckASN, GreencheckASNapprove +from .importers import CSVImporter +from ..accounts import models as ac_models User = get_user_model() @@ -101,6 +108,73 @@ def clean_is_staff(self): raise ValidationError("Alert staff: a bug has occurred.") +class ImporterCSVForm(forms.Form): + """ + A form for handling bulk IP range submissions in the django admin. + Uses the ImporterCSV class to handle imports + """ + + provider = forms.ModelChoiceField( + empty_label="Choose a Provider", + queryset=ac_models.Hostingprovider.objects.all(), + ) + csv_file = FileField(required=False) + skip_preview = BooleanField( + required=False, + help_text=("Do not show the preview of what would happen. Save to the import to the database"), + ) + # replace_with_import = BooleanField( + # required=False, + # label=("Replace networks with this import"), + # help_text=("Replace all the networks assigned to this hoster with the networks in this import"), + # ) + + ip_ranges = [] + processed_ips = [] + importer = None + + def initialize_importer(self): + """ + Clean our form, and return our importer with the + """ + + self.is_valid() + + uploaded_csv_string = self.cleaned_data["csv_file"].read().decode("utf-8") + csv_file = io.StringIO(uploaded_csv_string) + importer = CSVImporter(self.cleaned_data["provider"]) + + importable_values = importer.fetch_data_from_source(csv_file) + + self.ip_ranges = importable_values + + self.importer = importer + return importer + + def get_ip_ranges(self): + """ + Return a list of the IP Ranges, showing which ones would be updated, and + which ones would be created with this submission. + """ + self.initialize_importer() + provider = self.cleaned_data["provider"] + # make preview of generated ips + ips = self.importer.preview(provider, self.ip_ranges) + return ips + + def save(self): + """Save our list of IP ranges to the database""" + if self.importer is None: + self.initialize_importer() + logger.info("Skipping preview, running import") + + provider = self.cleaned_data["provider"] + + self.importer.process_addresses(self.ip_ranges) + self.processed_ips = self.importer.preview(provider, self.ip_ranges) + return self.processed_ips + + class GreencheckAsnForm(ModelForm, ApprovalMixin): ApprovalModel = GreencheckASNapprove @@ -116,8 +190,7 @@ class Meta: ) def save(self, commit=True): - """ - """ + """ """ # Like the GreencheckIpForm, we non-staff user creates an ip, instead of saving self._save_approval() return super().save(commit=True) @@ -186,8 +259,7 @@ class Meta: fields = "__all__" def save(self, commit=True): - """ - """ + """ """ ip_instance = self.instance.greencheck_ip if commit is True: if ip_instance: diff --git a/apps/greencheck/importers/__init__.py b/apps/greencheck/importers/__init__.py new file mode 100644 index 00000000..235e23c8 --- /dev/null +++ b/apps/greencheck/importers/__init__.py @@ -0,0 +1 @@ +from .importer_csv import CSVImporter diff --git a/apps/greencheck/importers/importer_csv.py b/apps/greencheck/importers/importer_csv.py new file mode 100644 index 00000000..17e34260 --- /dev/null +++ b/apps/greencheck/importers/importer_csv.py @@ -0,0 +1,147 @@ +import requests +import logging +import pandas as pd +import ipdb +import re +import ipaddress +import rich +from typing import List +from apps.accounts.models.hosting import Hostingprovider + +from apps.greencheck.importers.importer_interface import BaseImporter, Importer +from apps.greencheck.models import GreencheckIp, GreencheckASN + +from django.conf import settings + +logger = logging.getLogger(__name__) + + +class CSVImporter(BaseImporter): + def __init__(self, provider: Hostingprovider): + self.hosting_provider = provider + self.processed_ips = [] + + def fetch_data_from_source(cls, filepath_or_buffer) -> List: + """ + Return a list of the valid values from the provided CSV + for importing + """ + raw_data = pd.read_csv(filepath_or_buffer, header=None) + return cls.parse_to_list(raw_data) + + def parse_to_list(self, raw_data: pd.DataFrame) -> List: + """ + Parse the provided pandas DataFrame, and return a flattened list + of ip ranges, or importable IP networks, or AS numbers + """ + rows = raw_data.values + imported_networks = {"asns": [], "ip_networks": [], "ip_ranges": []} + for row in rows: + + # just one column? it's probably an AS or a IP network + if pd.isnull(row[1]): + + # is it an AS number? + if row[0].startswith("AS"): + # split out the as number from the row, + # add check for people getting AS number + as_number = row[0].split("AS ")[0] + just_as_with_no_number = as_number.lower() == "as" + + if as_number and not just_as_with_no_number: + imported_networks["asns"].append(row[0]) + else: + # if it isn't an AS number it's probably an IP network + try: + ip_network = ipaddress.ip_network(row[0]) + imported_networks["ip_networks"].append(row[0]) + except Exception: + logger.warn( + f"Item {row[0]} was not an ip network. Not importing." + ) + else: + try: + first_ip, last_ip = row[0].strip(), row[1].strip() + ip_begin = ipaddress.ip_address(first_ip) + ip_end = ipaddress.ip_address(last_ip) + imported_networks["ip_ranges"].append((first_ip, last_ip)) + except Exception: + logger.warn( + f"Row {row} does not look like an IP address. Not importing" + ) + + flattened_network_list = [ + *imported_networks["asns"], + *imported_networks["ip_networks"], + *imported_networks["ip_ranges"], + ] + + return flattened_network_list + + def preview(self, provider: Hostingprovider, list_of_networks: List) -> List: + """ + Return a list of the GreencheckIPs that would be updated + or created based on the current provided file. + + Return a preview of the networks to import, suitable for displaying + in a webpage. + """ + + green_ips = [] + green_asns = [] + # try to find a GreenIP + for network in list_of_networks: + + if self.is_as_number(network): + # this looks like an AS number + try: + as_number = network.split("AS")[1] + green_asn = GreencheckASN.objects.get( + asn=as_number, active=True, hostingprovider=provider + ) + green_asns.append(green_asn) + except GreencheckASN.DoesNotExist: + green_asn = GreencheckASN( + active=True, asn=as_number, hostingprovider=provider + ) + green_asns.append(green_asn) + + if ip_network := self.is_ip_network(network): + try: + green_ip = GreencheckIp.objects.get( + active=True, + ip_start=ip_network[1], + ip_end=ip_network[-1], + hostingprovider=provider, + ) + green_ips.append(green_ip) + + except GreencheckIp.DoesNotExist: + green_ip = GreencheckIp( + active=True, + ip_start=ip_network[1], + ip_end=ip_network[-1], + hostingprovider=provider, + ) + green_ips.append(green_ip) + + if ip_range := self.is_ip_range(network): + try: + green_ip = GreencheckIp.objects.get( + active=True, + hostingprovider=provider, + ip_start=ip_range[0], + ip_end=ip_range[1], + ) + green_ips.append(green_ip) + except GreencheckIp.DoesNotExist: + green_ip = GreencheckIp( + active=True, + hostingprovider=provider, + ip_start=ip_range[0], + ip_end=ip_range[1], + ) + green_ips.append(green_ip) + + # or make a new one, in memory + return {"green_ips": green_ips, "green_asns": green_asns} diff --git a/apps/greencheck/importers/importer_interface.py b/apps/greencheck/importers/importer_interface.py index 74cacab2..2474ae86 100644 --- a/apps/greencheck/importers/importer_interface.py +++ b/apps/greencheck/importers/importer_interface.py @@ -1,62 +1,145 @@ import ipaddress import re import logging +import rich -from typing import Protocol, runtime_checkable +from typing import Protocol, runtime_checkable, Union from apps.greencheck.models import GreencheckIp, GreencheckASN from apps.accounts.models import Hostingprovider logger = logging.getLogger(__name__) + @runtime_checkable class Importer(Protocol): - def fetch_data_from_source(cls) -> list: + def fetch_data_from_source(self) -> list: + """ + Fetches the data, and returns a data structure for parsing + with `parse_to_list` + """ raise NotImplementedError - def parse_to_list(cls) -> list: + def parse_to_list(self) -> list: + """ + Returns a list of either strings that can be parsed as AS numbers + or IP networks + """ raise NotImplementedError -class BaseImporter(): - hosting_provider_id:int - def process_addresses(cls, list_of_addresses: list): +class BaseImporter: + hosting_provider: Hostingprovider + def __init__(self, provider): + self.hosting_provider = provider + + + def is_ip_range(self, address: Union[str, tuple]) -> Union[tuple, bool]: + """ + Check if "address" is a usable ip range, and return the tuple containing + the required ip addresses, ready for importing as an ip range + """ + # return early if not a tuple we can parse + if not isinstance(address, tuple): + return + + first_ip = ipaddress.ip_address(address[0]) + last_ip = ipaddress.ip_address(address[-1]) + if first_ip and last_ip: + return address + + def is_ip_network(self, address: Union[str, tuple]): + """ + Chck that "address" is a string we can parse to an ip network, + ready for saving as an ip range. + """ + # return early if not a string we can parse + if not isinstance(address, str): + return + + # this looks an AS number return early + if address.startswith("AS"): + return + + try: + network = ipaddress.ip_network(address) + except ValueError: + logger.exception( + ( + f"Value for address: {address} has an invalid structure. " + "Must be IPv4 or IPv6 with subnetmask (101.102.103.104/27) " + "or AS number (AS123456)." + ) + ) + return + + if network: + return network + + def is_as_number(self, address: Union[str, tuple]) -> Union[str, bool]: + """ + Check that "address" is a string suitable for saving as a AS number + """ + # return early if not a string we can parse + if not isinstance(address, str): + return + + if address.startswith("AS"): + return address + + def process_addresses(self, list_of_addresses: list): count_asn = 0 count_ip = 0 - - # Determine the type of address (IPv4, IPv6 or ASN)for address in list_of_addresses: + + # Determine the type of address (IPv4, IPv6 or ASN) for + # address in list_of_addresses: try: for address in list_of_addresses: - if re.search("(AS)[0-9]+$", address): - # Address is ASN - cls.save_asn(cls, address) - count_asn += 1 - elif isinstance( - ipaddress.ip_network(address), - (ipaddress.IPv4Network, ipaddress.IPv6Network), - ): - # Address is IPv4 or IPv6 - cls.save_ip(cls, address) + + if self.is_ip_range(address): + # address looks like an IPv4 or IPv6 range + self.save_ip(address) + count_ip += 1 + continue + + if self.is_ip_network(address): + # address looks like IPv4 or IPv6 network + network = ipaddress.ip_network(address) + self.save_ip((network[1], network[-1])) count_ip += 1 - - return f"Processing complete. Added {count_asn} ASN's and {count_ip} IP's (either IPv4 and/or IPv6)" + continue + + if self.is_as_number(address): + # address is ASN + self.save_asn(address) + count_asn += 1 + continue + + return ( + f"Processing complete. Added {count_asn} ASN's and {count_ip} IP " + "ranges (either IPv4 and/or IPv6)" + ) except ValueError: - logger.exception( - "Value has invalid structure. Must be IPv4 or IPv6 with subnetmask (101.102.103.104/27) or AS number (AS123456)." + return ( + f"An error occurred while adding new entries. Added {count_asn} AS " + f"numbers and {count_ip} ip ranges (either IPv4 and/or IPv6)" ) - return f"An error occurred while adding new entries. Added {count_asn} ASN's and {count_ip} IP's (either IPv4 and/or IPv6)" except Exception as e: logger.exception("Something really unexpected happened. Aborting") logger.exception(e) - return f"An error occurred while adding new entries. Added {count_asn} ASN's and {count_ip} IP's (either IPv4 and/or IPv6)" + return ( + "An error occurred while adding new entries. " + f"Added {count_asn} ASN's and {count_ip} IP's " + "(either IPv4 and/or IPv6)" + ) - def save_asn(cls, address: str): - hoster = Hostingprovider.objects.get( - pk=cls.hosting_provider_id - ) + def save_asn(self, address: str): + gc_asn, created = GreencheckASN.objects.update_or_create( - active=True, asn=int(address.replace("AS", "")), hostingprovider=hoster + active=True, + asn=int(address.replace("AS", "")), + hostingprovider=self.hosting_provider ) gc_asn.save() # Save the newly created or updated object @@ -65,19 +148,25 @@ def save_asn(cls, address: str): logger.debug(gc_asn) return gc_asn - def save_ip(cls, address: str): - # Convert to IPv4 network with it's respective range - network = ipaddress.ip_network(address) - hoster = Hostingprovider.objects.get( - pk=cls.hosting_provider_id - ) + def save_ip(self, address): + if isinstance(address, tuple): + start_address = ipaddress.ip_address(address[0]) + ending_address = ipaddress.ip_address(address[1]) + elif isinstance(address, str): + network = ipaddress.ip_network(address) + + start_address = network[1] + ending_address = network[-1] gc_ip, created = GreencheckIp.objects.update_or_create( - active=True, ip_start=network[1], ip_end=network[-1], hostingprovider=hoster + active=True, + ip_start=start_address, + ip_end=ending_address, + hostingprovider=self.hosting_provider, ) gc_ip.save() # Save the newly created or updated object if created: # Only log and return when a new object was created logger.debug(gc_ip) - return gc_ip \ No newline at end of file + return gc_ip diff --git a/apps/greencheck/tests/test_base_importer.py b/apps/greencheck/tests/test_base_importer.py index 4e6d628f..21ba26e9 100644 --- a/apps/greencheck/tests/test_base_importer.py +++ b/apps/greencheck/tests/test_base_importer.py @@ -33,13 +33,12 @@ def hosting_provider(): @pytest.fixture -def base_importer(): +def base_importer(hosting_provider: Hostingprovider): """ Initialize a BaseImporter object Return: BaseImporter """ - importer = BaseImporter - importer.hosting_provider_id = 1234 + importer = BaseImporter(hosting_provider) return importer @@ -63,8 +62,13 @@ def test_save_ip(self, hosting_provider, base_importer): """ Test saving IPv4 and IPv6 networks to the database """ - testing_ipv4 = "191.233.8.24/29" - testing_ipv6 = "2603:1010:304::140/123" + testing_ipv4_range = ("191.233.8.25", "191.233.8.30") + testing_ipv6_range = ( + "2603:1010:0304:0000:0000:0000:0000:0140", + "2603:1010:0304:0000:0000:0000:0000:015f", + ) + testing_ipv4_network = "191.233.8.24/29" + testing_ipv6_network = "2603:1010:304::140/123" assert ( GreencheckIp.objects.all().count() == 0 @@ -72,17 +76,33 @@ def test_save_ip(self, hosting_provider, base_importer): hosting_provider.save() # Initialize hosting provider in database # Import a single IPv4 network - BaseImporter.save_ip(base_importer, testing_ipv4) + base_importer.save_ip(testing_ipv4_range) assert ( GreencheckIp.objects.all().count() == 1 + ) # Test: database is empty (for IP) + hosting_provider.save() # Initialize hosting provider in database + + # Import a single IPv4 network + base_importer.save_ip(testing_ipv6_range) + + assert ( + GreencheckIp.objects.all().count() == 2 + ) # Test: database is empty (for IP) + hosting_provider.save() # Initialize hosting provider in database + + # Import a single IPv4 network + base_importer.save_ip(testing_ipv4_network) + + assert ( + GreencheckIp.objects.all().count() == 3 ) # Test: IPv4 is saved after insertion # Import a single IPv6 network - BaseImporter.save_ip(base_importer, testing_ipv6) + base_importer.save_ip(testing_ipv6_network) assert ( - GreencheckIp.objects.all().count() == 2 + GreencheckIp.objects.all().count() == 4 ) # Test: IPv6 is saved after insertion def test_save_asn(self, hosting_provider, base_importer): @@ -117,7 +137,8 @@ def test_process_addresses(self, hosting_provider, base_importer, sample_data): hosting_provider.save() # Initialize hosting provider in database # Process list of addresses in JSON file - BaseImporter.process_addresses(base_importer, sample_data) + + base_importer.process_addresses(sample_data) assert ( GreencheckIp.objects.all().count() == 63 diff --git a/apps/greencheck/tests/test_importer_csv.py b/apps/greencheck/tests/test_importer_csv.py new file mode 100644 index 00000000..bed0e697 --- /dev/null +++ b/apps/greencheck/tests/test_importer_csv.py @@ -0,0 +1,168 @@ +import pytest +import pathlib +import re +import pandas as pd +import ipdb +from io import StringIO + +from django.core.management import call_command +from apps.accounts.models.hosting import Hostingprovider +from apps.greencheck.importers.importer_csv import CSVImporter + +from django.conf import settings + + +@pytest.fixture +def sample_data_raw(): + """ + Retrieve a locally saved sample of the population to use for this test + Return: CSV + """ + csv_path = pathlib.Path(settings.ROOT) / "apps" / "greencheck" / "fixtures" / "test_dataset_csv.csv" + return pd.read_csv(csv_path, header=None) + + +@pytest.fixture +def sample_data_as_list(sample_data_raw, hosting_provider: Hostingprovider): + """ + Retrieve a locally saved sample of the population to use for this test and parse it to a list + Return: List + """ + importer = CSVImporter() + return importer.parse_to_list(sample_data_raw) + + +@pytest.mark.django_db +class TestCSVImporter: + def test_parse_to_list(self, sample_data_raw, hosting_provider: Hostingprovider): + """ + Test the parsing function. + """ + # Initialize Csv importer + hosting_provider.save() + importer = CSVImporter(hosting_provider) + + # Run parse list with sample data + list_of_addresses = importer.parse_to_list(sample_data_raw) + + # Test: resulting list contains some items + assert len(list_of_addresses) > 0 + + # do we have an ip network? + assert "104.21.2.0/24" in list_of_addresses + + # have we filtered out our incorrect IP network? + assert "104.21.2.192/24" not in list_of_addresses + + # do we have our expected AS number? + assert "AS234" in list_of_addresses + + # have we filtered out our bad AS line? + assert "AS" not in list_of_addresses + + # do we have an IP range in our list + expected_ip_range = ("104.21.2.197", "104.21.2.199") + assert expected_ip_range in list_of_addresses + + def test_process_imports(self, sample_data_raw, hosting_provider: Hostingprovider): + + # Initialize Csv importer + hosting_provider.save() + importer = CSVImporter(hosting_provider) + + # Run parse list with sample data + list_of_addresses = importer.parse_to_list(sample_data_raw) + created_networks = importer.process_addresses(list_of_addresses) + + # we should have seen one AS network added + assert "1 ASN" in created_networks + # have we created two new IP ranges? + assert "2 IP" in created_networks + + # have we created the new Green ASN in the db? + green_asns = hosting_provider.greencheckasn_set.all() + assert green_asns .first().asn == 234 + + # have we created the green ip ranges in the db? + green_ips = hosting_provider.greencheckip_set.all().order_by('ip_start') + + # have we converted a network to a range? + assert green_ips[0].ip_start == "104.21.2.1" + assert green_ips[0].ip_end == "104.21.2.255" + + # have do we have the range added as well? + assert green_ips[1].ip_start == "104.21.2.197" + assert green_ips[1].ip_end == "104.21.2.199" + + def test_preview_imports(self, sample_data_raw, hosting_provider: Hostingprovider): + """ + Can we see a representation of the data we would import before + we run the import it? + """ + # Initialize Csv importer + hosting_provider.save() + importer = CSVImporter(hosting_provider) + + # Run parse list with sample data + list_of_addresses = importer.parse_to_list(sample_data_raw) + preview = importer.preview(hosting_provider, list_of_addresses) + + assert len(preview["green_ips"]) == 2 + assert len(preview["green_asns"]) == 1 + + def test_view_processed_imports(self, sample_data_raw, hosting_provider: Hostingprovider): + """ + Can we compare the state of an import to the networks already in the database + for this provider? + """ + # Initialize Csv importer + hosting_provider.save() + importer = CSVImporter(hosting_provider) + + # Run parse list with sample data + list_of_addresses = importer.parse_to_list(sample_data_raw) + # Run our import to save them to the database, simulating saving + # via our the form + created_networks = importer.process_addresses(list_of_addresses) + # Generate a view of the data, to check if we are fetching from + # the database now + preview = importer.preview(hosting_provider, list_of_addresses) + + green_ips = [gip for gip in preview['green_ips']] + green_asns = [gip for gip in preview['green_asns']] + + # are these the IPs checked against the database? + for green_ip in green_ips: + assert green_ip.id is not None + + # are these the ASNs checked against the database? + for green_asn in green_asns: + assert green_asn.id is not None + + +# @pytest.mark.django_db +# class TestCsvImportCommand: +# """ +# This just tests that we have a management command that can run. +# """ + +# def test_handle(self, mocker, sample_data_as_list): +# # mock the call to retrieve from source, to a locally stored +# # testing sample. By instead using the test sample, +# # we avoid unnecessary network requests. + +# # identify method we want to mock +# path_to_mock = ( +# "apps.greencheck.importers.importer_csv." +# "CSVImporter.fetch_data_from_source" +# ) + +# # define a different return when the targeted mock +# # method is called +# mocker.patch( +# path_to_mock, +# return_value=sample_data_as_list, +# ) + +# # TODO: Do we need this call command? +# # call_command("update_networks_in_db_csv") diff --git a/docs/how-to.md b/docs/how-to.md index e8315475..c1ee6e6c 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -1,4 +1,25 @@ # How to use.. ## Sphinx -## Tests +## Tests using pytest + +pytest.ini is called before running a pytest. +This file specifies what django settings (ds) to use, which annotated to exclude using the mark (-m) keyword and other functions. + +#### Run all tests +Important: make sure to be outside of an enviroment (deactivate). +``` +./run-tests.sh +``` + +#### Run all test until one fails +``` +pipenv run pytest -x +``` + +## Gitpod environment set up steps +1. Make sure there is a branch available in the Github repository +2. Go to the workspace overview in Gitpod of TGWF +3. Run pre-build + Click on the pre-build option in the workspace overview of the workspace you want to prepare. +4. After this preperation, open the workspace and it's ready to be used \ No newline at end of file diff --git a/makefile b/makefile index a7ae704f..c68e343d 100644 --- a/makefile +++ b/makefile @@ -11,6 +11,12 @@ release: PIPENV_DOTENV_LOCATION=.env.prod pipenv run ansible-playbook ansible/deploy.yml -i ansible/inventories/prod.yml PIPENV_DOTENV_LOCATION=.env.prod pipenv run sentry-cli releases finalize $(shell sentry-cli releases propose-version) +dev.createsuperuser: + python ./manage.py createsuperuser --username admin --email admin@admin.commits --noinput + python ./manage.py set_fake_passwords + +dev.runserver: + python manage.py runserver dev.test: pytest -s --create-db --looponfail --ds=greenweb.settings.testing diff --git a/pytest.ini b/pytest.ini index 25c37836..8629049c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,7 +3,7 @@ ; so it can take precedent over any environment variables ; that might point DJANGO_SETTINGS_MODULE to development ; or production, instead of the testings module -addopts = --reuse-db --maxfail=0 -m "not smoke_test and not dramatiq and not flaky" --ds="greenweb.settings.testing" +addopts = --create-db --maxfail=0 -m "not smoke_test and not dramatiq and not flaky" --ds="greenweb.settings.testing" python_files = tests.py test_*.py *_tests.py markers = only: Convenience method, so we can run a focussed test in pytest-watch diff --git a/run-tests.sh b/run-tests.sh index 08efada0..3e9f63ab 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -pipenv run pytest \ No newline at end of file +pipenv run pytest \ No newline at end of file