Add CSV importer for hosting provider (#296)

* Improved and documented pytests * WIP: Reading and validating a csv file * Fixed function call and provided some todo's * WIP: Included bulk import files, creating a csv upload page * Expanded the base importer to allow ip ranges * WIP: Implemented an option to import csv files * Update tests so we have our failing examples * Add missing pipffile.lock * Get CSV importer parsing info as intended Add tests and refactor importer * Pass a type, not a value in the method signature * Add tests and code for import preview in admin * Add indication of ip range length before import * Update method signature for updating tests * Update form to use new CSV importer * Add __init__ file for CSV Importer * Update the importer to work with hosting provider objects not ids * Add tests to check that the importer saves to db * Update flow for import show imported networks * Comment out unimplemented "replace" feature * update importer to use sightly more idiomatic python * Move upload CSV option to admin section So only admins can see it and use it Co-authored-by: Roald Teunissen <[email protected]>
thegreenwebfoundation · Sep 2, 2022 · ef7a43a · ef7a43a
1 parent 95e0abc
commit ef7a43a
Show file tree

Hide file tree

Showing 19 changed files with 1,381 additions and 330 deletions.
diff --git a/Pipfile b/Pipfile
@@ -76,6 +76,8 @@ python-dateutil = "*"
 geoip2 = "*"
 iso3166 = "*"
 django-logentry-admin = "*"
+pandas = "*"
+rich = "*"
 
 [pipenv]
 # Needed for `black`. See https://github.com/microsoft/vscode-python/pull/5967.

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/apps/accounts/admin.py b/apps/accounts/admin.py
@@ -5,6 +5,7 @@
 from django.contrib import admin
 from django.urls import reverse
 from django.contrib.auth.admin import UserAdmin, GroupAdmin, Group
+import django.forms as dj_forms
 from django.contrib.auth.forms import UserCreationForm
 from django.utils.safestring import mark_safe
 from django.shortcuts import redirect, render
@@ -24,7 +25,7 @@
     ActionListFilter,
     UserListFilter,
 )
-
+import rich
 
 from taggit.models import Tag
 import logging
@@ -41,6 +42,8 @@
 from apps.greencheck.models import GreencheckASNapprove
 from apps.greencheck.choices import StatusApproval
 
+from apps.greencheck.forms import ImporterCSVForm
+
 
 from .utils import get_admin_name, reverse_admin_name
 from .admin_site import greenweb_admin
@@ -290,7 +293,7 @@ class HostingAdmin(admin.ModelAdmin):
     ]
     # these are not really fields, but buttons
     # see the corresponding methods
-    readonly_fields = ["preview_email_button"]
+    readonly_fields = ["preview_email_button", "start_csv_import_button"]
     ordering = ("name",)
 
     # Factories
@@ -357,6 +360,79 @@ def preview_email(self, request, *args, **kwargs):
 
         return render(request, "preview_email.html", context)
 
+    def start_import_from_csv(self, request, *args, **kwargs):
+        """
+        Show the form, and preview required formate for the importer
+        for the given hosting provider.
+        """
+
+        # get our provider
+        provider = Hostingprovider.objects.get(pk=kwargs["provider"])
+
+        # get our document
+        data = {"provider": provider.id}
+        form = ImporterCSVForm(data)
+        form.fields["provider"].widget = dj_forms.widgets.HiddenInput()
+
+        return render(
+            request,
+            "import_csv_start.html",
+            {"form": form, "ip_ranges": [], "provider": provider},
+        )
+
+    def save_import_from_csv(self, request, *args, **kwargs):
+        """
+        Process the contents of the uploaded file, and either
+        show a preview of the IP ranges that would be created, or
+        create them, based on submitted form value
+        """
+        provider = Hostingprovider.objects.get(pk=kwargs["provider"])
+
+        if request.method == "POST":
+            # get our provider
+            data = {"provider": provider.id}
+
+            # try to get our document
+            form = ImporterCSVForm(request.POST, request.FILES)
+            form.fields["provider"].widget = dj_forms.widgets.HiddenInput()
+
+            valid = form.is_valid()
+            skip_preview = form.cleaned_data["skip_preview"]
+
+            if valid and skip_preview:
+                # not doing preview. Run the import
+                completed_importer = form.save()
+
+                context = {
+                    "ip_ranges": completed_importer,
+                    "provider": provider,
+                }
+                return render(request, "import_csv_results.html", context,)
+
+            if valid:
+                # the save default we don't save the contents
+                # just showing what would happen
+                ip_ranges = form.get_ip_ranges()
+                context = {
+                    "form": form,
+                    "ip_ranges": ip_ranges,
+                    "provider": provider,
+                }
+                return render(request, "import_csv_preview.html", context,)
+
+            # otherwise fallback to showing the form with errors,
+            # ready for another attempted submission
+
+            context = {
+                "form": form,
+                "ip_ranges": None,
+                "provider": provider,
+            }
+
+            return render(request, "import_csv_preview.html", context,)
+
+        return redirect("greenweb_admin:accounts_hostingprovider_change", provider.id)
+
     def send_email(self, request, *args, **kwargs):
         """
         Send the given email, log the outbound request in the admin, and
@@ -537,6 +613,16 @@ def get_urls(self):
                 self.send_email,
                 name=get_admin_name(self.model, "send_email"),
             ),
+            path(
+                "<provider>/start_import_from_csv",
+                self.start_import_from_csv,
+                name=get_admin_name(self.model, "start_import_from_csv"),
+            ),
+            path(
+                "<provider>/save_import_from_csv",
+                self.save_import_from_csv,
+                name=get_admin_name(self.model, "save_import_from_csv"),
+            ),
             path(
                 "<provider>/preview_email",
                 self.preview_email,
@@ -565,7 +651,11 @@ def get_fieldsets(self, request, obj=None):
         fieldset = [
             (
                 "Hostingprovider info",
-                {"fields": (("name", "website",), "country", "services")},
+                {
+                    "fields": (
+                        ("name", "website",), "country", "services",
+                    )
+                },
             )
         ]
 
@@ -577,6 +667,7 @@ def get_fieldsets(self, request, obj=None):
                     ("partner", "model"),
                     ("staff_labels",),
                     ("email_template", "preview_email_button"),
+                    ("start_csv_import_button"),
                 )
             },
         )
@@ -658,6 +749,20 @@ def preview_email_button(self, obj):
 
     preview_email_button.short_description = "Support Messages"
 
+    @mark_safe
+    def start_csv_import_button(self, obj):
+        """
+        Create clickable link to begin process of bulk import
+        of IP ranges.
+        """
+        url = reverse_admin_name(
+            Hostingprovider, name="start_import_from_csv", kwargs={"provider": obj.pk},
+        )
+        link = f'<a href="{url}" class="start_csv_import">Import IP Ranges from CSV</a>'
+        return link
+
+    send_button.short_description = "Import IP Ranges from a CSV file"
+
     @mark_safe
     def html_website(self, obj):
         html = f'<a href="{obj.website}" target="_blank">{obj.website}</a>'

diff --git a/apps/accounts/admin_site.py b/apps/accounts/admin_site.py
@@ -139,6 +139,7 @@ def get_urls(self):
         patterns = [
             path("try_out/", CheckUrlView.as_view(), name="check_url"),
             path("green-urls", GreenUrlsView.as_view(), name="green_urls"),
+            path("import-ip-ranges", GreenUrlsView.as_view(), name="import_ip_ranges"),
         ]
         return patterns + urls
 

diff --git a/apps/accounts/templates/import_csv_preview.html b/apps/accounts/templates/import_csv_preview.html
@@ -0,0 +1,85 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
+
+
+{% endblock %}
+
+{% block content %}
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import Preview</h2>
+
+<p>The following IP ranges would be imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th><th>Length</th>
+
+{% for ip in ip_ranges.green_ips %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+    <td>
+      {{  ip.ip_range_length }}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+<h2>AS Import Preview</h2>
+
+<p>The following AS Numbers ranges would be imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>AS number</th><th>Created / Updated</th>
+
+{% for as in ip_ranges.green_asns %}
+
+  <tr>
+    <td>{{ as.asn }}</td>
+    <td>
+      {% if as.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<form enctype="multipart/form-data" action="{% url 'greenweb_admin:accounts_hostingprovider_save_import_from_csv' provider.id %}" method="post">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="Submit"
+        style="margin-top: 0px; padding: 6px 15px">
+
+
+
+      <a style="margin-left:1rem;margin-right:1rem;" href="{% url 'greenweb_admin:accounts_hostingprovider_start_import_from_csv' provider.id %}">Back to import start</a>
+
+      <a style="margin-left:1rem;margin-right:1rem;" class="" href="{% url 'greenweb_admin:accounts_hostingprovider_change' provider.id %}">Back to hosting provider</a>
+</form>
+
+
+
+{% endblock content %}
diff --git a/apps/accounts/templates/import_csv_results.html b/apps/accounts/templates/import_csv_results.html
@@ -0,0 +1,86 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>CSV Import Results for {{ provider }}</h1>
+{% endblock %}
+
+{% block content %}
+
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import </h2>
+
+<p>The following IP ranges were imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th><th>Length</th>
+
+{% for ip in ip_ranges.green_ips %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+    <td>
+      {{  ip.ip_range_length }}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+<h2>AS Import</h2>
+
+<p>The following AS Numbers ranges were imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>AS number</th><th>Created / Updated</th>
+
+{% for as in ip_ranges.green_asns %}
+
+  <tr>
+    <td>{{ as.asn }}</td>
+    <td>
+      {% if as.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<style>
+.submit-row a.button{ 
+  margin-right: 1rem;
+}
+.submit-row a.button:visited{
+  color: #fff;
+}
+</style>
+
+<div class="submit-row">
+      <a class="button" href="{% url 'greenweb_admin:accounts_hostingprovider_change' provider.id %}">Back to hosting provider</a>
+
+      |
+
+      <a style="margin-left:1rem;" class="button" href="{% url 'greenweb_admin:accounts_hostingprovider_start_import_from_csv' provider.id %}">Make another CSV Import</a>
+</form>
+
+{% endblock content %}