285 enhancement csv upload admin portal

Pipfile

@@ -76,6 +76,7 @@ python-dateutil = "*"
 geoip2 = "*"
 iso3166 = "*"
 django-logentry-admin = "*"
+pandas = "*"
 
 [pipenv]
 # Needed for `black`. See https://github.com/microsoft/vscode-python/pull/5967.

apps/accounts/admin.py

@@ -5,6 +5,7 @@
 from django.contrib import admin
 from django.urls import reverse
 from django.contrib.auth.admin import UserAdmin, GroupAdmin, Group
+import django.forms as dj_forms
 from django.contrib.auth.forms import UserCreationForm
 from django.utils.safestring import mark_safe
 from django.shortcuts import redirect, render
@@ -41,6 +42,8 @@
 from apps.greencheck.models import GreencheckASNapprove
 from apps.greencheck.choices import StatusApproval
 
+from apps.greencheck.forms import ImporterCSVForm
+
 
 from .utils import get_admin_name, reverse_admin_name
 from .admin_site import greenweb_admin
@@ -290,7 +293,7 @@ class HostingAdmin(admin.ModelAdmin):
     ]
     # these are not really fields, but buttons
     # see the corresponding methods
-    readonly_fields = ["preview_email_button"]
+    readonly_fields = ["preview_email_button", "start_csv_import_button"]
     ordering = ("name",)
 
     # Factories
@@ -357,6 +360,82 @@ def preview_email(self, request, *args, **kwargs):
 
         return render(request, "preview_email.html", context)
 
+    def start_import_from_csv(self, request, *args, **kwargs):
+        """
+        Show the form, and preview required formate for the importer
+        for the given hosting provider.
+        """
+
+        # get our provider
+        provider = Hostingprovider.objects.get(pk=kwargs["provider"])
+
+        # get our document
+        data = {"provider": provider.id}
+        form = ImporterCSVForm(data)
+        form.fields["provider"].widget = dj_forms.widgets.HiddenInput()
+
+        return render(
+            request,
+            "import_csv_start.html",
+            {"form": form, "ip_ranges": [], "provider": provider},
+        )
+
+    def save_import_from_csv(self, request, *args, **kwargs):
+        """
+        Process the contents of the uploaded file, and either
+        show a preview of the IP ranges that would be created, or
+        create them, based on submitted form value
+        """
+        provider = Hostingprovider.objects.get(pk=kwargs["provider"])
+
+        if request.method == "POST":
+            # get our provider
+            data = {"provider": provider.id}
+
+            # try to get our document
+            form = ImporterCSVForm(request.POST, request.FILES)
+            form.fields["provider"].widget = dj_forms.widgets.HiddenInput()
+
+            valid = form.is_valid()
+            skip_preview = form.cleaned_data["skip_preview"]
+
+            rich.inspect(request.POST)
+            rich.inspect(form.cleaned_data)
+            if valid and skip_preview:
+                # not doing preview. Run the import
+                form.save()
+
+                ip_ranges = form.get_ip_ranges()
+                context = {
+                    "ip_ranges": ip_ranges,
+                    "provider": provider,
+                }
+                return render(request, "import_csv_preview.html", context,)
+
+            if valid:
+                # the save default we don't save the contents
+                # just showing what would happen
+                ip_ranges = form.get_ip_ranges()
+                context = {
+                    "form": form,
+                    "ip_ranges": ip_ranges,
+                    "provider": provider,
+                }
+                return render(request, "import_csv_preview.html", context,)
+
+            # otherwise fallback to showing the form with errors,
+            # ready for another attempted submission
+
+            context = {
+                "form": form,
+                "ip_ranges": None,
+                "provider": provider,
+            }
+
+            return render(request, "import_csv_preview.html", context,)
+
+        return redirect("greenweb_admin:accounts_hostingprovider_change", provider.id)
+
     def send_email(self, request, *args, **kwargs):
         """
         Send the given email, log the outbound request in the admin, and
@@ -537,6 +616,16 @@ def get_urls(self):
                 self.send_email,
                 name=get_admin_name(self.model, "send_email"),
             ),
+            path(
+                "<provider>/start_import_from_csv",
+                self.start_import_from_csv,
+                name=get_admin_name(self.model, "start_import_from_csv"),
+            ),
+            path(
+                "<provider>/save_import_from_csv",
+                self.save_import_from_csv,
+                name=get_admin_name(self.model, "save_import_from_csv"),
+            ),
             path(
                 "<provider>/preview_email",
                 self.preview_email,
@@ -565,7 +654,12 @@ def get_fieldsets(self, request, obj=None):
         fieldset = [
             (
                 "Hostingprovider info",
-                {"fields": (("name", "website",), "country", "services")},
+                {
+                    "fields": (
+                        ("name", "website",), "country", "services",
+                        ("start_csv_import_button"),
+                    )
+                },
             )
         ]
 
@@ -658,6 +752,20 @@ def preview_email_button(self, obj):
 
     preview_email_button.short_description = "Support Messages"
 
+    @mark_safe
+    def start_csv_import_button(self, obj):
+        """
+        Create clickable link to begin process of bulk import
+        of IP ranges.
+        """
+        url = reverse_admin_name(
+            Hostingprovider, name="start_import_from_csv", kwargs={"provider": obj.pk},
+        )
+        link = f'<a href="{url}" class="start_csv_import">Import IP Ranges from CSV</a>'
+        return link
+
+    send_button.short_description = "Import IP Ranges from a CSV file"
+
     @mark_safe
     def html_website(self, obj):
         html = f'<a href="{obj.website}" target="_blank">{obj.website}</a>'

apps/accounts/admin_site.py

@@ -139,6 +139,7 @@ def get_urls(self):
         patterns = [
             path("try_out/", CheckUrlView.as_view(), name="check_url"),
             path("green-urls", GreenUrlsView.as_view(), name="green_urls"),
+            path("import-ip-ranges", GreenUrlsView.as_view(), name="import_ip_ranges"),
         ]
         return patterns + urls
 
@@ -171,6 +172,19 @@ def get_app_list(self, request):
                     }
                 ],
             },
+            {
+                "name": "Upload ip ranges",
+                "app_label": "greencheck",
+                "app_url": reverse("admin:import_ip_ranges"),
+                "models": [
+                    {
+                        "name": "As CSV",
+                        "object_name": "greencheck_url",
+                        "admin_url": reverse("admin:import_ip_ranges"),
+                        "view_only": True,
+                    }
+                ],
+            },
         ]
         return app_list
 

apps/accounts/templates/import_csv_preview.html

@@ -0,0 +1,79 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
+
+<p> If you have a large number of CSV files, you can import them in bulk.</p>
+
+<p>Make sure the form follows the sample format</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.199</td>
+
+</tr>
+</table>
+
+<p>If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.</p>
+<p>So, uploading a CSV file with a single column IP addresses would be the equivalent to:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.197</td>
+
+</tr>
+</table>
+
+{% endblock %}
+
+
+{% block content %}
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import Preview</h2>
+
+<p>The following IP ranges would be imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+
+{% for ip in ip_ranges %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<form enctype="multipart/form-data" action="{% url 'greenweb_admin:accounts_hostingprovider_save_import_from_csv' provider.id %}" method="post">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="Submit"
+        style="margin-top: 0px; padding: 6px 15px">
+
+      <a href="{% url 'greenweb_admin:accounts_hostingprovider_start_import_from_csv' provider.id %}">Back to start</a>
+</form>
+
+
+{% endblock content %}

apps/accounts/templates/import_csv_results.html

@@ -0,0 +1,42 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
+
+<p> If you have a large number of CSV files, you can import them in bulk.</p>
+{% endblock %}
+
+{% block content %}
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import Preview</h2>
+
+<p>The following IP ranges have been created, and updated for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+
+{% for ip in ip_ranges %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+{% endblock content %}

apps/accounts/templates/import_csv_start.html

@@ -0,0 +1,76 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block pretitle %}
+<h1>Import IP Ranges from a CSV file for {{ provider }}</h1>
+
+<p> If you have a large number of Ip addresses or IP ranges, you can import them in bulk.</p>
+
+<p>Make sure the uploaded file follows the sample format</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.199</td>
+
+</tr>
+</table>
+
+<p>If no end range is provided in the second column, the first IP is used to represent an IP Range of a single IP address.</p>
+<p>So, uploading a CSV file with a single column IP addresses would be the equivalent to:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th>
+<tr>
+  <td>104.21.2.197</td>
+  <td>104.21.2.197</td>
+
+</tr>
+</table>
+
+{% endblock %}
+
+
+{% block content %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<form enctype="multipart/form-data" action="{% url 'greenweb_admin:accounts_hostingprovider_save_import_from_csv' provider.id %}" method="post">
+    {% csrf_token %}
+    {{ form.as_p }}
+    <input type="submit" value="Submit"
+        style="margin-top: 0px; padding: 6px 15px">
+</form>
+
+{% if ip_ranges %}
+
+<hr / style="margin-top:3rem; margin-bottom:3rem;">
+
+<h2>IP Range Import Preview</h2>
+
+<p>The following IP ranges would be imported for <strong>{{ provider }}</strong>:</p>
+
+<table>
+<th>IP Range start</th><th>Ip Range End</th><th>Created / Updated</th>
+
+{% for ip in ip_ranges %}
+
+  <tr>
+    <td>{{ ip.ip_start }}</td>
+    <td>{{ ip.ip_end }}</td>
+    <td>
+      {% if ip.id  %}
+          Updated
+      {% else %}
+          Created
+      {% endif %}
+    </td>
+  </tr>
+
+{% endfor %}
+</table>
+
+{% endif %}
+
+{% endblock content %}