Merge pull request #850 from nautobot/Release-v2.2.2

Release v2.2.2

docs/admin/release_notes/version_2.2.md

@@ -8,6 +8,18 @@ This document describes all new features and changes in the release. The format
 - Added Python 3.12 support.
 - Added REST API endpoint for Jinja as first part of journey towards a jinja live editor.
 
+## [v2.2.2 (2024-12-17)](https://github.com/nautobot/nautobot-app-golden-config/releases/tag/v2.2.2)
+
+### Added
+
+- [#840](https://github.com/nautobot/nautobot-app-golden-config/issues/840) - Added GraphQL output to the "Generate Intended Config" view.
+- [#841](https://github.com/nautobot/nautobot-app-golden-config/issues/841) - Added GraphQL query form field to the "Generate Intended Config" view.
+- [#844](https://github.com/nautobot/nautobot-app-golden-config/issues/844) - Added a diff output to the "Generate Intended Config" view and associated REST API.
+
+### Fixed
+
+- [#849](https://github.com/nautobot/nautobot-app-golden-config/issues/849) - Fixed failing tests in Nautobot v2.3.13.
+
 ## [v2.2.1 (2024-11-27)](https://github.com/nautobot/nautobot-app-golden-config/releases/tag/v2.2.1)
 
 ### Added

docs/user/app_feature_intended.md

@@ -45,7 +45,9 @@ curl -s -X GET \
     http://nautobot/api/plugins/golden-config/generate-intended-config/?device_id=231b8765-054d-4abe-bdbf-cd60e049cd8d
 ```
 
-The returned response will contain the rendered configuration for the specified device. The web UI provides a simple form to input the device and displays the rendered configuration when submitted.
+The returned response will contain the rendered configuration for the specified device, the GraphQL data that was used, and if applicable, a diff of the most recent intended config that was generated by the **Intended Configuration** job. The web UI provides a simple form to interact with this REST API. You can access the web UI by clicking on "Generate Intended Config" in the "Tools" section of the Golden Config navigation menu.
+
+For more advanced use cases, the REST API and web UI also accept a `graphql_query_id` parameter to specify a custom GraphQL query to use when rendering the configuration. If a `graphql_query_id` is not provided, the default query configured in the Device's Golden Config settings will be used.
 
 ![Intended Configuration Web UI](../images/generate-intended-config-ui.png#only-light)
 ![Intended Configuration Web UI](../images/generate-intended-config-ui-dark.png#only-dark)

nautobot_golden_config/api/serializers.py

@@ -131,3 +131,6 @@ class GenerateIntendedConfigSerializer(serializers.Serializer):  # pylint: disab
 
     intended_config = serializers.CharField(read_only=True)
     intended_config_lines = serializers.ListField(read_only=True, child=serializers.CharField())
+    graphql_data = serializers.JSONField(read_only=True)
+    diff = serializers.CharField(read_only=True)
+    diff_lines = serializers.ListField(read_only=True, child=serializers.CharField())

nautobot_golden_config/api/views.py

@@ -1,6 +1,7 @@
 """View for Golden Config APIs."""
 
 import datetime
+import difflib
 import json
 import logging
 from pathlib import Path
@@ -20,6 +21,7 @@
 from nautobot.dcim.models import Device
 from nautobot.extras.api.views import NautobotModelViewSet, NotesViewSetMixin
 from nautobot.extras.datasources.git import ensure_git_repository
+from nautobot.extras.models import GraphQLQuery
 from nautobot_plugin_nornir.constants import NORNIR_SETTINGS
 from nornir import InitNornir
 from nornir_nautobot.plugins.tasks.dispatcher import dispatcher
@@ -202,6 +204,26 @@ class GenerateIntendedConfigView(NautobotAPIVersionMixin, GenericAPIView):
     permission_classes = [IsAuthenticated]
     serializer_class = serializers.GenerateIntendedConfigSerializer
 
+    def _get_diff(self, device, intended_config):
+        """Generate a unified diff between the provided config and the intended config stored on the Device's GoldenConfig.intended_config."""
+        diff = None
+        try:
+            golden_config = device.goldenconfig
+            if golden_config.intended_last_success_date is not None:
+                prior_intended_config = golden_config.intended_config
+                diff = "".join(
+                    difflib.unified_diff(
+                        prior_intended_config.splitlines(keepends=True),
+                        intended_config.splitlines(keepends=True),
+                        fromfile="prior intended config",
+                        tofile="rendered config",
+                    )
+                )
+        except models.GoldenConfig.DoesNotExist:
+            pass
+
+        return diff
+
     def _get_object(self, request, model, query_param):
         """Get the requested model instance, restricted to requesting user."""
         pk = request.query_params.get(query_param)
@@ -232,19 +254,41 @@ def _get_jinja_template_path(self, settings, device, git_repository):
                 type=OpenApiTypes.UUID,
                 location=OpenApiParameter.QUERY,
             ),
+            OpenApiParameter(
+                name="graphql_query_id",
+                required=False,
+                type=OpenApiTypes.UUID,
+                location=OpenApiParameter.QUERY,
+            ),
         ]
     )
     def get(self, request, *args, **kwargs):
         """Generate intended configuration for a Device."""
         device = self._get_object(request, Device, "device_id")
+        graphql_query = None
+        graphql_query_id_param = request.query_params.get("graphql_query_id")
+        if graphql_query_id_param:
+            try:
+                graphql_query = GraphQLQuery.objects.get(pk=request.query_params.get("graphql_query_id"))
+            except GraphQLQuery.DoesNotExist as exc:
+                raise GenerateIntendedConfigException(
+                    f"GraphQLQuery with id '{graphql_query_id_param}' not found"
+                ) from exc
         settings = models.GoldenConfigSetting.objects.get_for_device(device)
         if not settings:
             raise GenerateIntendedConfigException("No Golden Config settings found for this device")
-        if not settings.sot_agg_query:
-            raise GenerateIntendedConfigException("Golden Config settings sot_agg_query not set")
         if not settings.jinja_repository:
             raise GenerateIntendedConfigException("Golden Config settings jinja_repository not set")
 
+        if graphql_query is None:
+            if settings.sot_agg_query is not None:
+                graphql_query = settings.sot_agg_query
+            else:
+                raise GenerateIntendedConfigException("Golden Config settings sot_agg_query not set")
+
+        if "device_id" not in graphql_query.variables:
+            raise GenerateIntendedConfigException("The selected GraphQL query is missing a 'device_id' variable")
+
         try:
             git_repository = settings.jinja_repository
             ensure_git_repository(git_repository)
@@ -253,21 +297,27 @@ def get(self, request, *args, **kwargs):
 
         filesystem_path = self._get_jinja_template_path(settings, device, git_repository)
 
-        status_code, context = graph_ql_query(request, device, settings.sot_agg_query.query)
+        status_code, graphql_data = graph_ql_query(request, device, graphql_query.query)
         if status_code == status.HTTP_200_OK:
             try:
                 intended_config = self._render_config_nornir_serial(
                     device=device,
                     jinja_template=filesystem_path.name,
                     jinja_root_path=filesystem_path.parent,
-                    graphql_data=context,
+                    graphql_data=graphql_data,
                 )
             except Exception as exc:
                 raise GenerateIntendedConfigException(f"Error rendering Jinja template: {exc}") from exc
+
+            diff = self._get_diff(device, intended_config)
+
             return Response(
                 data={
                     "intended_config": intended_config,
                     "intended_config_lines": intended_config.split("\n"),
+                    "graphql_data": graphql_data,
+                    "diff": diff,
+                    "diff_lines": diff.split("\n") if diff else [],
                 },
                 status=status.HTTP_200_OK,
             )

nautobot_golden_config/filter_extensions.py

@@ -0,0 +1,30 @@
+"""FilterSet and FilterForm extensions for the Golden Config app."""
+
+from django.db.models import Q
+from nautobot.apps.filters import FilterExtension, MultiValueCharFilter
+
+
+def filter_graphql_query_variables(queryset, name, value):  # pylint: disable=unused-argument
+    """Filter the queryset based on the presence of variables."""
+    query = Q()
+    for variable_name in value:
+        query |= Q(**{f"variables__{variable_name}__isnull": True})
+    return queryset.exclude(query)
+
+
+class GraphQLQueryFilterExtension(FilterExtension):
+    """Filter extensions for the extras.GraphQLQuery model."""
+
+    model = "extras.graphqlquery"
+
+    filterset_fields = {
+        "nautobot_golden_config_graphql_query_variables": MultiValueCharFilter(
+            field_name="variables",
+            lookup_expr="exact",
+            method=filter_graphql_query_variables,
+            label="Variable key(s) exist",
+        ),
+    }
+
+
+filter_extensions = [GraphQLQueryFilterExtension]

nautobot_golden_config/filters.py

@@ -245,6 +245,27 @@ class Meta:
 class GoldenConfigSettingFilterSet(NautobotFilterSet):
     """Inherits Base Class NautobotFilterSet."""
 
+    device_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=Device.objects.all(),
+        label="Device (ID)",
+        method="filter_device_id",
+    )
+
+    def filter_device_id(self, queryset, name, value):  # pylint: disable=unused-argument
+        """Filter by Device ID."""
+        if not value:
+            return queryset
+        golden_config_setting_ids = []
+        for instance in value:
+            if isinstance(instance, Device):
+                device = instance
+            else:
+                device = Device.objects.get(id=instance)
+            golden_config_setting = models.GoldenConfigSetting.objects.get_for_device(device)
+            if golden_config_setting is not None:
+                golden_config_setting_ids.append(golden_config_setting.id)
+        return queryset.filter(id__in=golden_config_setting_ids)
+
     class Meta:
         """Boilerplate filter Meta data for Config Remove."""
 

nautobot_golden_config/forms.py

@@ -7,7 +7,7 @@
 from nautobot.apps import forms
 from nautobot.dcim.models import Device, DeviceType, Location, Manufacturer, Platform, Rack, RackGroup
 from nautobot.extras.forms import NautobotBulkEditForm, NautobotFilterForm, NautobotModelForm
-from nautobot.extras.models import DynamicGroup, GitRepository, JobResult, Role, Status, Tag
+from nautobot.extras.models import DynamicGroup, GitRepository, GraphQLQuery, JobResult, Role, Status, Tag
 from nautobot.tenancy.models import Tenant, TenantGroup
 
 from nautobot_golden_config import models
@@ -608,3 +608,9 @@ class GenerateIntendedConfigForm(django_forms.Form):
         required=True,
         label="Device",
     )
+    graphql_query = forms.DynamicModelChoiceField(
+        queryset=GraphQLQuery.objects.all(),
+        required=True,
+        label="GraphQL Query",
+        query_params={"nautobot_golden_config_graphql_query_variables": "device_id"},
+    )

nautobot_golden_config/tables.py

@@ -63,7 +63,7 @@
     {% if record.config_type == 'json' %}
         <i class="mdi mdi-circle-small"></i>
     {% else %}
-        <a href="{% url 'extras:job_run_by_class_path' class_path='nautobot_golden_config.jobs.AllGoldenConfig' %}?device={{ record.device.pk }}"
+        <a href="{% url 'extras:job_run_by_class_path' class_path='nautobot_golden_config.jobs.AllGoldenConfig' %}?device={{ record.device.pk }}">
             <span class="text-primary">
                 <i class="mdi mdi-play-circle" title="Execute All Golden Config Jobs"></i>
             </span>

nautobot_golden_config/templates/nautobot_golden_config/configcompliance_devicetab.html

@@ -5,7 +5,8 @@
 
 {% block title %} {{ object }} - Config Compliance {% endblock %}
 
-{% block content %}
+{% block extra_styles %}
+{{ block.super }}
 <style>
     #compliance-content pre {
         font-size: 10px;
@@ -56,12 +57,15 @@
         transform: scale(1.1);
     }
 </style>
+{% endblock extra_styles %}
+
+{% block content %}
 {% block navigation %}
 <div class="panel panel-default" style="width:100%">
-        <div class="panel-heading"><strong>Feature Navigation</strong> 
-            <a href="{% url 'plugins:nautobot_golden_config:configcompliance_devicetab' pk=device.pk %}?tab={{ active_tab }}&compliance=compliant" class="btn btn-success">Compliant</a>  
-            <a href="{% url 'plugins:nautobot_golden_config:configcompliance_devicetab' pk=device.pk %}?tab={{ active_tab }}&compliance=non-compliant" class="btn btn-danger">Non-Compliant</a>  
-            <a href="{% url 'plugins:nautobot_golden_config:configcompliance_devicetab' pk=device.pk %}?tab={{ active_tab }}" class="btn btn-info">Clear</a>  
+        <div class="panel-heading"><strong>Feature Navigation</strong>
+            <a href="{% url 'plugins:nautobot_golden_config:configcompliance_devicetab' pk=device.pk %}?tab={{ active_tab }}&compliance=compliant" class="btn btn-success">Compliant</a>
+            <a href="{% url 'plugins:nautobot_golden_config:configcompliance_devicetab' pk=device.pk %}?tab={{ active_tab }}&compliance=non-compliant" class="btn btn-danger">Non-Compliant</a>
+            <a href="{% url 'plugins:nautobot_golden_config:configcompliance_devicetab' pk=device.pk %}?tab={{ active_tab }}" class="btn btn-info">Clear</a>
         </div>
     <div id="navigation">
         <table>
@@ -203,4 +207,4 @@
     </table>
     </div>
 {% endfor %}
-{% endblock %}
+{% endblock %}

nautobot_golden_config/templates/nautobot_golden_config/configcompliance_overview.html

@@ -6,7 +6,7 @@
 {% block title %}Overview Reports{% endblock %}
 
 {% block breadcrumbs %}
-<li><a href="{% url 'plugins:nautobot_golden_config:configcompliance_overview' %}">Overview Reports</a></li>  
+<li><a href="{% url 'plugins:nautobot_golden_config:configcompliance_overview' %}">Overview Reports</a></li>
 {% block extra_breadcrumbs %}{% endblock extra_breadcrumbs %}
 {% endblock breadcrumbs %}
 
@@ -74,5 +74,6 @@ <h3 class="text-center m-2 p-3">Feature Summary</h3>
 {% table_config_form table table_name="ObjectTable" %}
 {% endblock %}
 {% block javascript %}
+{{ block.super }}
 <script src="{% static 'js/tableconfig.js' %}"></script>
 {% endblock %}

nautobot_golden_config/templates/nautobot_golden_config/configcompliance_retrieve.html

@@ -2,13 +2,7 @@
 {% load helpers %}
 {% load static %}
 
-{% block content_left_page %}
-<script src="{% static 'clipboard.js-2.0.9/clipboard.min.js' %}"></script>
-<script>    
-    var clipboard = new ClipboardJS('.btn');
-    clipboard.on('success', function (e) {});
-    clipboard.on('error', function (e) {});
-</script>
+{% block extra_styles %}
 <style>
     #config-output pre {
         white-space: pre-wrap;       /* CSS 2.1 */
@@ -18,6 +12,9 @@
         word-wrap: break-word;       /* IE >= 5.5 */
     }
 </style>
+{% endblock extra_styles %}
+
+{% block content_left_page %}
     <div id="config-output" class="panel panel-default">
         <div class="panel-heading">
             <strong>Details</strong>
@@ -90,4 +87,13 @@
     </div>
     {% include 'inc/custom_fields_panel.html' %}
     {% include 'inc/relationships_panel.html' %}
-{% endblock %}
+{% endblock %}
+
+{% block javascript %}
+{{ block.super }}
+<script>
+    var clipboard = new ClipboardJS('.btn');
+    clipboard.on('success', function (e) {});
+    clipboard.on('error', function (e) {});
+</script>
+{% endblock javascript %}

nautobot_golden_config/templates/nautobot_golden_config/configplan_create.html

@@ -47,6 +47,7 @@
 {% endblock %}
 
 {% block javascript %}
+{{ block.super }}
 <script src="{% static 'toggle_fields.js' %}"></script>
 <script src="{% static 'run_job.js' %}"></script>
 <script>

nautobot_golden_config/templates/nautobot_golden_config/configplan_retrieve.html

@@ -1,16 +1,18 @@
 {% extends 'generic/object_detail.html' %}
 {% load helpers %}
 
-{% block content_left_page %}
-    <style> 
+{% block extra_styles %}
+    <style>
         .table-wrapper{
         display: block;
         overflow-x: auto;
         white-space: nowrap;
         width: 100%;
     }
     </style>
+{% endblock extra_styles %}
 
+{% block content_left_page %}
     <div class="panel panel-default">
         <div class="panel-heading">
             <strong>Config Plan Details</strong>