Analytics Export to Ingress

galaxy_ng/app/management/commands/analytics/automation_analytics/collector.py

@@ -0,0 +1,50 @@
+from django.conf import settings
+
+from galaxy_ng.app.management.commands.analytics.collector import Collector as BaseCollector
+from galaxy_ng.app.management.commands.analytics.automation_analytics.package import Package
+
+
+class Collector(BaseCollector):
+    @staticmethod
+    def _package_class():
+        return Package
+
+    def _is_shipping_configured(self):
+        # if self.is_shipping_enabled():
+            # TODO: Feature flags here for enabled/disabled?
+            # if not settings.INSIGHTS_TRACKING_STATE:
+            #     self.logger.log(self.log_level, "Insights for Ansible Automation Platform not enabled. " "Use --dry-run to gather locally without sending.")
+            #     return False
+
+            # if not (settings.AUTOMATION_ANALYTICS_URL and settings.REDHAT_USERNAME and settings.REDHAT_PASSWORD):
+            #     self.logger.log(self.log_level, "Not gathering analytics, configuration is invalid. " "Use --dry-run to gather locally without sending.")
+            #     return False
+
+        return True
+
+    def _last_gathering(self):
+        # TODO: There is no persistent DB storage in Hub
+        # https://issues.redhat.com/browse/AAH-2009
+        # return settings.AUTOMATION_ANALYTICS_LAST_GATHER
+        return None
+
+    def _load_last_gathered_entries(self):
+        # TODO: There is no persistent DB storage in Hub
+        # from awx.conf.models import Setting
+        #
+        # last_entries = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_ENTRIES').first()
+        # last_gathered_entries = json.loads((last_entries.value if last_entries is not None else '') or '{}', object_hook=datetime_hook)
+        last_gathered_entries = {}
+        return last_gathered_entries
+
+    def _save_last_gathered_entries(self, last_gathered_entries):
+        # TODO: There is no persistent DB storage in Hub
+        pass
+
+    def _save_last_gather(self):
+        # TODO: There is no persistent DB storage in Hub
+        pass
+
+
+
+

galaxy_ng/app/management/commands/analytics/automation_analytics/data.py

@@ -0,0 +1,132 @@
+import os
+from django.db import connection
+from insights_analytics_collector import CsvFileSplitter, register
+import galaxy_ng.app.management.commands.analytics.common_data as data
+
+
+@register("config", "1.0", description="General platform configuration.", config=True)
+def config(since, **kwargs):
+    return data.config()
+
+
+@register("instance_info", "1.0", description="Node information")
+def instance_info(since, **kwargs):
+    return data.instance_info()
+
+
+@register("collections", "1.0", format="csv", description="Data on ansible_collection")
+def collections(since, full_path, until, **kwargs):
+    query = data.collections_query()
+
+    return export_to_csv(full_path, "collections", query)
+
+
+@register(
+    "collection_versions",
+    "1.0",
+    format="csv",
+    description="Data on ansible_collectionversion",
+)
+def collection_versions(since, full_path, until, **kwargs):
+    query = data.collection_versions_query()
+
+    return export_to_csv(full_path, "collection_versions", query)
+
+
+@register(
+    "collection_version_tags",
+    "1.0",
+    format="csv",
+    description="Full sync: Data on ansible_collectionversion_tags"
+)
+def collection_version_tags(since, full_path, **kwargs):
+    query = data.collection_version_tags_query()
+    return export_to_csv(full_path, "collection_version_tags", query)
+
+
+@register(
+    "collection_tags",
+    "1.0",
+    format="csv",
+    description="Data on ansible_tag"
+)
+def collection_tags(since, full_path, **kwargs):
+    query = data.collection_tags_query()
+    return export_to_csv(full_path, "collection_tags", query)
+
+
+@register(
+    "collection_version_signatures",
+    "1.0",
+    format="csv",
+    description="Data on ansible_collectionversionsignature",
+)
+def collection_version_signatures(since, full_path, **kwargs):
+    query = data.collection_version_signatures_query()
+
+    return export_to_csv(full_path, "collection_version_signatures", query)
+
+
+@register(
+    "signing_services",
+    "1.0",
+    format="csv",
+    description="Data on core_signingservice"
+)
+def signing_services(since, full_path, **kwargs):
+    query = data.signing_services_query()
+    return export_to_csv(full_path, "signing_services", query)
+
+
+# @register(
+#     "collection_imports",
+#     "1.0",
+#     format="csv",
+#     description="Data on ansible_collectionimport",
+# )
+# def collection_imports(since, full_path, until, **kwargs):
+#     # currently no rows in the table, so no objects to base a query off
+#     source_query = """COPY (
+#             SELECT * FROM ansible_collectionimport
+#         ) TO STDOUT WITH CSV HEADER
+#     """
+#     return _simple_csv(full_path, "ansible_collectionimport", source_query)
+#
+
+@register(
+    "collection_stats",
+    "1.0",
+    format="csv",
+    description="Data from ansible_downloadlog"
+)
+def collection_stats(since, full_path, until, **kwargs):
+    query = data.collection_stats_query()
+    return export_to_csv(full_path, "collection_stats", query)
+
+
+def _get_csv_splitter(file_path, max_data_size=209715200):
+    return CsvFileSplitter(filespec=file_path, max_file_size=max_data_size)
+
+
+def export_to_csv(full_path, file_name, query):
+    copy_query = f"""COPY (
+    {query}
+    ) TO STDOUT WITH CSV HEADER
+    """
+    return _simple_csv(full_path, file_name, copy_query, max_data_size=209715200)
+
+
+def _simple_csv(full_path, file_name, query, max_data_size=209715200):
+    file_path = _get_file_path(full_path, file_name)
+    tfile = _get_csv_splitter(file_path, max_data_size)
+
+    with connection.cursor() as cursor:
+        with cursor.copy(query) as copy:
+            while data := copy.read():
+                tfile.write(str(data, 'utf8'))
+
+    return tfile.file_list()
+
+
+def _get_file_path(path, table):
+    return os.path.join(path, table + ".csv")

galaxy_ng/app/management/commands/analytics/automation_analytics/package.py

@@ -0,0 +1,36 @@
+from django.conf import settings
+
+from insights_analytics_collector import Package as InsightsAnalyticsPackage
+
+
+class Package(InsightsAnalyticsPackage):
+    # TODO The CERT
+    CERT_PATH = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"
+    PAYLOAD_CONTENT_TYPE = "application/vnd.redhat.automation-hub.hub_payload+tgz"
+
+    def _tarname_base(self):
+        timestamp = self.collector.gather_until
+        return f'galaxy-hub-analytics-{timestamp.strftime("%Y-%m-%d-%H%M")}'
+
+    def get_ingress_url(self):
+        # TODO: There is no persistent DB storage in Hub
+        # return getattr(settings, 'AUTOMATION_ANALYTICS_URL', None)
+        return "http://automation-analytics-backend_ingress_1:3000/api/ingress/v1/upload"
+
+    def _get_rh_user(self):
+        # return getattr(settings, 'REDHAT_USERNAME', None)
+        return "aa-user"
+
+    def _get_rh_password(self):
+        # return getattr(settings, 'REDHAT_PASSWORD', None)
+        return "aa-password"
+
+    def _get_http_request_headers(self):
+        headers = {
+            'Content-Type': 'application/json',
+            'User-Agent': '{} {} ({})'.format('Red Hat Ansible Automation Platform', 'TODO: version', 'TODO: license'),
+        }
+        return headers
+
+    def shipping_auth_mode(self):
+        return self.SHIPPING_AUTH_USERPASS

galaxy_ng/app/management/commands/analytics/collector.py

@@ -1,46 +1,12 @@
 from django.db import connection
-
 from insights_analytics_collector import Collector as BaseCollector
-from galaxy_ng.app.management.commands.analytics.package import Package
 
 
 class Collector(BaseCollector):
-    def __init__(self, collection_type, collector_module, logger):
-        super().__init__(
-            collection_type=collection_type, collector_module=collector_module, logger=logger
-        )
-
-    @staticmethod
-    def db_connection():
-        return connection
-
-    @staticmethod
-    def _package_class():
-        return Package
-
-    def get_last_gathering(self):
-        return self._last_gathering()
-
-    def _is_shipping_configured(self):
-        # TODO: need shipping configuration
-        return True
-
     def _is_valid_license(self):
         # TODO: need license information and validation logics
         return True
 
-    def _last_gathering(self):
-        # doing a full scan database dump
-        return None
-
-    def _load_last_gathered_entries(self):
-        # doing a full scan database dump
-        return {}
-
-    def _save_last_gathered_entries(self, last_gathered_entries):
-        # doing a full scan database dump
-        pass
-
-    def _save_last_gather(self):
-        # doing a full scan database dump
-        pass
+    @staticmethod
+    def db_connection():
+        return connection