diff --git a/galaxy_ng/app/management/commands/analytics/automation_analytics/__init__.py b/galaxy_ng/app/management/commands/analytics/automation_analytics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/galaxy_ng/app/management/commands/analytics/automation_analytics/collector.py b/galaxy_ng/app/management/commands/analytics/automation_analytics/collector.py new file mode 100644 index 0000000000..4e4bbcc874 --- /dev/null +++ b/galaxy_ng/app/management/commands/analytics/automation_analytics/collector.py @@ -0,0 +1,50 @@ +from django.conf import settings + +from galaxy_ng.app.management.commands.analytics.collector import Collector as BaseCollector +from galaxy_ng.app.management.commands.analytics.automation_analytics.package import Package + + +class Collector(BaseCollector): + @staticmethod + def _package_class(): + return Package + + def _is_shipping_configured(self): + # if self.is_shipping_enabled(): + # TODO: Feature flags here for enabled/disabled? + # if not settings.INSIGHTS_TRACKING_STATE: + # self.logger.log(self.log_level, "Insights for Ansible Automation Platform not enabled. " "Use --dry-run to gather locally without sending.") + # return False + + # if not (settings.AUTOMATION_ANALYTICS_URL and settings.REDHAT_USERNAME and settings.REDHAT_PASSWORD): + # self.logger.log(self.log_level, "Not gathering analytics, configuration is invalid. " "Use --dry-run to gather locally without sending.") + # return False + + return True + + def _last_gathering(self): + # TODO: There is no persistent DB storage in Hub + # https://issues.redhat.com/browse/AAH-2009 + # return settings.AUTOMATION_ANALYTICS_LAST_GATHER + return None + + def _load_last_gathered_entries(self): + # TODO: There is no persistent DB storage in Hub + # from awx.conf.models import Setting + # + # last_entries = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_ENTRIES').first() + # last_gathered_entries = json.loads((last_entries.value if last_entries is not None else '') or '{}', object_hook=datetime_hook) + last_gathered_entries = {} + return last_gathered_entries + + def _save_last_gathered_entries(self, last_gathered_entries): + # TODO: There is no persistent DB storage in Hub + pass + + def _save_last_gather(self): + # TODO: There is no persistent DB storage in Hub + pass + + + + diff --git a/galaxy_ng/app/management/commands/analytics/automation_analytics/data.py b/galaxy_ng/app/management/commands/analytics/automation_analytics/data.py new file mode 100644 index 0000000000..9330784f5c --- /dev/null +++ b/galaxy_ng/app/management/commands/analytics/automation_analytics/data.py @@ -0,0 +1,132 @@ +import os +from django.db import connection +from insights_analytics_collector import CsvFileSplitter, register +import galaxy_ng.app.management.commands.analytics.common_data as data + + +@register("config", "1.0", description="General platform configuration.", config=True) +def config(since, **kwargs): + return data.config() + + +@register("instance_info", "1.0", description="Node information") +def instance_info(since, **kwargs): + return data.instance_info() + + +@register("collections", "1.0", format="csv", description="Data on ansible_collection") +def collections(since, full_path, until, **kwargs): + query = data.collections_query() + + return export_to_csv(full_path, "collections", query) + + +@register( + "collection_versions", + "1.0", + format="csv", + description="Data on ansible_collectionversion", +) +def collection_versions(since, full_path, until, **kwargs): + query = data.collection_versions_query() + + return export_to_csv(full_path, "collection_versions", query) + + +@register( + "collection_version_tags", + "1.0", + format="csv", + description="Full sync: Data on ansible_collectionversion_tags" +) +def collection_version_tags(since, full_path, **kwargs): + query = data.collection_version_tags_query() + return export_to_csv(full_path, "collection_version_tags", query) + + +@register( + "collection_tags", + "1.0", + format="csv", + description="Data on ansible_tag" +) +def collection_tags(since, full_path, **kwargs): + query = data.collection_tags_query() + return export_to_csv(full_path, "collection_tags", query) + + +@register( + "collection_version_signatures", + "1.0", + format="csv", + description="Data on ansible_collectionversionsignature", +) +def collection_version_signatures(since, full_path, **kwargs): + query = data.collection_version_signatures_query() + + return export_to_csv(full_path, "collection_version_signatures", query) + + +@register( + "signing_services", + "1.0", + format="csv", + description="Data on core_signingservice" +) +def signing_services(since, full_path, **kwargs): + query = data.signing_services_query() + return export_to_csv(full_path, "signing_services", query) + + +# @register( +# "collection_imports", +# "1.0", +# format="csv", +# description="Data on ansible_collectionimport", +# ) +# def collection_imports(since, full_path, until, **kwargs): +# # currently no rows in the table, so no objects to base a query off +# source_query = """COPY ( +# SELECT * FROM ansible_collectionimport +# ) TO STDOUT WITH CSV HEADER +# """ +# return _simple_csv(full_path, "ansible_collectionimport", source_query) +# + +@register( + "collection_stats", + "1.0", + format="csv", + description="Data from ansible_downloadlog" +) +def collection_stats(since, full_path, until, **kwargs): + query = data.collection_stats_query() + return export_to_csv(full_path, "collection_stats", query) + + +def _get_csv_splitter(file_path, max_data_size=209715200): + return CsvFileSplitter(filespec=file_path, max_file_size=max_data_size) + + +def export_to_csv(full_path, file_name, query): + copy_query = f"""COPY ( + {query} + ) TO STDOUT WITH CSV HEADER + """ + return _simple_csv(full_path, file_name, copy_query, max_data_size=209715200) + + +def _simple_csv(full_path, file_name, query, max_data_size=209715200): + file_path = _get_file_path(full_path, file_name) + tfile = _get_csv_splitter(file_path, max_data_size) + + with connection.cursor() as cursor: + with cursor.copy(query) as copy: + while data := copy.read(): + tfile.write(str(data, 'utf8')) + + return tfile.file_list() + + +def _get_file_path(path, table): + return os.path.join(path, table + ".csv") diff --git a/galaxy_ng/app/management/commands/analytics/automation_analytics/package.py b/galaxy_ng/app/management/commands/analytics/automation_analytics/package.py new file mode 100644 index 0000000000..9d20dde44f --- /dev/null +++ b/galaxy_ng/app/management/commands/analytics/automation_analytics/package.py @@ -0,0 +1,36 @@ +from django.conf import settings + +from insights_analytics_collector import Package as InsightsAnalyticsPackage + + +class Package(InsightsAnalyticsPackage): + # TODO The CERT + CERT_PATH = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" + PAYLOAD_CONTENT_TYPE = "application/vnd.redhat.automation-hub.hub_payload+tgz" + + def _tarname_base(self): + timestamp = self.collector.gather_until + return f'galaxy-hub-analytics-{timestamp.strftime("%Y-%m-%d-%H%M")}' + + def get_ingress_url(self): + # TODO: There is no persistent DB storage in Hub + # return getattr(settings, 'AUTOMATION_ANALYTICS_URL', None) + return "http://automation-analytics-backend_ingress_1:3000/api/ingress/v1/upload" + + def _get_rh_user(self): + # return getattr(settings, 'REDHAT_USERNAME', None) + return "aa-user" + + def _get_rh_password(self): + # return getattr(settings, 'REDHAT_PASSWORD', None) + return "aa-password" + + def _get_http_request_headers(self): + headers = { + 'Content-Type': 'application/json', + 'User-Agent': '{} {} ({})'.format('Red Hat Ansible Automation Platform', 'TODO: version', 'TODO: license'), + } + return headers + + def shipping_auth_mode(self): + return self.SHIPPING_AUTH_USERPASS diff --git a/galaxy_ng/app/management/commands/analytics/collector.py b/galaxy_ng/app/management/commands/analytics/collector.py index 8a9a6db82d..f02153e426 100644 --- a/galaxy_ng/app/management/commands/analytics/collector.py +++ b/galaxy_ng/app/management/commands/analytics/collector.py @@ -1,46 +1,12 @@ from django.db import connection - from insights_analytics_collector import Collector as BaseCollector -from galaxy_ng.app.management.commands.analytics.package import Package class Collector(BaseCollector): - def __init__(self, collection_type, collector_module, logger): - super().__init__( - collection_type=collection_type, collector_module=collector_module, logger=logger - ) - - @staticmethod - def db_connection(): - return connection - - @staticmethod - def _package_class(): - return Package - - def get_last_gathering(self): - return self._last_gathering() - - def _is_shipping_configured(self): - # TODO: need shipping configuration - return True - def _is_valid_license(self): # TODO: need license information and validation logics return True - def _last_gathering(self): - # doing a full scan database dump - return None - - def _load_last_gathered_entries(self): - # doing a full scan database dump - return {} - - def _save_last_gathered_entries(self, last_gathered_entries): - # doing a full scan database dump - pass - - def _save_last_gather(self): - # doing a full scan database dump - pass + @staticmethod + def db_connection(): + return connection diff --git a/galaxy_ng/app/management/commands/analytics/common_data.py b/galaxy_ng/app/management/commands/analytics/common_data.py new file mode 100644 index 0000000000..a8091b073c --- /dev/null +++ b/galaxy_ng/app/management/commands/analytics/common_data.py @@ -0,0 +1,167 @@ +import requests +import logging +from urllib.parse import urljoin +from django.conf import settings +import platform +import distro +from django.conf import settings +# from pulpcore.plugin.models.telemetry import SystemID + + +logger = logging.getLogger("analytics.export_data") + + +def api_status(): + status_path = '/pulp/api/v3/status/' + try: + url = urljoin(settings.ANSIBLE_API_HOSTNAME, status_path) + response = requests.request("GET", url) + if response.status_code == 200: + return response.json() + else: + logger.error(f"export analytics: failed API call {status_path}: HTTP status {response.status_code}") + return {} + except Exception as e: + logger.error(f"export analytics: failed API call {status_path}: {e}") + return {} + + +def hub_version(): + status = api_status() + galaxy_version = '' + for version in status['versions']: + if version['component'] == 'galaxy': + galaxy_version = version['version'] + return galaxy_version + + +def config(): + # TODO: license_info = get_license() + license_info = {} + + return { + "platform": { + "system": platform.system(), + "dist": distro.linux_distribution(), + "release": platform.release(), + "type": "traditional", + }, + "install_uuid": "0189733f-856b-74d2-b47f-e3bd42377c5c", # SystemID.objects.first().pulp_id, # cluster_id -> core_systemid.pulp_id + "instance_uuid": "", # instances in cluster not distinguished + "hub_url_base": settings.ANSIBLE_API_HOSTNAME, + "hub_version": hub_version(), + "license_type": license_info.get("license_type", "UNLICENSED"), + "free_instances": license_info.get("free_instances", 0), + "total_licensed_instances": license_info.get("instance_count", 0), + "license_expiry": license_info.get("time_remaining", 0), + "authentication_backends": settings.AUTHENTICATION_BACKENDS, + "external_logger_enabled": "todo", + "external_logger_type": "todo", + "logging_aggregators": ["todo"], + "pendo_tracking": "todo", + } + + +def instance_info(): + status = api_status() + + return { + "versions": status.get('versions', {}), + "online_workers": status.get('online_workers', []), + "online_content_apps": status.get('online_content_apps', []), + "database_connection": status.get('database_connection', {}), + "redis_connection": status.get('redis_connection', {}), + "storage": status.get('storage', {}), + "content_settings": status.get('content_settings', {}), + "domain_enabled": status.get('domain_enabled', '') + } + + +def collections_query(): + return """ + SELECT "ansible_collection"."pulp_id" AS uuid, + "ansible_collection"."pulp_created", + "ansible_collection"."pulp_last_updated", + "ansible_collection"."namespace", + "ansible_collection"."name" + FROM "ansible_collection" + """ + + +def collection_versions_query(): + return """ + SELECT "ansible_collectionversion"."content_ptr_id" AS uuid, + "core_content"."pulp_created", + "core_content"."pulp_last_updated", + "ansible_collectionversion"."collection_id", + "ansible_collectionversion"."contents", + "ansible_collectionversion"."dependencies", + "ansible_collectionversion"."description", + "ansible_collectionversion"."license", + "ansible_collectionversion"."version", + "ansible_collectionversion"."requires_ansible", + "ansible_collectionversion"."is_highest", + "ansible_collectionversion"."repository" + FROM "ansible_collectionversion" + INNER JOIN "core_content" ON ( + "ansible_collectionversion"."content_ptr_id" = "core_content"."pulp_id" + ) + """ + + +def collection_version_tags_query(): + return ''' + SELECT id, + collectionversion_id AS collection_version_id, + tag_id + FROM ansible_collectionversion_tags + ''' + + +def collection_tags_query(): + return ''' + SELECT pulp_id AS uuid, + pulp_created, + pulp_last_updated, + name + FROM ansible_tag + ''' + + +def collection_version_signatures_query(): + return """ + SELECT "ansible_collectionversionsignature".content_ptr_id AS uuid, + "core_content".pulp_created, + "core_content".pulp_last_updated, + "ansible_collectionversionsignature".signed_collection_id AS collection_version_id, + "ansible_collectionversionsignature".data, + "ansible_collectionversionsignature".digest, + "ansible_collectionversionsignature".pubkey_fingerprint, + "ansible_collectionversionsignature".signing_service_id + FROM ansible_collectionversionsignature + INNER JOIN core_content ON core_content.pulp_id = "ansible_collectionversionsignature".content_ptr_id + """ + + +def signing_services_query(): + return """ + SELECT pulp_id AS uuid, + pulp_created, + pulp_last_updated, + public_key, + name + FROM core_signingservice + """ + + +def collection_stats_query(): + return """ + SELECT pulp_id AS uuid, + content_unit_id AS collection_version_id, + pulp_created, + pulp_last_updated, + ip, + extra_data->>'org_id' AS org_id, + user_agent + FROM ansible_downloadlog + """ \ No newline at end of file diff --git a/galaxy_ng/app/management/commands/analytics/lightspeed/__init__.py b/galaxy_ng/app/management/commands/analytics/lightspeed/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/galaxy_ng/app/management/commands/analytics/lightspeed/collector.py b/galaxy_ng/app/management/commands/analytics/lightspeed/collector.py new file mode 100644 index 0000000000..8908ba6f67 --- /dev/null +++ b/galaxy_ng/app/management/commands/analytics/lightspeed/collector.py @@ -0,0 +1,38 @@ +from galaxy_ng.app.management.commands.analytics.collector import Collector as BaseCollector +from galaxy_ng.app.management.commands.analytics.lightspeed.package import Package + + +class Collector(BaseCollector): + def __init__(self, collection_type, collector_module, logger): + super().__init__( + collection_type=collection_type, collector_module=collector_module, logger=logger + ) + + + + @staticmethod + def _package_class(): + return Package + + def get_last_gathering(self): + return self._last_gathering() + + def _is_shipping_configured(self): + # TODO: need shipping configuration + return True + + def _last_gathering(self): + # doing a full scan database dump + return None + + def _load_last_gathered_entries(self): + # doing a full scan database dump + return {} + + def _save_last_gathered_entries(self, last_gathered_entries): + # doing a full scan database dump + pass + + def _save_last_gather(self): + # doing a full scan database dump + pass diff --git a/galaxy_ng/app/management/commands/analytics/galaxy_collector.py b/galaxy_ng/app/management/commands/analytics/lightspeed/data.py similarity index 83% rename from galaxy_ng/app/management/commands/analytics/galaxy_collector.py rename to galaxy_ng/app/management/commands/analytics/lightspeed/data.py index 4ad8253c89..d2875dfed1 100644 --- a/galaxy_ng/app/management/commands/analytics/galaxy_collector.py +++ b/galaxy_ng/app/management/commands/analytics/lightspeed/data.py @@ -1,55 +1,18 @@ import os -import platform -import distro -from django.conf import settings +from django.db import connection from insights_analytics_collector import CsvFileSplitter, register - -from galaxy_ng.app.management.commands.analytics.collector import Collector +import galaxy_ng.app.management.commands.analytics.common_data as data @register("config", "1.0", description="General platform configuration.", config=True) def config(since, **kwargs): - # TODO: license_info = get_license() - license_info = {} - - return { - "platform": { - "system": platform.system(), - "dist": distro.linux_distribution(), - "release": platform.release(), - "type": "traditional", - }, - "external_logger_enabled": "todo", - "external_logger_type": "todo", - "install_uuid": "todo", - "instance_uuid": "todo", - "tower_url_base": "todo", - "tower_version": "todo", - "logging_aggregators": ["todo"], - "pendo_tracking": "todo", - "hub_url_base": "todo", - "hub_version": "todo", - "license_type": license_info.get("license_type", "UNLICENSED"), - "free_instances": license_info.get("free_instances", 0), - "total_licensed_instances": license_info.get("instance_count", 0), - "license_expiry": license_info.get("time_remaining", 0), - "authentication_backends": settings.AUTHENTICATION_BACKENDS, - } + return data.config() @register("instance_info", "1.0", description="Node information", config=True) def instance_info(since, **kwargs): - # TODO: - - return { - "versions": {"system": "todo"}, - "online_workers": "todo", - "online_content_apps": "todo", - "database_connection": "todo", - "redis_connection": "todo", - "storage": "todo", - } + return data.instance_info() @register("ansible_collection_table", "1.0", format="csv", description="Data on ansible_collection") @@ -222,7 +185,7 @@ def _simple_csv(full_path, file_name, query, max_data_size=209715200): file_path = _get_file_path(full_path, file_name) tfile = _get_csv_splitter(file_path, max_data_size) - with Collector.db_connection().cursor() as cursor: + with connection.cursor() as cursor: with cursor.copy(query) as copy: while data := copy.read(): tfile.write(str(data, 'utf8')) diff --git a/galaxy_ng/app/management/commands/analytics/package.py b/galaxy_ng/app/management/commands/analytics/lightspeed/package.py similarity index 100% rename from galaxy_ng/app/management/commands/analytics/package.py rename to galaxy_ng/app/management/commands/analytics/lightspeed/package.py diff --git a/galaxy_ng/app/management/commands/metrics-collection-automation-analytics.py b/galaxy_ng/app/management/commands/metrics-collection-automation-analytics.py new file mode 100644 index 0000000000..2d714d1b0f --- /dev/null +++ b/galaxy_ng/app/management/commands/metrics-collection-automation-analytics.py @@ -0,0 +1,55 @@ +import logging + +from django.core.management.base import BaseCommand +from galaxy_ng.app.management.commands.analytics.automation_analytics.collector import Collector +from galaxy_ng.app.management.commands.analytics.automation_analytics import data as automation_analytics_data + +logger = logging.getLogger("analytics.export_automation_analytics") + + +class Command(BaseCommand): + help = """Django management command to export collections data to ingress -> automation analytics""" + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', dest='dry-run', action='store_true', help='Gather analytics without shipping' + ) + parser.add_argument( + '--ship', dest='ship', action='store_true', help='Enable to ship metrics to the Red Hat Cloud' + ) + # parser.add_argument('--since', dest='since', action='store', help='Start date for collection') + # parser.add_argument('--until', dest='until', action='store', help='End date for collection') + + def handle(self, *args, **options): + """Handle command""" + + opt_ship = options.get('ship') + opt_dry_run = options.get('dry-run') + + # TODO: since and until not available for the first release + # opt_since = options.get('since') or None + # opt_until = options.get('until') or None + + # since = parser.parse(opt_since) if opt_since else None + # if since and since.tzinfo is None: + # since = since.replace(tzinfo=timezone.utc) + # until = parser.parse(opt_until) if opt_until else None + # if until and until.tzinfo is None: + # until = until.replace(tzinfo=timezone.utc) + + if opt_ship and opt_dry_run: + self.logger.error('Both --ship and --dry-run cannot be processed at the same time.') + return + + collector = Collector( + collector_module=automation_analytics_data, + collection_type=Collector.MANUAL_COLLECTION if opt_ship else Collector.DRY_RUN, + logger=logger + ) + + tgzfiles = collector.gather() + if tgzfiles: + for tgz in tgzfiles: + self.stdout.write(tgz) + else: + self.stdout.write("No analytics tarballs collected") diff --git a/galaxy_ng/app/management/commands/analytics-export-s3.py b/galaxy_ng/app/management/commands/metrics-collection-lightspeed.py similarity index 55% rename from galaxy_ng/app/management/commands/analytics-export-s3.py rename to galaxy_ng/app/management/commands/metrics-collection-lightspeed.py index c1a3dfff74..fa2d60497c 100644 --- a/galaxy_ng/app/management/commands/analytics-export-s3.py +++ b/galaxy_ng/app/management/commands/metrics-collection-lightspeed.py @@ -1,11 +1,11 @@ import logging from django.core.management.base import BaseCommand -from galaxy_ng.app.management.commands.analytics.collector import Collector -from galaxy_ng.app.management.commands.analytics import galaxy_collector +from galaxy_ng.app.management.commands.analytics.lightspeed.collector import Collector +from galaxy_ng.app.management.commands.analytics.lightspeed import data as lightspeed_data from django.utils.timezone import now, timedelta -logger = logging.getLogger("analytics") +logger = logging.getLogger("analytics.export_lightspeed") class Command(BaseCommand): @@ -15,14 +15,14 @@ def handle(self, *args, **options): """Handle command""" collector = Collector( - collector_module=galaxy_collector, - collection_type="manual", + collector_module=lightspeed_data, + collection_type=Collector.MANUAL_COLLECTION, logger=logger, ) collector.gather(since=now() - timedelta(days=8), until=now() - timedelta(days=1)) - print("Completed ") + self.stdout.write("Gather Analytics => S3(Lightspeed): Completed ") if __name__ == "__main__": diff --git a/galaxy_ng/tests/unit/app/management/commands/test_analytics_export_s3.py b/galaxy_ng/tests/unit/app/management/commands/test_analytics_export_lightspeed.py similarity index 88% rename from galaxy_ng/tests/unit/app/management/commands/test_analytics_export_s3.py rename to galaxy_ng/tests/unit/app/management/commands/test_analytics_export_lightspeed.py index 137f16094c..d88ede7845 100644 --- a/galaxy_ng/tests/unit/app/management/commands/test_analytics_export_s3.py +++ b/galaxy_ng/tests/unit/app/management/commands/test_analytics_export_lightspeed.py @@ -12,12 +12,12 @@ } -class TestAnalyticsExportS3Command(TestCase): +class TestMetricsCollectionLightspeedCommand(TestCase): def setUp(self): super().setUp() def test_command_output(self): - call_command("analytics-export-s3") + call_command("metrics-collection-lightspeed") @patch("galaxy_ng.app.management.commands.analytics.galaxy_collector._get_csv_splitter") @patch("builtins.open", new_callable=mock_open, read_data="data") @@ -34,7 +34,7 @@ def test_write_file_to_s3_success(self, boto3, mock_file, simpleCSVHelper): csvsplitter.file_list = MagicMock(name="file_list") simpleCSVHelper.return_value = csvsplitter - call_command("analytics-export-s3") + call_command("metrics-collection-lightspeed") simpleCSVHelper.assert_called() csvsplitter.file_list.assert_called()