diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 09bd51cfbe36c..01074304ea372 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -309,6 +309,11 @@ datadog_checks_base/datadog_checks/base/checks/windows/ @DataDog/wi /sophos_central_cloud/manifest.json @DataDog/saas-integrations @DataDog/documentation /sophos_central_cloud/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core +/squid/ @DataDog/saas-integrations +/squid/*.md @DataDog/saas-integrations @DataDog/documentation +/squid/manifest.json @DataDog/saas-integrations @DataDog/documentation +/squid/assets/logs/ @DataDog/saas-integrations @DataDog/documentation @DataDog/logs-backend + /ping_one/ @DataDog/saas-integrations /ping_one/*.md @DataDog/saas-integrations @DataDog/documentation /ping_one/manifest.json @DataDog/saas-integrations @DataDog/documentation diff --git a/squid/CHANGELOG.md b/squid/CHANGELOG.md index 93880d788518b..ba4f1a205f1b5 100644 --- a/squid/CHANGELOG.md +++ b/squid/CHANGELOG.md @@ -236,4 +236,4 @@ [1]: https://github.com/DataDog/integrations-core/pull/2788 [2]: https://github.com/DataDog/integrations-core/pull/1727 [3]: https://github.com/DataDog/integrations-core/pull/1643 -[4]: https://github.com/mnussbaum +[4]: https://github.com/mnussbaum \ No newline at end of file diff --git a/squid/README.md b/squid/README.md index fe1c00ce06817..7aeda3a54b0a4 100644 --- a/squid/README.md +++ b/squid/README.md @@ -1,6 +1,17 @@ # Squid Integration ## Overview +[Squid][1] is an open-source caching and forwarding web proxy server that operates as an intermediary between clients and servers on a network. It acts as a gateway, enabling clients to access various internet resources such as websites, files, and other content from servers. + +This integration provides enrichment and visualization for Squid logs. It helps you visualize detailed insights into Squid log analysis through the out-of-the-box dashboards and detection rules, enhancing detection and response capabilities. + +Additionally, it includes pre-configured monitors for proactive notifications on the following: + +1. High rate of server errors +2. CPU usage exceeded +3. High latency requests +4. High rate of client HTTP errors + This check monitors [Squid][1] metrics from the Cache Manager through the Datadog Agent. @@ -87,6 +98,38 @@ Collecting logs is disabled by default in the Datadog Agent. To enable it, see [ ## Data Collected +### Logs +The Squid integration collects access and cache logs. + +#### Supported Access Log Formats +|Name | Format Specification| +|---------------------|------------------------------| +| squid |`%ts.%03tu %6tr %>a %Ss/%03>Hs %a - %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %a - %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %h" "%{User-Agent}>h" %Ss:%Sh`| + +For more information, refer to [Squid log formats][12]. + +**Note**: The default `logformat` type is `squid`. You can update the supported log format in `/etc/squid/squid.conf`, then restart Squid. + +To use the `combined` type for `logformat`, add the following lines to your `/etc/squid/squid.conf` file: + +``` +logformat combined %>a %[ui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %h" "%{User-Agent}>h" %Ss:%Sh +access_log /var/log/squid/access.log combined +``` +Next, restart the `squid` service using the following command: + +```shell +sudo systemctl restart squid +``` + +**Note**: + +- The `Top Avg Request Duration by URL Host` panel will be loaded only if the default `squid` type of `logformat` is configured. +- The `Top Browsers` and `Top HTTP Referrer` panels will be loaded only if the `combined` type of `logformat` is configured. + + ### Metrics See [metadata.csv][9] for a list of metrics provided by this check. @@ -115,3 +158,4 @@ Need help? Contact [Datadog support][11]. [9]: https://github.com/DataDog/integrations-core/blob/master/squid/metadata.csv [10]: https://github.com/DataDog/integrations-core/blob/master/squid/assets/service_checks.json [11]: https://docs.datadoghq.com/help/ +[12]: https://www.squid-cache.org/Doc/config/logformat/ \ No newline at end of file diff --git a/squid/assets/dashboards/squid.json b/squid/assets/dashboards/squid.json new file mode 100644 index 0000000000000..43f96c9d56629 --- /dev/null +++ b/squid/assets/dashboards/squid.json @@ -0,0 +1,3023 @@ +{ + "description": "This dashboard provides information about the Squid logs generated in Squid Proxy server.", + "layout_type": "ordered", + "notify_list": [ + ], + "reflow_type": "fixed", + "template_variables": [ + { + "available_values": [ + "TCP", + "UDP" + ], + "default": "*", + "name": "Protocol", + "prefix": "@network.protocol" + }, + { + "available_values": [ + ], + "default": "*", + "name": "User", + "prefix": "@usr.name" + }, + { + "available_values": [ + "200", + "400", + "403", + "407", + "500" + ], + "default": "*", + "name": "Status_Code", + "prefix": "@http.status_code" + }, + { + "available_values": [ + "Success", + "Notice", + "Warning", + "Error", + "Critical" + ], + "default": "*", + "name": "Status_Category", + "prefix": "@http.status_category" + }, + { + "available_values": [ + "DENIED", + "MISS", + "HIT" + ], + "default": "*", + "name": "Squid_Status", + "prefix": "@squid.status" + } + ], + "title": "Squid", + "widgets": [ + { + "definition": { + "has_background": true, + "has_border": true, + "horizontal_align": "center", + "sizing": "contain", + "type": "image", + "url": "https://www.squid-cache.org/Artwork/Banner.png", + "url_dark_theme": "https://www.squid-cache.org/Artwork/Banner.png", + "vertical_align": "center" + }, + "id": 993530611301326, + "layout": { + "height": 2, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "definition": { + "background_color": "vivid_blue", + "layout_type": "ordered", + "show_title": true, + "title": "Squid Monitors Summary", + "type": "group", + "widgets": [ + { + "definition": { + "color_preference": "text", + "count": 50, + "display_format": "countsAndList", + "hide_zero_counts": true, + "last_triggered_format": "relative", + "query": "tag:squid", + "show_last_triggered": false, + "show_priority": false, + "show_status": true, + "sort": "status,asc", + "start": 0, + "summary_type": "monitors", + "title": "Squid Monitors Summary", + "type": "manage_status" + }, + "id": 5292238472737108, + "layout": { + "height": 5, + "width": 6, + "x": 0, + "y": 0 + } + } + ] + }, + "id": 6105759479053774, + "layout": { + "height": 6, + "width": 6, + "x": 6, + "y": 0 + } + }, + { + "definition": { + "background_color": "white", + "content": "Gain a comprehensive view of your network's web and cache activities by monitoring detailed Squid Access logs through this dashboard.\n\nSquid Access Logs provide in-depth insights into web access patterns, such as Total Squid Logs and Logs Over Time. Analyze Top Client IPs and Top Users by Error Status Category to identify which clients and users are experiencing the most issues. \n\nOperational metrics provides a comprehensive overview of server performance, caching effectiveness, and client behavior. This helps you monitor key metrics, identify potential issues, and optimize your caching configuration. The dashboard covers CPU and memory usage, HTTP requests and errors, and FTP traffic.\n\nFor more information, see the [Squid Integration Documentation](https://docs.datadoghq.com/integrations/squid)\n \n**Tips**\n- Use the timeframe selector in the top right of the dashboard to change the default timeframe.\n- Clone this dashboard to rearrange, modify and add widgets and visualizations.", + "font_size": "14", + "has_padding": true, + "show_tick": false, + "text_align": "left", + "tick_edge": "left", + "tick_pos": "50%", + "type": "note", + "vertical_align": "top" + }, + "id": 6404359088381374, + "layout": { + "height": 4, + "width": 6, + "x": 0, + "y": 2 + } + }, + { + "definition": { + "background_color": "vivid_orange", + "layout_type": "ordered", + "show_title": true, + "title": "Datadog Cloud SIEM", + "title_align": "center", + "type": "group", + "widgets": [ + { + "definition": { + "background_color": "vivid_blue", + "content": "\nDatadog Cloud SIEM analyzes and correlates Squid logs to detect threats to your environment in real time. If you don't see signals, make sure you've enabled [Datadog Cloud SIEM](/security?query=source%3Asquid). ", + "font_size": "14", + "has_padding": true, + "show_tick": false, + "text_align": "left", + "tick_edge": "left", + "tick_pos": "50%", + "type": "note", + "vertical_align": "center" + }, + "id": 8135504331313588, + "layout": { + "height": 1, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "definition": { + "autoscale": true, + "custom_links": [ + ], + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#bc303c", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:critical source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "CRITICALs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 5242677795523096, + "layout": { + "height": 2, + "width": 2, + "x": 0, + "y": 1 + } + }, + { + "definition": { + "autoscale": true, + "custom_links": [ + ], + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#e5a21c", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:medium source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "MEDIUMs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 4007960417434418, + "layout": { + "height": 2, + "width": 2, + "x": 2, + "y": 1 + } + }, + { + "definition": { + "custom_links": [ + ], + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#bc303c", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + { + "facet": "@workflow.rule.name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:critical source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + }, + "title": "Critical Security Signals", + "type": "toplist" + }, + "id": 8976826436340736, + "layout": { + "height": 4, + "width": 8, + "x": 4, + "y": 1 + } + }, + { + "definition": { + "autoscale": true, + "custom_links": [ + ], + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#d33043", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:high source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "HIGHs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 1502810820524424, + "layout": { + "height": 2, + "width": 2, + "x": 0, + "y": 3 + } + }, + { + "definition": { + "autoscale": true, + "custom_links": [ + ], + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#ffb52b", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:low source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "LOWs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 6966298510286182, + "layout": { + "height": 1, + "width": 2, + "x": 2, + "y": 3 + } + }, + { + "definition": { + "autoscale": true, + "custom_links": [ + ], + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#84c1e0", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:info source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "INFOs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 57798377027784, + "layout": { + "height": 1, + "width": 2, + "x": 2, + "y": 4 + } + }, + { + "definition": { + "custom_links": [ + ], + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#d33043", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + { + "facet": "@workflow.rule.name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:high source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + }, + "title": "High Security Signals", + "type": "toplist" + }, + "id": 8289651080064226, + "layout": { + "height": 4, + "width": 6, + "x": 0, + "y": 5 + } + }, + { + "definition": { + "custom_links": [ + ], + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#e5a21c", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "security_signals", + "group_by": [ + { + "facet": "@workflow.rule.name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "status:medium source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + } + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + }, + "title": "Medium Security Signals", + "type": "toplist" + }, + "id": 3439516698249068, + "layout": { + "height": 4, + "width": 6, + "x": 6, + "y": 5 + } + } + ] + }, + "id": 3255980586575102, + "layout": { + "height": 1, + "width": 12, + "x": 0, + "y": 6 + } + }, + { + "definition": { + "background_color": "vivid_green", + "layout_type": "ordered", + "show_title": true, + "title": "Squid Log Details", + "type": "group", + "widgets": [ + { + "definition": { + "autoscale": true, + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#dbdef5", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "title": "Total Squid Logs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 2544834834046598, + "layout": { + "height": 3, + "width": 3, + "x": 0, + "y": 0 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "horizontal", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "alias": "Access Logs", + "formula": "query1", + "style": { + "palette": "classic", + "palette_index": 1 + } + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "Squid Logs over time", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 6929593419963792, + "layout": { + "height": 4, + "width": 9, + "x": 3, + "y": 0 + } + }, + { + "definition": { + "autoscale": true, + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "palette": "white_on_green", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte" + }, + "unit_scale": { + "type": "canonical_unit", + "unit_name": "byte" + } + } + } + ], + "queries": [ + { + "compute": { + "aggregation": "sum", + "metric": "@network.bytes_written" + }, + "data_source": "logs", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "Total Bytes Sent to Client", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 4301951353466508, + "layout": { + "height": 3, + "width": 3, + "x": 0, + "y": 3 + } + }, + { + "definition": { + "legend": { + "type": "table" + }, + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.status_category", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "title": "Distribution by Status Category ", + "title_align": "left", + "title_size": "16", + "type": "sunburst" + }, + "id": 5210794857327700, + "layout": { + "height": 4, + "width": 9, + "x": 3, + "y": 4 + } + }, + { + "definition": { + "autoscale": true, + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#db3333", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid @squid.status:DENIED @network.protocol:TCP $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "Total TCP Denied Logs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 1552732595455472, + "layout": { + "height": 3, + "width": 3, + "x": 0, + "y": 6 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "avg", + "metric": "@duration" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.url_details.host", + "limit": 10, + "sort": { + "aggregation": "avg", + "metric": "@duration", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + }, + "palette": "dog_classic" + }, + "title": "Top Avg Request Duration by URL Host", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 1821045533603400, + "layout": { + "height": 4, + "width": 9, + "x": 3, + "y": 8 + } + }, + { + "definition": { + "autoscale": true, + "precision": 2, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#db3333", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid @squid.status:DENIED @network.protocol:UDP $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area" + }, + "title": "Total UDP Denied Logs", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 7196619253169032, + "layout": { + "height": 3, + "width": 3, + "x": 0, + "y": 9 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "auto", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@squid.status", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "Squid Status Over Time", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 4592726907280998, + "layout": { + "height": 4, + "width": 6, + "x": 0, + "y": 12 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "auto", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@squid.peer_status", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "Squid Peer Status Over Time", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 6060795960521114, + "layout": { + "height": 4, + "width": 6, + "x": 6, + "y": 12 + } + }, + { + "definition": { + "legend": { + "type": "table" + }, + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@network.protocol", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "title": "Distribution by Protocol", + "title_align": "left", + "title_size": "16", + "type": "sunburst" + }, + "id": 452983347022802, + "layout": { + "height": 4, + "width": 6, + "x": 0, + "y": 16 + } + }, + { + "definition": { + "legend": { + "type": "table" + }, + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.method", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + }, + "style": { + "palette": "datadog16" + } + } + ], + "title": "Distribution by HTTP Methods", + "title_align": "left", + "title_size": "16", + "type": "sunburst" + }, + "id": 2209657631600598, + "layout": { + "height": 4, + "width": 6, + "x": 6, + "y": 16 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.useragent_details.browser.family", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Browsers", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 6753071570035770, + "layout": { + "height": 5, + "width": 4, + "x": 0, + "y": 20 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@network.client.ip", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Client IPs", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 6865073291942784, + "layout": { + "height": 5, + "width": 4, + "x": 4, + "y": 20 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@network.destination.ip", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Destination IPs", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 8115192646989834, + "layout": { + "height": 5, + "width": 4, + "x": 8, + "y": 20 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@network.client.ip", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + }, + { + "facet": "@http.status_category", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid @http.status_code:>=400 $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 100, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Client IPs by Error Status Category", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 374174089069424, + "layout": { + "height": 5, + "width": 6, + "x": 0, + "y": 25 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.url_details.host", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Sites", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 2144528520612260, + "layout": { + "height": 5, + "width": 6, + "x": 6, + "y": 25 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@usr.name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + }, + { + "facet": "@http.status_category", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid @http.status_code:>=400 $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 100, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Users by Error Status Category", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 2586504510271436, + "layout": { + "height": 5, + "width": 6, + "x": 0, + "y": 30 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@usr.name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Users", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 5832480649610576, + "layout": { + "height": 5, + "width": 6, + "x": 6, + "y": 30 + } + }, + { + "definition": { + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#8c3131", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.url_details.host", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid @squid.status:DENIED $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Denied URL Host ", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 2363694544637748, + "layout": { + "height": 5, + "width": 6, + "x": 0, + "y": 35 + } + }, + { + "definition": { + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#e29d3c", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.url_details.host", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid @squid.status:MISS $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Missed URL Host ", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 6502090782347636, + "layout": { + "height": 5, + "width": 6, + "x": 6, + "y": 35 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@http.referer", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top HTTP Referrer", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 760898709982526, + "layout": { + "height": 5, + "width": 6, + "x": 0, + "y": 40 + } + }, + { + "definition": { + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#a63f3f", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@error.message", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "display": { + "legend": "automatic", + "type": "stacked" + } + }, + "title": "Top Error Messages", + "title_align": "left", + "title_size": "16", + "type": "toplist" + }, + "id": 4376134812588716, + "layout": { + "height": 5, + "width": 6, + "x": 6, + "y": 40 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@network.client.geoip.country.iso_code", + "limit": 250, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 250, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "palette": "hostmap_blues", + "palette_flip": false + }, + "title": "Traffic Over Client Geoip Location", + "title_align": "left", + "title_size": "16", + "type": "geomap", + "view": { + "focus": "WORLD" + } + }, + "id": 6204659465091390, + "layout": { + "height": 5, + "width": 12, + "x": 0, + "y": 45 + } + }, + { + "definition": { + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "compute": { + "aggregation": "count" + }, + "data_source": "logs", + "group_by": [ + { + "facet": "@network.destination.geoip.country.iso_code", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "indexes": [ + "*" + ], + "name": "query1", + "search": { + "query": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "index": 0, + "order": "desc", + "type": "formula" + } + ] + } + } + ], + "style": { + "palette": "hostmap_blues", + "palette_flip": false + }, + "title": "Traffic Over Destination Geoip Location", + "title_align": "left", + "title_size": "16", + "type": "geomap", + "view": { + "focus": "WORLD" + } + }, + "id": 6079251722162984, + "layout": { + "height": 5, + "width": 12, + "x": 0, + "y": 50 + } + }, + { + "definition": { + "requests": [ + { + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "timestamp", + "width": "auto" + }, + { + "field": "host", + "width": "auto" + }, + { + "field": "@network.client.ip", + "width": "auto" + }, + { + "field": "@http.status_category", + "width": "auto" + }, + { + "field": "@http.status_code", + "width": "auto" + }, + { + "field": "@network.protocol", + "width": "auto" + }, + { + "field": "@network.bytes_written", + "width": "auto" + }, + { + "field": "http.url_details.host", + "width": "auto" + } + ], + "query": { + "data_source": "logs_stream", + "indexes": [ + ], + "query_string": "source:squid $Protocol $User $Status_Code $Status_Category $Squid_Status ", + "storage": "hot" + }, + "response_format": "event_list" + } + ], + "title": "Squid Log Details", + "title_align": "left", + "title_size": "16", + "type": "list_stream" + }, + "id": 8589078903940764, + "layout": { + "height": 6, + "width": 12, + "x": 0, + "y": 55 + } + } + ] + }, + "id": 2539290368467106, + "layout": { + "height": 62, + "width": 12, + "x": 0, + "y": 7 + } + }, + { + "definition": { + "background_color": "blue", + "layout_type": "ordered", + "show_title": true, + "title": "Squid Metric Details", + "type": "group", + "widgets": [ + { + "definition": { + "autoscale": true, + "precision": 0, + "requests": [ + { + "conditional_formats": [ + { + "comparator": "<", + "palette": "white_on_green", + "value": 80 + }, + { + "comparator": ">=", + "palette": "black_on_light_red", + "value": 80 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.cpu_time{*}" + } + ], + "response_format": "scalar" + } + ], + "title": "CPU Usage ", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 3820300543090186, + "layout": { + "height": 3, + "width": 4, + "x": 0, + "y": 0 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "horizontal", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "alias": "CPU Usage", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.cpu_time{*}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "time": { + "hide_incomplete_cost_data": true + }, + "title": "CPU Usage Over Time", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 6415929501862064, + "layout": { + "height": 3, + "width": 8, + "x": 4, + "y": 0 + } + }, + { + "definition": { + "autoscale": true, + "precision": 0, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">=", + "palette": "white_on_green", + "value": 0 + }, + { + "comparator": ">", + "palette": "white_on_yellow", + "value": 750 + }, + { + "comparator": ">=", + "palette": "black_on_light_red", + "value": 1000 + } + ], + "formulas": [ + { + "formula": "query1 + query2" + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.cd.memory{*}" + }, + { + "aggregator": "last", + "data_source": "metrics", + "name": "query2", + "query": "sum:squid.cachemgr.cd.local_memory{*}" + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area" + }, + "title": "Memory Usage ", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 4258165939341752, + "layout": { + "height": 3, + "width": 4, + "x": 0, + "y": 3 + } + }, + { + "definition": { + "autoscale": true, + "precision": 0, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">", + "custom_bg_color": "#26b0df", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1 + query2", + "number_format": { + "unit": { + "per_unit_name": "second", + "type": "canonical_unit", + "unit_name": "request" + } + } + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.client_http.requests{*}" + }, + { + "aggregator": "last", + "data_source": "metrics", + "name": "query2", + "query": "sum:squid.cachemgr.server.all.requests{*}" + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "HTTP Requests Per Second", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 2674308426925946, + "layout": { + "height": 3, + "width": 4, + "x": 4, + "y": 3 + } + }, + { + "definition": { + "autoscale": true, + "precision": 0, + "requests": [ + { + "conditional_formats": [ + { + "comparator": ">=", + "custom_bg_color": "#e66833", + "palette": "custom_bg", + "value": 0 + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "aggregator": "last", + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.aborted_requests{*}" + } + ], + "response_format": "scalar" + } + ], + "timeseries_background": { + "type": "area", + "yaxis": { + "include_zero": true + } + }, + "title": "Aborted Requests Per Second", + "title_align": "left", + "title_size": "16", + "type": "query_value" + }, + "id": 2786611055040880, + "layout": { + "height": 3, + "width": 4, + "x": 8, + "y": 3 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "horizontal", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "alias": "Client HTTP Requests", + "formula": "query1", + "style": { + "palette": "classic", + "palette_index": 1 + } + }, + { + "alias": "Client HTTP Hits", + "formula": "query2" + }, + { + "alias": "Client Hits Rate", + "formula": "query2 / query1 * 100" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.client_http.requests{*}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:squid.cachemgr.client_http.hits{*}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "Client HTTP Requests vs HTTP Hits Over Time", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 2069184515494228, + "layout": { + "height": 3, + "width": 6, + "x": 0, + "y": 6 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "auto", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "alias": "Client HTTP Errors", + "formula": "query1", + "style": { + "palette": "warm", + "palette_index": 5 + } + }, + { + "alias": "Server HTTP Errors", + "formula": "query2", + "style": { + "palette": "purple", + "palette_index": 5 + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.client_http.errors{*}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:squid.cachemgr.server.all.errors{*}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "Client Errors vs Server Errors ", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 6482725862183798, + "layout": { + "height": 3, + "width": 6, + "x": 6, + "y": 6 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "horizontal", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "alias": "Traffic Read", + "formula": "query1", + "style": { + "palette": "warm", + "palette_index": 3 + } + }, + { + "alias": "Traffic Write", + "formula": "query2", + "style": { + "palette": "classic", + "palette_index": 3 + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.server.all.kbytes_in{*}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:squid.cachemgr.server.all.kbytes_out{*}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "time": { + "hide_incomplete_cost_data": true + }, + "title": "Squid Cache Server Traffic Read Vs Traffic Write", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 3053023205098324, + "layout": { + "height": 3, + "width": 6, + "x": 0, + "y": 9 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "horizontal", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "alias": "Server HTTP Requests", + "formula": "query1" + }, + { + "alias": "Traffic Read", + "formula": "query2" + }, + { + "alias": "Traffic Write", + "formula": "query3" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.server.http.requests{*}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:squid.cachemgr.server.http.kbytes_in{*}" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:squid.cachemgr.server.http.kbytes_out{*}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "title": "HTTP Traffic Analysis", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 1805246720143070, + "layout": { + "height": 3, + "width": 6, + "x": 6, + "y": 9 + } + }, + { + "definition": { + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "legend_layout": "auto", + "requests": [ + { + "display_type": "line", + "formulas": [ + { + "alias": "FTP Requests", + "formula": "query1", + "style": { + "palette": "dd20", + "palette_index": 3 + } + }, + { + "alias": "FTP Errors", + "formula": "query2", + "style": { + "palette": "dd20", + "palette_index": 4 + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query1", + "query": "sum:squid.cachemgr.server.ftp.requests{*}" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "sum:squid.cachemgr.server.ftp.errors{*}" + } + ], + "response_format": "timeseries", + "style": { + "line_type": "solid", + "line_width": "normal", + "order_by": "values", + "palette": "dog_classic" + } + } + ], + "show_legend": true, + "time": { + "hide_incomplete_cost_data": true + }, + "title": "FTP Requests vs FTP Errors Over Time", + "title_align": "left", + "title_size": "16", + "type": "timeseries" + }, + "id": 1932830340289702, + "layout": { + "height": 3, + "width": 12, + "x": 0, + "y": 12 + } + } + ] + }, + "id": 4126822935793754, + "layout": { + "height": 16, + "width": 12, + "x": 0, + "y": 69 + } + } + ] +} \ No newline at end of file diff --git a/squid/assets/logs/squid.yaml b/squid/assets/logs/squid.yaml index a57a2af49647d..50510e2ed3ae4 100644 --- a/squid/assets/logs/squid.yaml +++ b/squid/assets/logs/squid.yaml @@ -32,36 +32,134 @@ facets: name: URL scheme path: http.url_details.scheme source: log + - groups: + - Web Access + name: Browser + path: http.useragent_details.browser.family + source: log + - groups: + - Web Access + name: Device + path: http.useragent_details.device.family + source: log + - groups: + - Web Access + name: OS + path: http.useragent_details.os.family + source: log + - groups: + - Geoip + name: City Name + path: network.client.geoip.city.name + source: log + - groups: + - Geoip + name: Continent Code + path: network.client.geoip.continent.code + source: log + - groups: + - Geoip + name: Continent Name + path: network.client.geoip.continent.name + source: log + - groups: + - Geoip + name: Country ISO Code + path: network.client.geoip.country.iso_code + source: log + - groups: + - Geoip + name: Country Name + path: network.client.geoip.country.name + source: log + - groups: + - Geoip + name: Subdivision ISO Code + path: network.client.geoip.subdivision.iso_code + source: log + - groups: + - Geoip + name: Subdivision Name + path: network.client.geoip.subdivision.name + source: log - groups: - Web Access name: Client IP path: network.client.ip source: log + - groups: + - Geoip + name: Destination City Name + path: network.destination.geoip.city.name + source: log + - groups: + - Geoip + name: Destination Continent Code + path: network.destination.geoip.continent.code + source: log + - groups: + - Geoip + name: Destination Continent Name + path: network.destination.geoip.continent.name + source: log + - groups: + - Geoip + name: Destination Country ISO Code + path: network.destination.geoip.country.iso_code + source: log + - groups: + - Geoip + name: Destination Country Name + path: network.destination.geoip.country.name + source: log + - groups: + - Geoip + name: Destination Subdivision ISO Code + path: network.destination.geoip.subdivision.iso_code + source: log + - groups: + - Geoip + name: Destination Subdivision Name + path: network.destination.geoip.subdivision.name + source: log - groups: - Web Access name: Destination IP path: network.destination.ip source: log - groups: + - User + name: User Name + path: usr.name + source: log + - facetType: list + groups: - Network name: Network Protocol path: network.protocol source: log - - groups: + type: string + - facetType: list + groups: - Squid - name: Status - path: squid.status + name: Instance Name + path: squid.instance_name source: log - - groups: + type: string + - facetType: list + groups: - Squid name: Peer Status path: squid.peer_status source: log - - groups: + type: string + - facetType: list + groups: - Squid - name: Instance Name - path: squid.instance_name + name: Status + path: squid.status source: log + type: string pipeline: type: pipeline name: Squid @@ -79,13 +177,44 @@ pipeline: - 1570784722.610 196 127.0.0.1 TCP_MISS/200 12712 GET http://www.google.com/ - HIER_DIRECT/172.217.20.68 text/html - 1570784681.846 12140 127.0.0.1 TCP_MISS_ABORTED/000 0 GET http://8.8.8.8/ - HIER_DIRECT/8.8.8.8 - - 2019/10/11 09:14:49 kid1| Accepting HTTP Socket connections at local=[::]:3128 remote=[::] FD 11 flags=9 + - 1725599181.650 0 10.212.128.16 TCP_MISS_ABORTED/000 0 - error:transaction-end-before-headers - HIER_NONE/- - + - 1725599185.240 0 10.212.128.16 TCP_DENIED/407 4660 GET http://clientservices.googleapis.com/chrome-variations/seed? - HIER_NONE/- text/html [http_request_headers] [http_response_headers] + - 10.212.128.16 - - [06/Sep/2024:10:32:43 +0530] "CONNECT fcmconnection.googleapis.com:443 HTTP/1.1" 407 4053 TCP_DENIED:HIER_NONE + - 10.212.128.16 - - [06/Sep/2024:10:34:44 +0530] "CONNECT mobile.events.data.microsoft.com:443 HTTP/1.1" 407 4277 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Code/1.92.2 Chrome/124.0.6367.243 Electron/30.1.2 Safari/537.36" TCP_DENIED:HIER_NONE + - 2019/10/11 09:14:49 kid1| Accepting HTTP Socket connections at local=[::]:3128 remote=[::] FD 11 flags=9 grok: - matchRules: | - squid_access_parser %{number:timestamp:scale(1000)}\s+%{number:duration:scale(1000000)} %{ip:network.client.ip} %{regex("[^_]*"):network.protocol}_%{regex("[^/]*"):squid.status}/(000|%{integer:http.status_code}) %{number:network.bytes_written} %{word:http.method} (%{regex("(http|https)://[^\\s]+"):http.url}|%{notSpace:http.url_details.path}) (-|%{notSpace:squid.user_identity}) %{word:squid.peer_status}/(-|%{ip:network.destination.ip}) (:|-|%{notSpace:http.resource.content_type})?( \[%{regex("[^]]*"):http.headers.request}\])?( \[%{regex("[^]]*"):http.headers.response}\])?.* + supportRules: > + parse_prefix_combined_log_rule (-|%{ip:network.client.ip}) - + (-|%{word:squid.user_identity}) \[%{date("dd/MMM/yyyy:HH:mm:ss + Z"):timestamp}\] \"(-|%{word:http.method}) + (-|%{regex("(http|https)://[^\\s]+"):http.url}|error:%{data:error.message}|%{parse_url}) + %{regex("[^\"]*"):http.version}\" (-|%{number:http.status_code}) + %{number:network.bytes_written} + + parse_suffix_combined_log_rule (NONE|%{regex("[^_]*"):network.protocol}_%{regex("[^:]*"):squid.status})\:(-|%{word:squid.peer_status}).* + + parse_url %{hostname:http.url_details.host}(:%{number:http.url_details.port})?(%{regex("[^?|^#]*"):http.url_details.path})?(\?%{regex("[^#]*"):http.url_details.queryString:keyvalue("=","+%","","&")})?(\#%{notSpace:http.url_details.hash})? + matchRules: > + squid_access_parser + %{number:timestamp:scale(1000)}\s+(-|%{number:duration:scale(1000000)}) + (-|%{ip:network.client.ip}) + %{regex("[^_]*"):network.protocol}_%{regex("[^/]*"):squid.status}/(000|%{integer:http.status_code}) + (-|%{number:network.bytes_written}) (-|%{word:http.method}) + (-|%{regex("(http|https)://[^\\s]+"):http.url}|error:%{data:error.message}|%{parse_url}) + (-|%{notSpace:squid.user_identity}) + (-|%{word:squid.peer_status})/(-|%{ip:network.destination.ip}) + (:|-|%{notSpace:http.resource.content_type})( + \[%{regex("[^]]*"):http.headers.request}\])?( + \[%{regex("[^]]*"):http.headers.response}\])?.* + + combined_access_parser %{parse_prefix_combined_log_rule} \"(-|%{regex("[^\"]*"):http.referer})\" \"(-|%{regex("[^\"]*"):http.useragent})\" %{parse_suffix_combined_log_rule}.* + + common_access_parser %{parse_prefix_combined_log_rule} %{parse_suffix_combined_log_rule}.* squid_cache_parser %{date("yyyy/MM/dd HH:mm:ss"):timestamp}( %{word:squid.instance_name})?\| %{data:squid.properties:keyvalue("=","\\[\\]:")} #Extra samples: + #Access.log #1570784659.019 114 127.0.0.1 TCP_MISS/503 4114 GET http://www.data/ - HIER_NONE/- text/html #1570784685.245 62 127.0.0.1 TCP_MISS/403 1380 GET http://99.86.88.128 - HIER_DIRECT/99.86.88.128 text/html @@ -93,23 +222,24 @@ pipeline: #1570796039.630 217 127.0.0.1 TCP_TUNNEL/200 4094 CONNECT 8.8.8.8:443/ - HIER_DIRECT/8.8.8.8 - #1570788669.732 95 127.0.0.1 TCP_MISS/502 4006 GET http://www.google.com:443/ - HIER_DIRECT/172.217.20.68 text/html #1570797241.241 0 127.0.0.1 TCP_DENIED/403 3903 GET http://http:8/ - HIER_NONE/- text/html + #Cache.log #2019/10/14 17:42:41 kid1| Logfile: opening log daemon:/var/log/squid/access.log #2019/10/11 09:14:49| pinger: Initialising ICMP pinger ... #Access, Cache: in Log Samples - supportRules: '' - type: date-remapper name: Define `timestamp` as the official date of the log enabled: true sources: - timestamp - type: url-parser - name: Access logs - URL parser + name: Extract details from `http.url` enabled: true sources: - http.url target: http.url_details + normalizeEndingSlashes: false - type: category-processor name: Access logs - HTTP Status category processor enabled: true @@ -138,3 +268,35 @@ pipeline: enabled: true sources: - http.status_category + - type: attribute-remapper + name: Map `squid.user_identity` to `usr.name` + enabled: true + sources: + - squid.user_identity + sourceType: attribute + target: usr.name + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: user-agent-parser + name: Extract details from `http.useragent` + enabled: true + sources: + - http.useragent + target: http.useragent_details + encoded: false + combineVersionDetails: false + - type: geo-ip-parser + name: Extracting Geo Location from Source IP Address + enabled: true + sources: + - network.client.ip + target: network.client.geoip + ip_processing_behavior: do-nothing + - type: geo-ip-parser + name: Extracting Geo Location from Destination IP Address + enabled: true + sources: + - network.destination.ip + target: network.destination.geoip + ip_processing_behavior: do-nothing diff --git a/squid/assets/logs/squid_tests.yaml b/squid/assets/logs/squid_tests.yaml index b970e88276644..422f1cd8887a4 100644 --- a/squid/assets/logs/squid_tests.yaml +++ b/squid/assets/logs/squid_tests.yaml @@ -20,8 +20,10 @@ tests: network: bytes_written: 491.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "99.86.88.89" protocol: "TCP" squid: @@ -53,8 +55,10 @@ tests: network: bytes_written: 491.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "99.86.88.128" protocol: "TCP" squid: @@ -85,6 +89,7 @@ tests: network: bytes_written: 4114.0 client: + geoip: {} ip: "127.0.0.1" protocol: "TCP" squid: @@ -111,8 +116,10 @@ tests: network: bytes_written: 0.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "8.8.8.8" protocol: "TCP" squid: @@ -142,8 +149,10 @@ tests: network: bytes_written: 1380.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "99.86.88.128" protocol: "TCP" squid: @@ -174,8 +183,10 @@ tests: network: bytes_written: 12712.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "172.217.20.68" protocol: "TCP" squid: @@ -197,12 +208,15 @@ tests: status_category: "Success" status_code: 200 url_details: - path: "www.google.com:443" + host: "www.google.com" + port: 443 network: bytes_written: 15851.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "172.217.20.68" protocol: "TCP" squid: @@ -224,12 +238,16 @@ tests: status_category: "Success" status_code: 200 url_details: - path: "8.8.8.8:443/" + host: "8.8.8.8" + path: "/" + port: 443 network: bytes_written: 4094.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "8.8.8.8" protocol: "TCP" squid: @@ -261,8 +279,10 @@ tests: network: bytes_written: 4006.0 client: + geoip: {} ip: "127.0.0.1" destination: + geoip: {} ip: "172.217.20.68" protocol: "TCP" squid: @@ -294,6 +314,7 @@ tests: network: bytes_written: 3903.0 client: + geoip: {} ip: "127.0.0.1" protocol: "TCP" squid: @@ -340,4 +361,160 @@ tests: tags: - "source:LOGS_SOURCE" timestamp: 1570785289000 - + - + sample: "10.10.10.10 - testuser1 [25/Sep/2024:11:25:20 +0530] \"CONNECT ssl.gstatic.com:443 HTTP/1.1\" 200 3139 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36\" TCP_TUNNEL:HIER_DIRECT" + result: + custom: + http: + method: "CONNECT" + status_category: "Success" + status_code: 200.0 + url_details: + host: "ssl.gstatic.com" + port: 443 + useragent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "129" + minor: "0" + patch: "0" + patch_minor: "0" + device: + category: "Desktop" + family: "Other" + os: + family: "Windows" + major: "10" + version: "HTTP/1.1" + network: + bytes_written: 3139.0 + client: + geoip: {} + ip: "10.10.10.10" + protocol: "TCP" + squid: + peer_status: "HIER_DIRECT" + status: "TUNNEL" + user_identity: "testuser1" + timestamp: 1727243720000 + usr: + name: "testuser1" + message: "10.10.10.10 - testuser1 [25/Sep/2024:11:25:20 +0530] \"CONNECT ssl.gstatic.com:443 HTTP/1.1\" 200 3139 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36\" TCP_TUNNEL:HIER_DIRECT" + status: "ok" + tags: + - "source:LOGS_SOURCE" + timestamp: 1727243720000 + - + sample: "10.10.10.10 - testuser1 [25/Sep/2024:15:20:11 +0530] \"GET http://proxy.squid.local:3128/squid-internal-mgr/counters HTTP/1.1\" 407 4724 TCP_DENIED:HIER_NONE'" + result: + custom: + http: + method: "GET" + status_category: "Warning" + status_code: 407.0 + url: "http://proxy.squid.local:3128/squid-internal-mgr/counters" + url_details: + host: "proxy.squid.local" + path: "/squid-internal-mgr/counters" + port: 3128 + scheme: "http" + version: "HTTP/1.1" + network: + bytes_written: 4724.0 + client: + geoip: {} + ip: "10.10.10.10" + protocol: "TCP" + squid: + peer_status: "HIER_NONE" + status: "DENIED" + user_identity: "testuser1" + timestamp: 1727257811000 + usr: + name: "testuser1" + message: "10.10.10.10 - testuser1 [25/Sep/2024:15:20:11 +0530] \"GET http://proxy.squid.local:3128/squid-internal-mgr/counters HTTP/1.1\" 407 4724 TCP_DENIED:HIER_NONE'" + status: "warn" + tags: + - "source:LOGS_SOURCE" + timestamp: 1727257811000 + - + sample: "10.10.10.10 - testuser1 [25/Sep/2024:15:20:17 +0530] \"CONNECT ssl.gstatic.com:443 HTTP/1.1\" 200 0 NONE_NONE:HIER_DIRECT" + result: + custom: + http: + method: "CONNECT" + status_category: "Success" + status_code: 200.0 + url_details: + host: "ssl.gstatic.com" + port: 443 + version: "HTTP/1.1" + network: + bytes_written: 0.0 + client: + geoip: {} + ip: "10.10.10.10" + protocol: "NONE" + squid: + peer_status: "HIER_DIRECT" + status: "NONE" + user_identity: "testuser1" + timestamp: 1727257817000 + usr: + name: "testuser1" + message: "10.10.10.10 - testuser1 [25/Sep/2024:15:20:17 +0530] \"CONNECT ssl.gstatic.com:443 HTTP/1.1\" 200 0 NONE_NONE:HIER_DIRECT" + status: "ok" + tags: + - "source:LOGS_SOURCE" + timestamp: 1727257817000 + - + sample: "1726124251 0 10.10.10.10 TCP_DENIED/403 3920 CONNECT ts01-gyr-maverick.cloudsink.net:443 - HIER_NONE/- text/html" + result: + custom: + duration: 0.0 + http: + method: "CONNECT" + resource: + content_type: "text/html" + status_category: "Warning" + status_code: 403 + url_details: + host: "ts01-gyr-maverick.cloudsink.net" + port: 443 + network: + bytes_written: 3920.0 + client: + geoip: {} + ip: "10.10.10.10" + protocol: "TCP" + squid: + peer_status: "HIER_NONE" + status: "DENIED" + timestamp: 1.726124251E12 + message: "1726124251 0 10.10.10.10 TCP_DENIED/403 3920 CONNECT ts01-gyr-maverick.cloudsink.net:443 - HIER_NONE/- text/html" + status: "warn" + tags: + - "source:LOGS_SOURCE" + timestamp: 1726124251000 + - + sample: "1726124255.221 0 10.10.10.10 NONE_NONE/000 0 - error:transaction-end-before-headers - HIER_NONE/- -" + result: + custom: + duration: 0.0 + error: + message: "transaction-end-before-headers" + network: + bytes_written: 0.0 + client: + geoip: {} + ip: "10.10.10.10" + protocol: "NONE" + squid: + peer_status: "HIER_NONE" + status: "NONE" + timestamp: 1.726124255221E12 + message: "1726124255.221 0 10.10.10.10 NONE_NONE/000 0 - error:transaction-end-before-headers - HIER_NONE/- -" + tags: + - "source:LOGS_SOURCE" + timestamp: 1726124255221 \ No newline at end of file diff --git a/squid/assets/monitors/cpu_usage_exceeded.json b/squid/assets/monitors/cpu_usage_exceeded.json new file mode 100644 index 0000000000000..6a215dae9886e --- /dev/null +++ b/squid/assets/monitors/cpu_usage_exceeded.json @@ -0,0 +1,33 @@ +{ + "version": 2, + "created_at": "2024-09-12", + "last_updated_at": "2024-09-12", + "title": "CPU usage exceeded", + "description": "CPU usage monitored.", + "definition": { + "id": 153453104, + "name": "CPU usage exceeded", + "type": "query alert", + "query": "avg(last_5m):sum:squid.cachemgr.cpu_time{*} >= 90", + "message": "{{#is_warning}} \nCPU Usage exceeded the threshold. \nThreshold: {{warn_threshold}} %\n{{/is_warning}}\n\n{{#is_alert}} \nCPU Usage exceeded the threshold. \nCurrent Usage : {{value}} %\nThreshold: {{threshold}} % \n{{/is_alert}}", + "tags": [ "squid" ], + "options": { + "thresholds": { + "critical": 90, + "warning": 50 + }, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_host_delay": 300, + "silenced": { } + }, + "priority": 4, + "restriction_policy": { + "bindings": [ ] + } + }, + "tags": [ + "integration:squid" + ] +} \ No newline at end of file diff --git a/squid/assets/monitors/high_latency_requests.json b/squid/assets/monitors/high_latency_requests.json new file mode 100644 index 0000000000000..282c64d310a02 --- /dev/null +++ b/squid/assets/monitors/high_latency_requests.json @@ -0,0 +1,35 @@ +{ + "version": 2, + "created_at": "2024-09-12", + "last_updated_at": "2024-09-12", + "title": "High latency requests", + "description": "High latency requests monitored.", + "definition": { + "id": 153450604, + "name": "High latency requests", + "type": "log alert", + "query": "logs(\"source:squid @duration:>6000000000\").index(\"*\").rollup(\"count\").last(\"5m\") >= 5", + "message": "{{#is_warning}} \nThe requests with long duration exceeded the threshold. \nThreshold: {{warn_threshold}}\n{{/is_warning}}\n\n{{#is_alert}} \nThe requests with long duration exceeded the threshold. \nCurrent requests: {{value}} \nThreshold: {{threshold}} \n{{/is_alert}}", + "tags": [ "squid" ], + "options": { + "thresholds": { + "critical": 5, + "warning": 2 + }, + "enable_logs_sample": false, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_host_delay": 300, + "groupby_simple_monitor": false, + "silenced": { } + }, + "priority": 4, + "restriction_policy": { + "bindings": [ ] + } + }, + "tags": [ + "integration:squid" + ] +} \ No newline at end of file diff --git a/squid/assets/monitors/high_rate_of_client_http_errors.json b/squid/assets/monitors/high_rate_of_client_http_errors.json new file mode 100644 index 0000000000000..c6334762ceedd --- /dev/null +++ b/squid/assets/monitors/high_rate_of_client_http_errors.json @@ -0,0 +1,33 @@ +{ + "version": 2, + "created_at": "2024-09-12", + "last_updated_at": "2024-09-12", + "title": "High rate of client HTTP errors", + "description": "High rate of client HTTP errors monitored.", + "definition": { + "id": 153452296, + "name": "High rate of client HTTP errors", + "type": "query alert", + "query": "max(last_5m):sum:squid.cachemgr.client_http.errors{*} / sum:squid.cachemgr.client_http.requests{*} * 100 >= 15", + "message": "{{#is_warning}} \nToo many client errors exceeded the threshold. \nThreshold: {{warn_threshold}} % \n{{/is_warning}}\n\n{{#is_alert}} \nToo many client errors exceeded the threshold. \nCurrent errors : {{value}} % \nThreshold: {{threshold}} % \n{{/is_alert}}", + "tags": [ "squid" ], + "options": { + "thresholds": { + "critical": 15, + "warning": 10 + }, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_host_delay": 300, + "silenced": { } + }, + "priority": 4, + "restriction_policy": { + "bindings": [ ] + } + }, + "tags": [ + "integration:squid" + ] +} \ No newline at end of file diff --git a/squid/assets/monitors/high_rate_of_server_errors.json b/squid/assets/monitors/high_rate_of_server_errors.json new file mode 100644 index 0000000000000..cc99cd49a17c7 --- /dev/null +++ b/squid/assets/monitors/high_rate_of_server_errors.json @@ -0,0 +1,33 @@ +{ + "version": 2, + "created_at": "2024-09-12", + "last_updated_at": "2024-09-12", + "title": "High rate of server errors", + "description": "High rate of server errors is monitored.", + "definition": { + "id": 153451847, + "name": "High rate of server errors", + "type": "query alert", + "query": "max(last_5m):sum:squid.cachemgr.server.all.errors{*} / sum:squid.cachemgr.server.all.requests{*} * 100 >= 15", + "message": "{{#is_warning}} \nToo many server errors exceeded the threshold. \nThreshold: {{warn_threshold}} % \n{{/is_warning}}\n\n{{#is_alert}} \nToo many server errors exceeded the threshold. \nCurrent errors : {{value}} % \nThreshold: {{threshold}} % \n{{/is_alert}}", + "tags": [ "squid" ], + "options": { + "thresholds": { + "critical": 15, + "warning": 10 + }, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_host_delay": 300, + "silenced": { } + }, + "priority": 4, + "restriction_policy": { + "bindings": [ ] + } + }, + "tags": [ + "integration:squid" + ] +} \ No newline at end of file diff --git a/squid/images/squid.png b/squid/images/squid.png new file mode 100644 index 0000000000000..393b62e6f4111 Binary files /dev/null and b/squid/images/squid.png differ diff --git a/squid/manifest.json b/squid/manifest.json index f1a26f119a775..d3f50e2680d72 100644 --- a/squid/manifest.json +++ b/squid/manifest.json @@ -1,50 +1,68 @@ { - "manifest_version": "2.0.0", - "app_uuid": "de18c581-69ee-48cf-ba23-7794bfb7a4bd", - "app_id": "squid", - "display_on_public_website": true, - "tile": { - "overview": "README.md#Overview", - "configuration": "README.md#Setup", - "support": "README.md#Support", - "changelog": "CHANGELOG.md", - "description": "Track metrics from your squid-cache servers with Datadog", - "title": "Squid", - "media": [], - "classifier_tags": [ - "Category::Caching", - "Category::Log Collection", - "Supported OS::Linux", - "Supported OS::Windows", - "Supported OS::macOS", - "Offering::Integration" - ] - }, - "author": { - "support_email": "help@datadoghq.com", - "name": "Datadog", - "homepage": "https://www.datadoghq.com", - "sales_email": "info@datadoghq.com" - }, - "assets": { - "integration": { - "source_type_name": "Squid", - "configuration": { - "spec": "assets/configuration/spec.yaml" - }, - "events": { - "creates_events": false - }, - "metrics": { - "prefix": "squid.", - "check": "squid.cachemgr.cpu_time", - "metadata_path": "metadata.csv" - }, - "service_checks": { - "metadata_path": "assets/service_checks.json" - }, - "source_type_id": 10022, - "auto_install": true + "manifest_version": "2.0.0", + "app_uuid": "de18c581-69ee-48cf-ba23-7794bfb7a4bd", + "app_id": "squid", + "display_on_public_website": true, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Track metrics from your squid-cache servers with Datadog", + "title": "Squid", + "media": [ + { + "caption": "Squid", + "image_url": "images/squid.png", + "media_type": "image" + } + ], + "classifier_tags": [ + "Category::Caching", + "Category::Log Collection", + "Supported OS::Linux", + "Supported OS::Windows", + "Supported OS::macOS", + "Offering::Integration" + ] + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + }, + "assets": { + "integration": { + "source_type_name": "Squid", + "configuration": { + "spec": "assets/configuration/spec.yaml" + }, + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "squid.", + "check": "squid.cachemgr.cpu_time", + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + }, + "source_type_id": 10022, + "auto_install": true + }, + "dashboards": { + "Squid": "assets/dashboards/squid.json" + }, + "monitors": { + "CPU usage exceeded": "assets/monitors/cpu_usage_exceeded.json", + "High latency requests": "assets/monitors/high_latency_requests.json", + "High rate of client HTTP errors": "assets/monitors/high_rate_of_client_http_errors.json", + "High rate of server errors": "assets/monitors/high_rate_of_server_errors.json" + }, + "logs": { + "source": "squid" + } } - } } \ No newline at end of file