cf_deployment_diego_health_screen.json.erb

<%
    bosh_event_query_string = "\\\"Begin update deployment for '#{diego_deployment}' against Director\\\" OR \\\"Finish update deployment for '#{diego_deployment}' against Director\\\" OR \\\"Error during update deployment for '#{diego_deployment}' against Director\\\" OR \\\"Begin update deployment for '#{deployment}' against Director\\\" OR \\\"Finish update deployment for '#{deployment}' against Director\\\" OR \\\"Error during update deployment for '#{deployment}' against Director\\\""
%>

{
  "title": "<%= environment %> CF-Deployment Diego Health",
  "description": "<%= environment %> Diego Health",
  "graphs":[
    {
      "title": "BBS: LRPs",
      "definition":{
        "requests": [
          {
            "q": "datadog.nozzle.LRPsDesired{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsClaimed{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.LRPsUnclaimed{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsRunning{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.CrashedActualLRPs{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsDesired{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsClaimed{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.LRPsUnclaimed{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsRunning{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.CrashedActualLRPs{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Non-Running LRPs",
      "definition":{
        "requests": [
          {
            "q": "datadog.nozzle.LRPsClaimed{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.LRPsUnclaimed{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.CrashedActualLRPs{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.SuspectRunningActualLRPs{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.SuspectClaimedActualLRPs{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsClaimed{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.LRPsUnclaimed{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.CrashedActualLRPs{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.SuspectRunningActualLRPs{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.SuspectClaimedActualLRPs{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Missing and Extra LRPs",
      "definition":{
        "requests": [
          {
            "q": "datadog.nozzle.LRPsMissing{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsExtra{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsMissing{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.LRPsExtra{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Tasks",
      "definition":{
        "requests":[
          {
            "q": "avg:datadog.nozzle.TasksPending{origin:bbs,deployment:<%= metron_agent_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksClaimed{origin:bbs,deployment:<%= metron_agent_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksRunning{origin:bbs,deployment:<%= metron_agent_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksCompleted{origin:bbs,deployment:<%= metron_agent_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksResolving{origin:bbs,deployment:<%= metron_agent_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksPending{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksClaimed{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksRunning{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksCompleted{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.TasksResolving{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Cell Presence",
      "definition":{
        "requests": [
          {
            "q": "datadog.nozzle.PresentCells{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.SuspectCells{origin:bbs,deployment: <%= metron_agent_deployment %>}"
          },
          {
            "q": "datadog.nozzle.PresentCells{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "datadog.nozzle.SuspectCells{origin:bbs,deployment: <%= metron_agent_diego_deployment %>}"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Canary App Instance Count",
      "definition":{
        "requests":[
          {
            "q": "sum:diego.canary.app.instance{deployment:<%= metron_agent_diego_deployment %>}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Garden Unhealthy",
      "definition":{
        "requests":[
          {
            "q": "sum:datadog.nozzle.GardenHealthCheckFailed{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}, sum:datadog.nozzle.GardenHealthCheckFailed{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Available Memory",
      "definition":{
        "requests":[
          {
            "q": "min:datadog.nozzle.CapacityRemainingMemory{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} / 0.000001024, min:datadog.nozzle.CapacityRemainingMemory{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} / 0.000001024",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Available Disk",
      "definition":{
        "requests":[
          {
            "q": "min:datadog.nozzle.CapacityRemainingDisk{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} / 0.000001048576, min:datadog.nozzle.CapacityRemainingDisk{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} / 0.000001048576",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Locket Lock Held",
      "definition":{
        "requests": [
          {
            "q": "datadog.nozzle.LockHeld{origin:bbs,deployment: <%= metron_agent_deployment %>} by {job,index,ip}, datadog.nozzle.LockHeld{origin:bbs,deployment: <%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Request Count (per minute)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "per_minute(avg:datadog.nozzle.RequestCount{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(avg:datadog.nozzle.RequestCount{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Request Latency (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "max:datadog.nozzle.RequestLatency{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.RequestLatency{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Time To Converge (seconds)",
      "definition":{
        "requests": [
          {
            "q": "sum:datadog.nozzle.ConvergenceLRPDuration{origin:bbs,deployment:<%= metron_agent_deployment %>}/1000000000",
            "type": "line"
          },
          {
            "q": "sum:datadog.nozzle.ConvergenceTaskDuration{origin:bbs,deployment:<%= metron_agent_deployment %>}/1000000000",
            "type": "line"
          },
          {
            "q": "sum:datadog.nozzle.ConvergenceLRPDuration{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}/1000000000",
            "type": "line"
          },
          {
            "q": "sum:datadog.nozzle.ConvergenceTaskDuration{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}/1000000000",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: cf-apps Domain Freshness (bool)",
      "definition":{
        "requests":[
          {
            "q": "max:datadog.nozzle.Domain.cf_apps{origin:bbs,deployment:<%= metron_agent_deployment %>}",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.Domain.cf_apps{origin:bbs,deployment:<%= metron_agent_diego_deployment %>}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Locket: Active Locks",
      "definition":{
        "requests":[
          {
            "q": "avg:datadog.nozzle.ActiveLocks{origin:locket,deployment:<%= metron_agent_deployment %>} by {job,index,ip}",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.ActiveLocks{origin:locket,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Locket: Active Presences",
      "definition":{
        "requests":[
          {
            "q": "avg:datadog.nozzle.ActivePresences{origin:locket,deployment:<%= metron_agent_deployment %>} by {job,index,ip}",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.ActivePresences{origin:locket,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Locket: Locks Expired",
      "definition":{
        "requests":[
          {
            "q": "avg:datadog.nozzle.LocksExpired{origin:locket,deployment:<%= metron_agent_deployment %>} by {job,index,ip}",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.LocksExpired{origin:locket,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Locket: Presence Expired",
      "definition":{
        "requests":[
          {
            "q": "avg:datadog.nozzle.PresenceExpired{origin:locket,deployment:<%= metron_agent_deployment %>} by {job,index,ip}",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.PresenceExpired{origin:locket,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Locket: Request Latency (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "max:datadog.nozzle.RequestLatencyMax{origin:locket,deployment:<%= metron_agent_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.RequestLatencyMax{origin:locket,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Route Emitter: Total Routes",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "avg:datadog.nozzle.RoutesTotal{origin:route_emitter,deployment:<%= metron_agent_diego_deployment %>}"
          },
          {
            "q": "avg:datadog.nozzle.RoutesTotal{origin:route_emitter,deployment:<%= metron_agent_diego_deployment %>}"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Local Route Emitter: Total HTTP Routes",
      "definition":{
        "requests":[
          {
            "q": "max:datadog.nozzle.HTTPRouteCount{origin:route_emitter,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}, max:datadog.nozzle.HTTPRouteCount{origin:route_emitter,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Local Route Emitter: Total TCP Routes",
      "definition":{
        "requests":[
          {
            "q": "max:datadog.nozzle.TCPRouteCount{origin:route_emitter,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}, max:datadog.nozzle.TCPRouteCount{origin:route_emitter,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Auctioneer: Locket Lock Held",
      "definition":{
        "requests": [
          {
            "q": "datadog.nozzle.LockHeld{origin:auctioneer,deployment: <%= metron_agent_deployment %>} by {job,index,ip}, datadog.nozzle.LockHeld{origin:auctioneer,deployment: <%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Auctioneer: Request Count (per minute)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "per_minute(avg:datadog.nozzle.RequestCount{origin:auctioneer,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(avg:datadog.nozzle.RequestCount{origin:auctioneer,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Auctioneer: Request Latency (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "max:datadog.nozzle.RequestLatency{origin:auctioneer,deployment:<%= metron_agent_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.RequestLatency{origin:auctioneer,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Auctioneer: LRP Auctions (per minute)",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "per_minute(datadog.nozzle.AuctioneerLRPAuctionsStarted{origin:auctioneer,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(datadog.nozzle.AuctioneerLRPAuctionsFailed{origin:auctioneer,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(datadog.nozzle.AuctioneerLRPAuctionsStarted{origin:auctioneer,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(datadog.nozzle.AuctioneerLRPAuctionsFailed{origin:auctioneer,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Auctioneer: Task Auctions (per minute)",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "per_minute(datadog.nozzle.AuctioneerTaskAuctionsStarted{origin:auctioneer,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(datadog.nozzle.AuctioneerTaskAuctionsFailed{origin:auctioneer,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(datadog.nozzle.AuctioneerTaskAuctionsStarted{origin:auctioneer,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(datadog.nozzle.AuctioneerTaskAuctionsFailed{origin:auctioneer,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Auctioneer: Fetch Cell States Duration (seconds)",
      "definition":{
        "requests": [
          {
            "q": "max:datadog.nozzle.AuctioneerFetchStatesDuration{origin:auctioneer,deployment:<%= metron_agent_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.AuctioneerFetchStatesDuration{origin:auctioneer,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: LRP Convergence Actions Taken (per minute)",
      "definition":{
        "requests": [
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceLRPRuns{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceLRPRuns{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Task Convergence Actions Taken (per minute)",
      "definition":{
        "requests": [
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceTaskRuns{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceTasksKicked{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceTasksPruned{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceTaskRuns{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceTasksKicked{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          },
          {
            "q": "per_minute(sum:datadog.nozzle.ConvergenceTasksPruned{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Master Elections",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "avg:datadog.nozzle.BBSMasterElected{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip}, avg:datadog.nozzle.BBSMasterElected{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Migration Duration (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "max:datadog.nozzle.MigrationDuration{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip}/1000000000, max:datadog.nozzle.MigrationDuration{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}/1000000000",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Encryption Duration (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "max:datadog.nozzle.EncryptionDuration{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip}/1000000000, max:datadog.nozzle.EncryptionDuration{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}/1000000000",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BBS: Open File Descriptors",
      "definition":{
        "requests":[
          {
            "q": "avg:datadog.nozzle.OpenFileDescriptors{origin:bbs,deployment:<%= metron_agent_deployment %>} by {job,index,ip}",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.OpenFileDescriptors{origin:bbs,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Total Containers",
      "definition":{
        "requests":[
          {
            "q": "min:datadog.nozzle.ContainerCount{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "line"
          },
          {
            "q": "min:datadog.nozzle.ContainerCount{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Starting Containers",
      "definition":{
        "requests":[
          {
            "q": "max:datadog.nozzle.StartingContainerCount{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}, max:datadog.nozzle.StartingContainerCount{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Bulk Duration (seconds)",
      "definition":{
        "requests": [
          {
            "q": "max:datadog.nozzle.RepBulkSyncDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} /1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.RepBulkSyncDuration{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}/1000000000",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Stalled Garden Duration (seconds)",
      "definition":{
        "requests": [
          {
            "q": "max:datadog.nozzle.StalledGardenDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} /1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.StalledGardenDuration{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} /1000000000",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Stranded Evacuating ActualLRPs",
      "definition":{
        "requests":[
          {
            "q": "max:datadog.nozzle.StrandedEvacuatingActualLRPs{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}, max:datadog.nozzle.StrandedEvacuatingActualLRPs{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}",
            "type": "bars"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: State Request Latency (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "max:datadog.nozzle.RequestLatencyMax{origin:rep,deployment:<%= metron_agent_deployment %>,request-type:state} by {job,index,ip}/1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.RequestLatencyMax{origin:rep,deployment:<%= metron_agent_diego_deployment %>,request-type:state} by {job,index,ip}/1000000000",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Container Setup Succeeded Duration (seconds)",
      "definition":{
        "viz": "heatmap",
        "requests": [
          {
            "q": "avg:datadog.nozzle.ContainerSetupSucceededDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}.rollup(max).fill(null)/1000000000",
            "style": {
              "palette": "cool",
              "type": "solid",
              "width": "normal"
            }
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Container Setup Failed Duration (seconds)",
      "definition":{
        "viz": "heatmap",
        "requests": [
          {
            "q": "avg:datadog.nozzle.ContainerSetupFailedDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}.rollup(max).fill(null)/1000000000",
            "style": {
              "palette": "cool",
              "type": "solid",
              "width": "normal"
            }
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Garden Container Creation Succeeded Duration (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "avg:datadog.nozzle.GardenContainerCreationSucceededDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} / 1000000000",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.GardenContainerCreationSucceededDuration{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} / 1000000000",
            "type": "line"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Garden Container Creation Failed Duration (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "avg:datadog.nozzle.GardenContainerCreationFailedDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} / 1000000000",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.GardenContainerCreationFailedDuration{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} / 1000000000",
            "type": "line"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Garden Container Destruction Succeeded Duration (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "avg:datadog.nozzle.GardenContainerDestructionSucceededDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} / 1000000000",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.GardenContainerDestructionSucceededDuration{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} / 1000000000",
            "type": "line"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Garden Container Destruction Failed Duration (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "avg:datadog.nozzle.GardenContainerDestructionFailedDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} / 1000000000",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.GardenContainerDestructionFailedDuration{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} / 1000000000",
            "type": "line"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Credential Creation Succeeded Count (per minute)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "per_minute(avg:datadog.nozzle.CredCreationSucceededCount{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})"
          },
          {
            "q": "per_minute(avg:datadog.nozzle.CredCreationSucceededCount{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment})"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Credential Creation Failed Count (per minute)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "per_minute(avg:datadog.nozzle.CredCreationFailedCount{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip})"
          },
          {
            "q": "per_minute(avg:datadog.nozzle.CredCreationFailedCount{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment})"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Cell Rep: Credential Creation Succeeded Duration (seconds)",
      "definition":{
        "viz": "timeseries",
        "requests": [
          {
            "q": "avg:datadog.nozzle.CredCreationSucceededDuration{origin:rep,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip} / 1000000000",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.CredCreationSucceededDuration{origin:rep,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment} / 1000000000",
            "type": "line"
          }
        ]
      }
    },
    {
      "title": "Route Emitter: Incremental Updates (per second)",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "rate(datadog.nozzle.RoutesRegistered{origin:route_emitter,deployment:<%= metron_agent_diego_deployment %>} + datadog.nozzle.RoutesRegistered{origin:route_emitter,bosh_environment:<%= bosh_deployment %>})"
          },
          {
            "q": "-rate(datadog.nozzle.RoutesUnregistered{origin:route_emitter,deployment:<%= metron_agent_diego_deployment %>} + datadog.nozzle.RoutesUnregistered{origin:route_emitter,bosh_environment:<%= bosh_deployment %>})"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "Route Emitter: Bulk Duration (seconds)",
      "definition":{
        "requests": [
          {
            "q": "max:datadog.nozzle.RouteEmitterSyncDuration{origin:route_emitter,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}/1000000000",
            "type": "line"
          },
          {
            "q": "max:datadog.nozzle.RouteEmitterSyncDuration{origin:route_emitter,bosh_environment:<%= bosh_deployment %>} by {job,index,ip,deployment}/1000000000",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "SSH Proxy: SSH Connections",
      "definition":{
        "requests": [
          {
            "q": "avg:datadog.nozzle.ssh_connections{origin:ssh_proxy,deployment:<%= metron_agent_deployment %>} by {job,index,ip}",
            "type": "line"
          },
          {
            "q": "avg:datadog.nozzle.ssh_connections{origin:ssh_proxy,deployment:<%= metron_agent_diego_deployment %>} by {job,index,ip}",
            "type": "line"
          }
        ],
        "viz": "timeseries",
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BOSH VMs: System Memory Percent",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "bosh.healthmonitor.system.mem.percent{deployment:<%= diego_deployment %>} by {job,id,index}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ],
        "markers": [
          {
            "value": "y > 80",
            "type": "error bold"
          },
          {
            "value": "y < 80",
            "type": "ok dashed"
          }
        ]
      }
    },
    {
      "title": "BOSH VMs: Load Average (1m)",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "bosh.healthmonitor.system.load.1m{deployment:<%= diego_deployment %>} by {job,id,index}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    },
    {
      "title": "BOSH VMs: CPU System",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "bosh.healthmonitor.system.cpu.sys{deployment:<%= diego_deployment %>} by {job,id,index}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ],
        "markers": [
          {
            "value": "y > 80",
            "type": "error bold"
          },
          {
            "value": "y < 80",
            "type": "ok dashed"
          }
        ]
      }
    },
    {
      "title": "BOSH VMs: CPU User",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "bosh.healthmonitor.system.cpu.user{deployment:<%= diego_deployment %>} by {job,id,index}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ],
        "markers": [
          {
            "value": "y > 80",
            "type": "error bold"
          },
          {
            "value": "y < 80",
            "type": "ok dashed"
          }
        ]
      }
    },
    {
      "title": "BOSH VMs: CPU Wait",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "bosh.healthmonitor.system.cpu.wait{deployment:<%= diego_deployment %>} by {job,id,index}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ],
        "markers": [
          {
            "value": "y > 80",
            "type": "error bold"
          },
          {
            "value": "y < 80",
            "type": "ok dashed"
          }
        ]
      }
    },
    {
      "title": "BOSH VMs: System Disk Percent",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "bosh.healthmonitor.system.disk.system.percent{deployment:<%= diego_deployment %>} by {job,id,index}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ],
        "markers": [
          {
            "value": "y > 80",
            "type": "error bold"
          },
          {
            "value": "y < 80",
            "type": "ok dashed"
          }
        ]
      }
    },
    {
      "title": "BOSH VMs: Ephemeral Disk Percent",
      "definition":{
          "viz": "timeseries",
          "requests": [
          {
            "q": "bosh.healthmonitor.system.disk.ephemeral.percent{deployment:<%= diego_deployment %>} by {job,id,index}",
            "type": "line"
          }
        ],
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    }
    <%
    [
      {:prefix => "API and Brain", :control_plane => true,  :processes => ["auctioneer", "bbs", "file_server", "locket", "ssh_proxy"]},
      {:prefix => "Cell",          :control_plane => false, :processes => ["garden_linux", "rep", "route_emitter"]},
      {:prefix => "CC-Bridge",     :control_plane => true,  :processes => ["cc_uploader","nsync_bulker", "nsync_listener", "stager", "tps_listener", "tps_watcher"]},
    ].each do |group|
    [
      {:title => "Go: Heap (bytes)", :metric => "memoryStats.numBytesAllocatedHeap"},
      {:title => "Go: Stack (bytes)", :metric => "memoryStats.numBytesAllocatedStack"},
      {:title => "Go: GC Pause Time (NS)", :metric => "memoryStats.lastGCPauseTimeNS"},
      {:title => "Go: # Goroutines", :metric => "numGoRoutines"},
    ].each do |metric|
      requests = group[:processes].map { |process|
        if group[:control_plane]
        [{
          "q" => "datadog.nozzle.#{process}.#{metric[:metric]}{deployment:#{metron_agent_deployment}} by {job,index,ip}",
          "type" => "line",
        },
        {
          "q" => "datadog.nozzle.#{process}.#{metric[:metric]}{deployment:#{metron_agent_diego_deployment}} by {job,index,ip}",
          "type" => "line",
        }]
        else
        [{
          "q" => "datadog.nozzle.#{process}.#{metric[:metric]}{deployment:#{metron_agent_diego_deployment}} by {job,index,ip}",
          "type" => "line",
        },
        {
          "q" => "datadog.nozzle.#{process}.#{metric[:metric]}{bosh_environment:#{bosh_deployment}} by {job,index,ip,deployment}",
          "type" => "line",
        }]
        end
      }
    %>
    ,
    {
      "title": "<%= group[:prefix] + " " + metric[:title] %>",
      "definition":{
        "requests": <%= requests.flatten.to_json %>,
        "events":[
          {
            "q": "<%= bosh_event_query_string %>"
          }
        ]
      }
    }
    <% end %>
    <% end %>
  ]
}