{
    "__inputs": [
      {
        "name": "DS_PROMETHEUS",
        "label": "Prometheus",
        "description": "",
        "type": "datasource",
        "pluginId": "prometheus",
        "pluginName": "Prometheus"
      }
    ],
    "__elements": {},
    "__requires": [
      {
        "type": "grafana",
        "id": "grafana",
        "name": "Grafana",
        "version": "11.1.1"
      },
      {
        "type": "datasource",
        "id": "prometheus",
        "name": "Prometheus",
        "version": "1.0.0"
      },
      {
        "type": "panel",
        "id": "stat",
        "name": "Stat",
        "version": ""
      },
      {
        "type": "panel",
        "id": "timeseries",
        "name": "Time series",
        "version": ""
      }
    ],
    "annotations": {
      "list": [
        {
          "builtIn": 1,
          "datasource": {
            "type": "grafana",
            "uid": "-- Grafana --"
          },
          "enable": true,
          "hide": true,
          "iconColor": "rgba(0, 211, 255, 1)",
          "name": "Annotations & Alerts",
          "type": "dashboard"
        }
      ]
    },
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 1,
    "id": null,
    "links": [],
    "panels": [
      {
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 0
        },
        "id": 13,
        "title": "Core metrics / peers",
        "type": "row"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "Number of active connected peers",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "decimals": 2,
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                }
              ]
            },
            "unit": "short"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 4,
          "w": 12,
          "x": 0,
          "y": 1
        },
        "id": 2,
        "options": {
          "colorMode": "value",
          "graphMode": "area",
          "justifyMode": "auto",
          "orientation": "auto",
          "percentChangeColorMode": "standard",
          "reduceOptions": {
            "calcs": [
              "lastNotNull"
            ],
            "fields": "",
            "values": false
          },
          "showPercentChange": true,
          "textMode": "auto",
          "wideLayout": true
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "sum(active_peers{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\"}) by (cluster,environment,job)",
            "fullMetaSearch": false,
            "includeNullMetadata": true,
            "legendFormat": "{{cluster}}/{{environment}}/{{job}}/",
            "range": true,
            "refId": "A",
            "useBackend": false
          }
        ],
        "title": "Total active peers",
        "type": "stat"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "Duration of how long a peer was connected\n\nThis is mostly informational, but if most of the connections are short lived, you may need to check any reverse proxy or connectivity issues",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 1
        },
        "id": 1,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.5,sum(increase(peer_connection_duration_seconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p50",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.90,sum(increase(peer_connection_duration_seconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p90",
            "range": true,
            "refId": "B",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.99,sum(increase(peer_connection_duration_seconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p99",
            "range": true,
            "refId": "C",
            "useBackend": false
          }
        ],
        "title": "Peer connection duration",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "Number of active connected peers by host",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "max": 25000,
            "min": 0,
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "#EAB839",
                  "value": 15000
                },
                {
                  "color": "red",
                  "value": 20000
                }
              ]
            },
            "unit": "none"
          },
          "overrides": [
            {
              "__systemRef": "hideSeriesFrom",
              "matcher": {
                "id": "byNames",
                "options": {
                  "mode": "exclude",
                  "names": [
                    "{__name__=\"active_peers\", application=\"signal\", environment=\"prod\", host=\"signal-1-prod\", instance=\"localhost:9090\", job=\"prometheus.scrape.local\"}"
                  ],
                  "prefix": "All except:",
                  "readOnly": true
                }
              },
              "properties": []
            }
          ]
        },
        "gridPos": {
          "h": 4,
          "w": 12,
          "x": 0,
          "y": 5
        },
        "id": 10,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "exemplar": false,
            "expr": "active_peers{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}",
            "format": "time_series",
            "fullMetaSearch": false,
            "includeNullMetadata": true,
            "instant": false,
            "legendFormat": "{{cluster}}/{{environment}}/{{job}}/{{host}}",
            "range": true,
            "refId": "A",
            "useBackend": false
          }
        ],
        "title": "Active Peer-Connections per Host",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 9
        },
        "id": 14,
        "panels": [],
        "title": "Core metrics / messages",
        "type": "row"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "No. of messages forwarded per seconds. \nThis doesn't include the failed message count. They are tracked separately.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "mps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 5,
          "w": 12,
          "x": 0,
          "y": 10
        },
        "id": 15,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "editorMode": "code",
            "expr": "rate(messages_forwarded_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])",
            "instant": false,
            "legendFormat": "{{cluster}}/{{environment}}/{{job}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Forwarded messages rate",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "mps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 5,
          "w": 12,
          "x": 12,
          "y": 10
        },
        "id": 16,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "editorMode": "code",
            "expr": "sum(rate(message_forward_failures_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (cluster,environment,job,type)",
            "instant": false,
            "legendFormat": "{{type}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Forwarded messages failure rate",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 15
        },
        "id": 17,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.5,sum(increase(message_forward_latency_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p50",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.90,sum(increase(message_forward_latency_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p90",
            "range": true,
            "refId": "B",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.99,sum(increase(message_forward_latency_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p99",
            "range": true,
            "refId": "C",
            "useBackend": false
          }
        ],
        "title": "Forwarded messages latency",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 23
        },
        "id": 18,
        "panels": [],
        "title": "Core metrics / registrations",
        "type": "row"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "rate of peer Registration and deregistration",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "never",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                }
              ]
            },
            "unit": "cps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 24
        },
        "id": 5,
        "options": {
          "legend": {
            "calcs": [
              "lastNotNull",
              "min",
              "mean",
              "max"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "hideZeros": false,
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.5.0-81732",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "rate(registrations_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])",
            "fullMetaSearch": false,
            "includeNullMetadata": true,
            "legendFormat": "registration",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "rate(deregistrations_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": true,
            "legendFormat": "deregistration",
            "range": true,
            "refId": "B",
            "useBackend": false
          }
        ],
        "title": "Registration and deregistration rate",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "mps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 24
        },
        "id": 19,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "editorMode": "code",
            "expr": "sum(rate(registration_failures_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (cluster,environment,job,error)",
            "instant": false,
            "legendFormat": "{{error}}",
            "range": true,
            "refId": "A"
          }
        ],
        "title": "Registration failure rate",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "time needed to register new peers",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 32
        },
        "id": 20,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.5,sum(increase(registration_delay_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p50",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.90,sum(increase(registration_delay_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p90",
            "range": true,
            "refId": "B",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.99,sum(increase(registration_delay_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p99",
            "range": true,
            "refId": "C",
            "useBackend": false
          }
        ],
        "title": "Peer registration latency",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "Duration of how long it takes to load a connection from the registry.\n\nNOTE: It should be below 1 ms",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "s"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 32
        },
        "id": 21,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.5,sum(increase(get_registration_delay_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p50",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.90,sum(increase(get_registration_delay_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p90",
            "range": true,
            "refId": "B",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.99,sum(increase(get_registration_delay_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\"}[$__rate_interval])) by (le,cluster,environment,job))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p99",
            "range": true,
            "refId": "C",
            "useBackend": false
          }
        ],
        "title": "get_registration request latency",
        "type": "timeseries"
      },
      {
        "collapsed": false,
        "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 40
        },
        "id": 12,
        "panels": [],
        "title": "RPC metrics",
        "type": "row"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "Send method is used by peers intending to send a message to another peer. The send covers the RPC call of sending the message from the initiator to signal.",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "reqps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 41
        },
        "id": 9,
        "options": {
          "legend": {
            "calcs": [
              "lastNotNull",
              "min",
              "mean",
              "max"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "hideZeros": false,
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.5.0-81732",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "editorMode": "code",
            "expr": "sum(rate(rpc_server_requests_per_rpc_count{rpc_method=~\"Send\"}[$__rate_interval])) by (rpc_method,rpc_grpc_status_code)",
            "hide": false,
            "instant": false,
            "legendFormat": "code: {{rpc_grpc_status_code}}",
            "range": true,
            "refId": "B"
          }
        ],
        "title": "RPC requests rate (method: Send)",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "reqps"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 41
        },
        "id": 23,
        "options": {
          "legend": {
            "calcs": [
              "lastNotNull",
              "min",
              "mean",
              "max"
            ],
            "displayMode": "table",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "hideZeros": false,
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.5.0-81732",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "editorMode": "code",
            "expr": "sum(rate(rpc_server_requests_per_rpc_count{rpc_method=~\"ConnectStream\"}[$__rate_interval])) by (rpc_method,rpc_grpc_status_code)",
            "hide": false,
            "instant": false,
            "legendFormat": "code: {{rpc_grpc_status_code}}",
            "range": true,
            "refId": "B"
          }
        ],
        "title": "RPC requests rate (method: ConnectStream)",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "ms"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 49
        },
        "id": 22,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.5,sum(increase(rpc_server_duration_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\",rpc_method=~\"Send\"}[$__rate_interval])) by (le,cluster,environment,job,rpc_method))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p50",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.90,sum(increase(rpc_server_duration_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\",rpc_method=~\"Send\"}[$__rate_interval])) by (le,cluster,environment,job,rpc_method))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p90",
            "range": true,
            "refId": "B",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.99,sum(increase(rpc_server_duration_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\",rpc_method=~\"Send\"}[$__rate_interval])) by (le,cluster,environment,job,rpc_method))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p99",
            "range": true,
            "refId": "C",
            "useBackend": false
          }
        ],
        "title": "RPC request latency (method: Send)",
        "type": "timeseries"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${datasource}"
        },
        "description": "Shows estimated time of how long a peer was connected",
        "fieldConfig": {
          "defaults": {
            "color": {
              "mode": "palette-classic"
            },
            "custom": {
              "axisBorderShow": false,
              "axisCenteredZero": false,
              "axisColorMode": "text",
              "axisLabel": "",
              "axisPlacement": "auto",
              "barAlignment": 0,
              "drawStyle": "line",
              "fillOpacity": 0,
              "gradientMode": "none",
              "hideFrom": {
                "legend": false,
                "tooltip": false,
                "viz": false
              },
              "insertNulls": false,
              "lineInterpolation": "linear",
              "lineWidth": 1,
              "pointSize": 5,
              "scaleDistribution": {
                "type": "linear"
              },
              "showPoints": "auto",
              "spanNulls": false,
              "stacking": {
                "group": "A",
                "mode": "none"
              },
              "thresholdsStyle": {
                "mode": "off"
              }
            },
            "mappings": [],
            "thresholds": {
              "mode": "absolute",
              "steps": [
                {
                  "color": "green",
                  "value": null
                },
                {
                  "color": "red",
                  "value": 80
                }
              ]
            },
            "unit": "ms"
          },
          "overrides": []
        },
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 49
        },
        "id": 25,
        "options": {
          "legend": {
            "calcs": [],
            "displayMode": "list",
            "placement": "bottom",
            "showLegend": true
          },
          "tooltip": {
            "mode": "single",
            "sort": "none"
          }
        },
        "pluginVersion": "11.1.1",
        "targets": [
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.5,sum(increase(rpc_server_duration_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\",rpc_method=~\"ConnectStream\"}[$__rate_interval])) by (le,cluster,environment,job,rpc_method))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p50",
            "range": true,
            "refId": "A",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.90,sum(increase(rpc_server_duration_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\",rpc_method=~\"ConnectStream\"}[$__rate_interval])) by (le,cluster,environment,job,rpc_method))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p90",
            "range": true,
            "refId": "B",
            "useBackend": false
          },
          {
            "datasource": {
              "type": "prometheus",
              "uid": "${datasource}"
            },
            "disableTextWrap": false,
            "editorMode": "code",
            "expr": "histogram_quantile(0.99,sum(increase(rpc_server_duration_milliseconds_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",host=~\"$host\",rpc_method=~\"ConnectStream\"}[$__rate_interval])) by (le,cluster,environment,job,rpc_method))",
            "format": "heatmap",
            "fullMetaSearch": false,
            "hide": false,
            "includeNullMetadata": false,
            "legendFormat": "p99",
            "range": true,
            "refId": "C",
            "useBackend": false
          }
        ],
        "title": "RPC request latency (method: ConnectStream)",
        "type": "timeseries"
      }
    ],
    "refresh": "",
    "schemaVersion": 39,
    "tags": [],
    "templating": {
      "list": [
        {
          "current": {},
          "hide": 0,
          "includeAll": false,
          "multi": false,
          "name": "datasource",
          "options": [],
          "query": "prometheus",
          "queryValue": "",
          "refresh": 1,
          "regex": "",
          "skipUrlSync": false,
          "type": "datasource"
        },
        {
          "current": {},
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "definition": "label_values(active_peers,cluster)",
          "hide": 0,
          "includeAll": false,
          "multi": false,
          "name": "cluster",
          "options": [],
          "query": {
            "qryType": 1,
            "query": "label_values(active_peers,cluster)",
            "refId": "PrometheusVariableQueryEditor-VariableQuery"
          },
          "refresh": 1,
          "regex": "",
          "skipUrlSync": false,
          "sort": 0,
          "type": "query"
        },
        {
          "current": {},
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "definition": "label_values(active_peers{cluster=\"$cluster\"},environment)",
          "description": "",
          "hide": 0,
          "includeAll": false,
          "label": "environment",
          "multi": false,
          "name": "environment",
          "options": [],
          "query": {
            "qryType": 1,
            "query": "label_values(active_peers{cluster=\"$cluster\"},environment)",
            "refId": "PrometheusVariableQueryEditor-VariableQuery"
          },
          "refresh": 1,
          "regex": "",
          "skipUrlSync": false,
          "sort": 0,
          "type": "query"
        },
        {
          "current": {},
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "definition": "label_values(active_peers{cluster=\"$cluster\", environment=\"$environment\"},job)",
          "hide": 0,
          "includeAll": false,
          "multi": false,
          "name": "job",
          "options": [],
          "query": {
            "qryType": 1,
            "query": "label_values(active_peers{cluster=\"$cluster\", environment=\"$environment\"},job)",
            "refId": "PrometheusVariableQueryEditor-VariableQuery"
          },
          "refresh": 1,
          "regex": "",
          "skipUrlSync": false,
          "sort": 0,
          "type": "query"
        },
        {
          "allValue": "",
          "current": {},
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "definition": "label_values(active_peers{cluster=\"$cluster\", environment=\"$environment\", job=\"$job\"},host)",
          "hide": 0,
          "includeAll": true,
          "label": "",
          "multi": true,
          "name": "host",
          "options": [],
          "query": {
            "qryType": 1,
            "query": "label_values(active_peers{cluster=\"$cluster\", environment=\"$environment\", job=\"$job\"},host)",
            "refId": "PrometheusVariableQueryEditor-VariableQuery"
          },
          "refresh": 2,
          "regex": "",
          "skipUrlSync": false,
          "sort": 1,
          "type": "query"
        }
      ]
    },
    "time": {
      "from": "now-24h",
      "to": "now"
    },
    "timepicker": {},
    "timezone": "browser",
    "title": "Netbird / Signal",
    "uid": "cebyq0fs0m-v001",
    "version": 15,
    "weekStart": ""
  }