remove inaccurate dht client data (#135)

probe-lab · Oct 10, 2024 · eb19d41 · eb19d41
1 parent a3fb21b
commit eb19d41
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 23 deletions.
diff --git a/config/plotdefs/ipfs-servers-vs-clients.yaml b/config/plotdefs/ipfs-servers-vs-clients.yaml
@@ -2,24 +2,19 @@ frequency: daily
 datasets:
   # unique peers is calculated from total unique peers seen connecting to preload and bootstrap servers
   # caracol started collecting this data automatically from week ending 20 Mar 2023
+  # between 2024-09-09 and 2024-09-30 the IPFS team had an outage and we didn't collect data
   - name: "unique peers"
     source: "caracol"
     query: >
-      select * from (values
-        ('2023-01-09 00:00:00 Z'::timestamptz, 317798),
-        ('2023-01-16 00:00:00 Z'::timestamptz, 420280),
-        ('2023-01-23 00:00:00 Z'::timestamptz, 487695),
-        ('2023-01-30 00:00:00 Z'::timestamptz, 458967),
-        ('2023-02-06 00:00:00 Z'::timestamptz, 490907),
-        ('2023-02-13 00:00:00 Z'::timestamptz, 397445),
-        ('2023-02-20 00:00:00 Z'::timestamptz, 342812),
-        ('2023-02-27 00:00:00 Z'::timestamptz, 327501),
-        ('2023-03-06 00:00:00 Z'::timestamptz, 401014),
-        ('2023-03-13 00:00:00 Z'::timestamptz, 304623)
-      ) as t (date,value)
-      union
-      select date, value from get_collected_values(3, '2023-03-20 00:00:00 Z'::timestamptz, {{ .Now | timestamptz }})
-      order by date;
+      SELECT date, value
+      FROM get_collected_values(3, {{ .StartOfWeek | timestamptz }}-'14 week'::interval, {{ .Now | timestamptz }})
+      WHERE date NOT IN (
+        '2024-09-09 00:00:00 Z'::timestamptz,
+        '2024-09-16 00:00:00 Z'::timestamptz,
+        '2024-09-23 00:00:00 Z'::timestamptz,
+        '2024-09-30 00:00:00 Z'::timestamptz
+      )
+      ORDER BY date;
 
   - name: "dht servers"
     source: "nebula_ipfs"
@@ -127,7 +122,6 @@ series:
     color: "black"
     marker: "circle"
 
-
 layout:
   barmode: stack
   title:
@@ -144,4 +138,5 @@ layout:
     autorange: true
     rangemode: "tozero"
     title:
-      text: "Number of unique peers"
+      text: "Number of unique peers"
+
diff --git a/config/plotdefs/ipfs-unique-agents.yaml b/config/plotdefs/ipfs-unique-agents.yaml
@@ -5,8 +5,16 @@ datasets:
   - name: "preload unique agents"
     source: "caracol"
     query: >
-      select date, value from get_collected_values(17, {{ .StartOfWeek | timestamptz }}-'14 week'::interval, {{ .Now | timestamptz }})
-      order by date;
+      SELECT date, value
+      FROM get_collected_values(17, {{ .StartOfWeek | timestamptz }}-'14 week'::interval, {{ .Now | timestamptz }})
+      WHERE date NOT IN (
+        '2024-09-09 00:00:00 Z'::timestamptz,
+        '2024-09-16 00:00:00 Z'::timestamptz,
+        '2024-09-23 00:00:00 Z'::timestamptz,
+        '2024-09-30 00:00:00 Z'::timestamptz
+      )
+      ORDER BY date;
+
   - name: "nebula unique agents"
     source: "nebula_ipfs"
     query: >
@@ -82,4 +90,5 @@ layout:
     autorange: true
     rangemode: "tozero"
     title:
-      text: "Number of unique agents"
+      text: "Number of unique agents"
+
diff --git a/content.en/ipfs/kpi/_index.md b/content.en/ipfs/kpi/_index.md
@@ -12,6 +12,13 @@ Our KPIs are currently focusing primarily on the public [IPFS DHT](https://docs.
 
 ## Network Size & Stability
 
+<!--TODO: remove after gap isn't visible anymore-->
+{{< hint info >}}
+⚠️ Due to a migration, data from preload servers was not collected between
+`2024-09-07` and `2024-09-26`. As a result, the DHT clients count and bootstrap
+unique agents for this period are unavailable.
+{{< /hint >}}
+
 ### Client vs Server Node Estimate
 
 The total number of peers in the network is estimated using the number of unique [Peer IDs](https://docs.ipfs.tech/concepts/glossary/#peer-id) seen by Protocol Labs' [bootstrap nodes](https://docs.ipfs.tech/concepts/glossary/#bootstrap-node). The number of unique [DHT Server](https://docs.ipfs.tech/concepts/dht/#routing-tables) peer IDs identified by the [Nebula crawler](/tools/nebula/) is then subtracted from the total number of peers (seen by the bootstrappers) to estimate the number of peers that exclusively function as DHT Clients.

diff --git a/content.en/plots/2024/10/07/ipfs-unique-agents.json b/content.en/plots/2024/10/07/ipfs-unique-agents.json
@@ -1 +1 @@
-{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,519,797,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null}
+{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null}
diff --git a/content.en/plots/latest/ipfs-servers-vs-clients.json b/content.en/plots/latest/ipfs-servers-vs-clients.json
@@ -1 +1 @@
-{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Servers","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[46765,46216,45090,46471,71321,71548,70538,70621,70258,65908,63746,63683,64576,54748,44456],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"DHT Clients","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[150778,167309,158079,157676,197012,224680,231106,295128,245549,210690,61465,79951,179234],"yaxis":""},{"type":"scatter","marker":{"color":"black","symbol":"circle"},"mode":"lines+markers","name":"DHT Server IP Addresses","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[64565,60568,60247,61050,142903,144477,136871,138850,134683,120864,108473,112514,102854,63695,43577],"yaxis":""}],"layout":{"annotations":null,"barmode":"stack","legend":{"orientation":"h"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique peers"}}},"params":null,"dynamicLayout":null,"config":null}
+{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Servers","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[46765,46216,45090,46471,71321,71548,70538,70621,70258,65908,63746,63683,64576,54748,44456],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"DHT Clients","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-10-07T00:00:00Z"],"y":[150778,167309,158079,157676,197012,224680,231106,295128,245549,210690,179234],"yaxis":""},{"type":"scatter","marker":{"color":"black","symbol":"circle"},"mode":"lines+markers","name":"DHT Server IP Addresses","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[64565,60568,60247,61050,142903,144477,136871,138850,134683,120864,108473,112514,102854,63695,43577],"yaxis":""}],"layout":{"annotations":null,"barmode":"stack","legend":{"orientation":"h"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique peers"}}},"params":null,"dynamicLayout":null,"config":null}
diff --git a/content.en/plots/latest/ipfs-unique-agents.json b/content.en/plots/latest/ipfs-unique-agents.json
@@ -1 +1 @@
-{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,519,797,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null}
+{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null}
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,519,797,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null}
		{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null}