forked from InternetHealthReport/internet-yellow-pages
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_all_crawlers.sh
49 lines (37 loc) · 2.55 KB
/
run_all_crawlers.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
echo "Starting all crawlers"
date
# MANRS
python3 -m iyp.crawlers.manrs.members
# AS Names
python3 -m iyp.crawlers.ripe.as_names
python3 -m iyp.crawlers.bgptools.as_names
python3 -m iyp.crawlers.emileaben.as_names
# AS Peers
python3 -m iyp.crawlers.inetintel.siblings_asdb
# Rankings
python3 -m iyp.crawlers.apnic.eyeball
python3 -m iyp.crawlers.caida.asrank
python3 -m iyp.crawlers.ihr.country_dependency
# BGP data
python3 -m iyp.crawlers.bgpkit.pfx2asn
python3 -m iyp.crawlers.bgpkit.as2rel
python3 -m iyp.crawlers.bgpkit.peerstats
python3 -m iyp.crawlers.ripe.roa
# IHR
python3 -m iyp.crawlers.ihr.local_hegemony
python3 -m iyp.crawlers.ihr.rov
# DNS
python3 -m iyp.crawlers.tranco.top1M
python3 -m iyp.crawlers.cloudflare.top100
#BGP.Tools tags, and anycast prefixes
python3 -m iyp.crawlers.bgptools.tags
python3 -m iyp.crawlers.bgptools.anycast_prefixes
#PeeringDB
python3 -m iyp.crawlers.peeringdb.org
python3 -m iyp.crawlers.peeringdb.ix
# Delegated files
python3 -m iyp.crawlers.nro.delegated_stats
# URL data
python3 -m iyp.crawlers.citizenlab.urldb
echo "All crawlers finished"
date