1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
#!/usr/bin/python
#
# Generate html pages based on the Mirmon data
#
# This script will parse the json files that contain the mirmon
# data and have been generated before. The json data is used to
# generate html pages that visualize the mirmon data in a human
# readable way.
#
#
# Leaflet is used to visualize the mirrors on a world map. Further-
# more Datatables are used to display the mirrors.
#
import datetime
import socket
import ssl
import os
import tempfile
import urllib.request, json
import xml.etree.ElementTree as ET
import jinja2
html_folder = "/var/www/mirrorstats.gentoo.org/htdocs/"
template_folder = "/var/www/mirrorstats.gentoo.org/gentoo-mirrorstats/html/"
cache_path = "/var/www/mirrorstats.gentoo.org/var/html_data_cache.json"
mirrorstats = [ 'https://mirrorstats.gentoo.org/rsync/state.json',
'https://mirrorstats.gentoo.org/distfiles/state.json',
'https://mirrorstats.gentoo.org/snapshots/state.json',
'https://mirrorstats.gentoo.org/releases/state.json',
'https://mirrorstats.gentoo.org/experimental/state.json'
]
cache_data = {}
#
# Retrieve and parse the JSON at the given URL
#
def getJson(url):
req = urllib.request.Request(url)
r = urllib.request.urlopen(req).read()
return json.loads(r.decode('utf-8'))
#
# Retrieve and parse the XML at the given URL
#
def getXML(url):
req = urllib.request.Request(url)
r = urllib.request.urlopen(req).read()
return ET.loads(r.decode('utf-8')).getroot()
#
# Get the ip of the given host
#
def getIp(hostname):
try:
ip = socket.gethostbyname(hostname)
except:
ip = ""
return ip
#
# Get TLS version of the given host
#
def getTLSVersion(hostname):
socket.setdefaulttimeout(2)
try:
with socket.create_connection((hostname, 443)) as sock:
with ssl.create_default_context().wrap_socket(sock, server_hostname=hostname) as ssock:
return ssock.version().replace("TLSv", "", 1)
except:
print("err")
return ''
#
# Render the stats template for a given page, i.e. all, rsync, distfiles...
#
def renderStatsTemplate(templateEnv, page):
if page == "all":
mirrorstatsList = mirrorstats
page_name = "index"
else:
mirrorstatsList = ['https://mirrorstats.gentoo.org/' + page + '/state.json']
page_name = page
lastUpdate, hostList = getHostList(mirrorstatsList)
template = templateEnv.get_template("stats.jinja2")
template.stream(lastUpdate=lastUpdate, type=page, mirrors=hostList, locations=json.dumps(hostList)).dump(html_folder + page_name + ".html")
return lastUpdate
# read the cache
if os.path.exists(cache_path):
with open(cache_path, mode='rt') as json_file:
try:
cache_data = json.load(json_file)
if 'ips' not in cache_data:
cache_data['ips'] = {}
if 'hosts' not in cache_data:
cache_data['hosts'] = {}
if 'tls_version' not in cache_data:
cache_data['tls_version'] = {}
except:
pass
#
# The all mirrors that are present in the given list
#
def getHostList(mirrorstatsList):
hosts = {}
lastUpdate = ""
# process all mirrors
for mirror_url in mirrorstatsList:
mirrorData = None
try:
mirrorData = getJson(mirror_url)
except Exception as err:
print("Exception fetching "+mirror_url+": ", type(err).__name__, err)
continue
lastUpdate = datetime.datetime.fromtimestamp(int(mirrorData['LastUpdate'])).strftime('%Y-%m-%d %H:%M:%S')
for mirror in mirrorData['Mirrors']:
if len(mirrorData['Mirrors'][mirror]) >= 0:
for mirrorHost in mirrorData['Mirrors'][mirror]:
hostname = mirrorHost['Host']
if hostname not in hosts:
hosts[hostname] = {}
hosts[hostname]['Hostname'] = hostname
if hostname in cache_data['hosts']:
ip = cache_data['hosts'][hostname]
else:
ip = getIp(hostname)
# populate cache with new value
cache_data['hosts'][hostname] = ip
if ip != "":
hosts[hostname]['Ip'] = ip
if 'Stats' not in hosts[hostname]:
hosts[hostname]['Stats'] = []
mirrorHost['Type'] = mirror_url.replace("https://mirrorstats.gentoo.org/", "").replace("/state.json", "")
hosts[hostname]['Stats'].append(mirrorHost)
# compute available protocols
for host in hosts:
protocols = []
for stat in hosts[host]['Stats']:
protocols.append(stat['Protocol'])
hosts[host]['Protocols'] = list(set(protocols))
# compute mirror locations
for host in hosts:
if 'Ip' not in hosts[host]:
continue
ip = hosts[host]['Ip']
if ip in cache_data['ips']:
hosts[host]['Location'] = cache_data['ips'][ip]
else:
mirrorGeoData = getJson("https://ipinfo.io/" + ip + "/json")
hosts[host]['Location'] = mirrorGeoData['loc']
# populate cache with new value
cache_data['ips'][ip] = mirrorGeoData['loc']
for host in hosts:
if 'https' not in hosts[host]['Protocols']:
hosts[host]['TLSVersion'] = ''
continue
if host in cache_data['tls_version']:
hosts[host]['TLSVersion'] = cache_data['tls_version'][host]
else:
tlsVersion = getTLSVersion(host)
hosts[host]['TLSVersion'] = tlsVersion
cache_data['tls_version'][host] = tlsVersion
return lastUpdate, hosts
#
# render jinja2
#
templateLoader = jinja2.FileSystemLoader(searchpath=template_folder)
templateEnv = jinja2.Environment(loader=templateLoader)
## stats
lastUpdate = renderStatsTemplate(templateEnv, "all")
renderStatsTemplate(templateEnv, "rsync")
renderStatsTemplate(templateEnv, "distfiles")
renderStatsTemplate(templateEnv, "snapshots")
renderStatsTemplate(templateEnv, "releases")
renderStatsTemplate(templateEnv, "experimental")
## about
template = templateEnv.get_template("help.jinja2")
template.stream(lastUpdate=lastUpdate).dump(html_folder + "help.html")
#
# write the cache
#
with tempfile.NamedTemporaryFile(dir=os.path.dirname(cache_path), delete=False, mode='wt') as fout:
json.dump(cache_data, fout)
os.chmod(fout.name, 0o644)
os.replace(fout.name, cache_path)
#
# finish
#
#print("Finished.")
|
GitWeb