Initial commit

8 files changed, 609 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..c1d18d8
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+_out
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..3c6177f
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Carlo Landmeter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..682dcf2
--- /dev/null
+++ b/README.md
@@ -0,0 +1,2 @@
+# alpine-mirror-status
+Scripts to generate Alpine mirror statistics
diff --git a/apkindex.list b/apkindex.list
new file mode 100644
index 0000000..49c2b89
--- /dev/null
+++ b/apkindex.list
@@ -0,0 +1,86 @@
+alpine/edge/community/aarch64/APKINDEX.tar.gz
+alpine/edge/community/all/APKINDEX.tar.gz
+alpine/edge/community/armhf/APKINDEX.tar.gz
+alpine/edge/community/ppc64le/APKINDEX.tar.gz
+alpine/edge/community/s390x/APKINDEX.tar.gz
+alpine/edge/community/x86/APKINDEX.tar.gz
+alpine/edge/community/x86_64/APKINDEX.tar.gz
+alpine/edge/main/aarch64/APKINDEX.tar.gz
+alpine/edge/main/armhf/APKINDEX.tar.gz
+alpine/edge/main/ppc64le/APKINDEX.tar.gz
+alpine/edge/main/s390x/APKINDEX.tar.gz
+alpine/edge/main/x86/APKINDEX.tar.gz
+alpine/edge/main/x86_64/APKINDEX.tar.gz
+alpine/edge/testing/aarch64/APKINDEX.tar.gz
+alpine/edge/testing/armhf/APKINDEX.tar.gz
+alpine/edge/testing/ppc64le/APKINDEX.tar.gz
+alpine/edge/testing/s390x/APKINDEX.tar.gz
+alpine/edge/testing/x86/APKINDEX.tar.gz
+alpine/edge/testing/x86_64/APKINDEX.tar.gz
+alpine/v2.4/main/x86/APKINDEX.tar.gz
+alpine/v2.4/main/x86_64/APKINDEX.tar.gz
+alpine/v2.4/testing/x86/APKINDEX.tar.gz
+alpine/v2.4/testing/x86_64/APKINDEX.tar.gz
+alpine/v2.5/main/x86/APKINDEX.tar.gz
+alpine/v2.5/main/x86_64/APKINDEX.tar.gz
+alpine/v2.6/main/x86/APKINDEX.tar.gz
+alpine/v2.6/main/x86_64/APKINDEX.tar.gz
+alpine/v2.7/backports/x86/APKINDEX.tar.gz
+alpine/v2.7/backports/x86_64/APKINDEX.tar.gz
+alpine/v2.7/main/x86/APKINDEX.tar.gz
+alpine/v2.7/main/x86_64/APKINDEX.tar.gz
+alpine/v3.0/main/x86/APKINDEX.tar.gz
+alpine/v3.0/main/x86_64/APKINDEX.tar.gz
+alpine/v3.0/testing/x86/APKINDEX.tar.gz
+alpine/v3.0/testing/x86_64/APKINDEX.tar.gz
+alpine/v3.1/main/armhf/APKINDEX.tar.gz
+alpine/v3.1/main/x86/APKINDEX.tar.gz
+alpine/v3.1/main/x86_64/APKINDEX.tar.gz
+alpine/v3.1/testing/armhf/APKINDEX.tar.gz
+alpine/v3.2/main/armhf/APKINDEX.tar.gz
+alpine/v3.2/main/x86/APKINDEX.tar.gz
+alpine/v3.2/main/x86_64/APKINDEX.tar.gz
+alpine/v3.3/community/armhf/APKINDEX.tar.gz
+alpine/v3.3/community/x86/APKINDEX.tar.gz
+alpine/v3.3/community/x86_64/APKINDEX.tar.gz
+alpine/v3.3/main/armhf/APKINDEX.tar.gz
+alpine/v3.3/main/x86/APKINDEX.tar.gz
+alpine/v3.3/main/x86_64/APKINDEX.tar.gz
+alpine/v3.4/community/armhf/APKINDEX.tar.gz
+alpine/v3.4/community/x86/APKINDEX.tar.gz
+alpine/v3.4/community/x86_64/APKINDEX.tar.gz
+alpine/v3.4/main/armhf/APKINDEX.tar.gz
+alpine/v3.4/main/x86/APKINDEX.tar.gz
+alpine/v3.4/main/x86_64/APKINDEX.tar.gz
+alpine/v3.5/community/aarch64/APKINDEX.tar.gz
+alpine/v3.5/community/armhf/APKINDEX.tar.gz
+alpine/v3.5/community/x86/APKINDEX.tar.gz
+alpine/v3.5/community/x86_64/APKINDEX.tar.gz
+alpine/v3.5/main/aarch64/APKINDEX.tar.gz
+alpine/v3.5/main/armhf/APKINDEX.tar.gz
+alpine/v3.5/main/x86/APKINDEX.tar.gz
+alpine/v3.5/main/x86_64/APKINDEX.tar.gz
+alpine/v3.6/community/aarch64/APKINDEX.tar.gz
+alpine/v3.6/community/armhf/APKINDEX.tar.gz
+alpine/v3.6/community/ppc64le/APKINDEX.tar.gz
+alpine/v3.6/community/s390x/APKINDEX.tar.gz
+alpine/v3.6/community/x86/APKINDEX.tar.gz
+alpine/v3.6/community/x86_64/APKINDEX.tar.gz
+alpine/v3.6/main/aarch64/APKINDEX.tar.gz
+alpine/v3.6/main/armhf/APKINDEX.tar.gz
+alpine/v3.6/main/ppc64le/APKINDEX.tar.gz
+alpine/v3.6/main/s390x/APKINDEX.tar.gz
+alpine/v3.6/main/x86/APKINDEX.tar.gz
+alpine/v3.6/main/x86_64/APKINDEX.tar.gz
+alpine/v3.7/community/aarch64/APKINDEX.tar.gz
+alpine/v3.7/community/armhf/APKINDEX.tar.gz
+alpine/v3.7/community/ppc64le/APKINDEX.tar.gz
+alpine/v3.7/community/s390x/APKINDEX.tar.gz
+alpine/v3.7/community/x86/APKINDEX.tar.gz
+alpine/v3.7/community/x86_64/APKINDEX.tar.gz
+alpine/v3.7/main/aarch64/APKINDEX.tar.gz
+alpine/v3.7/main/armhf/APKINDEX.tar.gz
+alpine/v3.7/main/ppc64le/APKINDEX.tar.gz
+alpine/v3.7/main/s390x/APKINDEX.tar.gz
+alpine/v3.7/main/x86/APKINDEX.tar.gz
+alpine/v3.7/main/x86_64/APKINDEX.tar.gz
diff --git a/generate-html.lua b/generate-html.lua
new file mode 100755
index 0000000..4decea2
--- /dev/null
+++ b/generate-html.lua
@@ -0,0 +1,157 @@
+#!/usr/bin/lua5.3
+
+local json = require("cjson")
+local inspect = require("inspect")
+local lustache = require("lustache")
+local utils = require("utils")
+
+local outdir = "_out"
+local mirrors_file = outdir.."/mirror-status.json"
+
+function get_branches(indexes)
+	local res = {}
+	for k,v in ipairs(indexes.master.branch) do
+		table.insert(res, v.name)
+	end
+	return res
+end
+
+----
+-- need to create one flipped table with both mirros and master
+function flip_branches(branches)
+	local res = {}
+	for _,b in ipairs(branches) do
+		for _,r in ipairs(b.repo) do
+			for _,a in ipairs(r.arch) do
+				if type(res[b.name]) == "nil" then res[b.name] = {} end
+				if type(res[b.name][r.name]) == "nil" then
+					res[b.name][r.name] = {}
+				end
+				res[b.name][r.name][a.name] = a
+			end
+		end
+	end
+	return res
+end
+
+----
+-- convert table to array (values to keys)
+function get_repo_arch(indexes)
+	local t = {}
+	for _,b in ipairs(indexes.master.branch) do
+		for _,r in ipairs(b.repo) do
+			for _,a in ipairs(r.arch) do
+				if type(t[r.name]) == "nil" then t[r.name] = {} end
+				t[r.name][a.name] = 1
+			end
+		end
+	end
+	local res = {}
+	for repo in utils.kpairs(t, utils.sort_repo) do
+		for arch in utils.kpairs(t[repo], utils.sort_arch) do
+			table.insert(res, repo.."/"..arch)
+		end
+	end
+	return res
+end
+
+----
+-- convert table to array (values to keys) for each mirror
+function flip_mirrors(mirrors)
+	local res = {}
+	for _,m in ipairs(mirrors) do
+		res[m.url] = flip_branches(m.branch)
+	end
+	return res
+end
+
+----
+-- format timestamp difference
+function format_age(ts)
+	local res = {}
+	if ts < 3600 then
+		res.text = "OK"
+		res.class = "status-ok"
+	elseif ts < 86400 then
+		res.text = ("%dh"):format(math.ceil(ts/3600))
+		res.class = "status-warn"
+	elseif ts > 86400 then
+		res.text = ("%dd"):format(math.ceil(ts/86400))
+		res.class = "status-error"
+	end
+	return res
+end
+
+----
+-- format status based on http status message
+function format_status(status, age)
+	local res = {}
+	if status ~= "failed" then
+		if status == "200" then
+			res = format_age(age)
+		elseif status == "404" then
+			res.class = "status-na"
+			res.text = "N/A"
+		else
+			res.class = "status-unk"
+			res.text = status
+		end
+	else
+		res.class = "status-na"
+		res.text = "N/A"
+	end
+	return res
+end
+
+----
+-- get status and format it based on http status and modified time
+function get_status(fm, fb, mirror, branch, repo, arch)
+	local res = { text = "N/A", class = "status-na" }
+	if type(fm[mirror]) == "table" and
+		type(fm[mirror][branch]) == "table" and
+		type(fm[mirror][branch][repo]) == "table" and
+		type(fm[mirror][branch][repo][arch]) == "table" then
+		if type(fm[mirror][branch][repo][arch]) == "table" then
+			local status = fm[mirror][branch][repo][arch]
+			local age
+			if type(status.modified) == "number" then
+				age = fb[branch][repo][arch].modified - status.modified
+			end
+			res = format_status(status.status, age)
+		end
+	end
+	return res
+end
+
+----
+-- build the html table
+function build_tables(indexes)
+	local res = {}
+	local fm = flip_mirrors(indexes.mirrors)
+	local fb = flip_branches(indexes.master.branch)
+	for idx,mirror in ipairs(indexes.mirrors) do
+		local rows = {}
+		for _,ra in ipairs(get_repo_arch(indexes)) do
+			local repo, arch = ra:match("(.*)/(.*)")
+			local row = {}
+			table.insert(row, {text = ra})
+			for _,branch in ipairs(get_branches(indexes)) do
+				local status = get_status(fm, fb, mirror.url, branch, repo, arch)
+				table.insert(row, status)
+			end
+			table.insert(rows, { row = row })
+		end
+		res[idx] = { 
+			url = mirror.url, tbody = rows, duration = mirror.duration, 
+			count = mirror.count
+		}
+	end
+	return res
+end
+
+local indexes = json.decode(utils.read_file(mirrors_file))
+local thead = get_branches(indexes)
+table.insert(thead, 1, "branch/release")
+local view = { lupdate = os.date("%c", indexes.date), mirrors = build_tables(indexes), thead = thead }
+local tpl = utils.read_file("index.tpl")
+utils.write_file(outdir.."/output.html", lustache:render(tpl, view))
diff --git a/generate-json.lua b/generate-json.lua
new file mode 100755
index 0000000..1af51cb
--- /dev/null
+++ b/generate-json.lua
@@ -0,0 +1,150 @@
+#!/usr/bin/lua5.3
+
+local inspect = require("inspect")
+local request = require("http.request")
+local yaml = require("yaml")
+local json = require("cjson")
+local utils = require("utils")
+
+local app_version = "v0.0.1"
+local apkindex_list = "apkindex.list"
+local mirrors_yaml = "https://git.alpinelinux.org/cgit/aports/plain/main/alpine-mirrors/mirrors.yaml"
+local master = "http://rsync.alpinelinux.org/alpine/"
+local output = "_out/mirror-status.json"
+local http_timeout = 3
+
+----
+-- convert apkindex list to a table
+function get_apkindexes()
+	local res = {}
+	local qty = 0
+	for line in io.lines(apkindex_list) do
+		branch, repo, arch = line:match("^alpine/(.*)/(.*)/(.*)/APKINDEX.tar.gz")
+		if type(res[branch]) == "nil" then res[branch] = {} end
+		if type(res[branch][repo]) == "nil" then res[branch][repo] = {} end
+		res[branch][repo][arch] = 1
+		qty = qty + 1
+	end
+	return res, qty
+end
+
+----
+-- convert last-modified header date to timestamp
+function rfc2616_date_to_ts(s)
+	local day,month,year,hour,min,sec
+	local m = { Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9,
+		Oct=10,Nov=11,Dec=12 }
+	local format = "%a+, (%d+) (%a+) (%d+) (%d+):(%d+):(%d+) GMT"
+	local day,month,year,hour,min,sec = s:match(format)
+	return os.time({day=day,month=m[month],year=year,hour=hour,min=min,sec=sec})
+end
+
+----
+-- get a list of http urls from mirrors yaml
+function get_mirrors(uri)
+	local res = {}
+	local headers, stream = assert(request.new_from_uri(uri):go())
+	if headers:get(":status") ~= "200" then
+		error("Failed to get mirrors yaml!")
+	end
+	local y = assert(stream:get_body_as_string())
+	local mirrors = yaml.load(y)
+	for idx, mirror in ipairs(mirrors) do
+		for _,url in ipairs(mirror.urls) do
+			if url:match("http://") then
+				table.insert(res, url)
+			end
+		end
+	end
+	return res
+end
+
+function get_index_status(uri)
+	local res = {}
+	local status, modified
+	local headers = request.new_from_uri(uri):go(http_timeout)
+	if headers then 
+		status = headers:get(":status")
+	else
+		return "failed"
+	end
+	if status == "200" then
+		modified = headers:get("last-modified")
+		modified = rfc2616_date_to_ts(modified)
+	end
+	return status, modified
+end
+
+--- write results to json file on disk
+function write_json(t)
+	local f = assert(io.open(output, "w"))
+	local json = assert(json.encode(t))
+	f:write(json)
+	f:close()
+end
+
+--- show a process indicator on stdout
+function progress(num)
+	num = (num < 10) and "0"..num or num
+	io.write(("Indexes left: %s\r"):format(num))
+	io.flush()
+end
+
+-- check all apkindex for specific mirror
+function check_apkindexes(mirror)
+	local indexes, num_indexes = get_apkindexes()
+	local branches = {}
+	local qty = 0
+	local cnt = 0
+	for branch in utils.kpairs(indexes, utils.sort_branch) do
+		local repos = {}
+		for repo in utils.kpairs(indexes[branch], utils.sort_repo) do
+			local archs = {}
+			for arch in utils.kpairs(indexes[branch][repo], utils.sort_arch) do
+				if type(utils.allowed.archs[arch]) == "number" then
+					local uri = ("%s/%s/%s/%s/APKINDEX.tar.gz"):format(mirror, branch, repo, arch)
+					status, modified = get_index_status(uri)
+					table.insert(archs, {name=arch, status=status, modified=modified})
+					if status == "200" then qty = qty+1 end
+				end
+				cnt = cnt + 1
+				progress(num_indexes-cnt)
+			end
+			table.insert(repos, {name=repo, arch=archs})
+		end
+		table.insert(branches, {name=branch, repo=repos})
+	end
+	return branches, qty
+end
+
+function process_mirrors()
+	local res = {}
+	local mirrors = get_mirrors(mirrors_yaml)
+	for idx,mirror in ipairs(mirrors) do
+		local start_time = os.time()
+		res[idx] = {}
+		res[idx].url = mirror
+		print(("[%s/%s] Getting indexes from mirror: %s"):format(idx, 
+			#mirrors, mirror))
+		res[idx].branch, res[idx].count = check_apkindexes(mirror)
+		res[idx].duration = os.difftime(os.time(),start_time)
+	end
+	return res
+end
+
+function process_master()
+	print(("Getting indexes from master: %s"):format(master))
+	local res = {}
+	res.url = master
+	res.branch = check_apkindexes(master)
+	return res
+end
+
+write_json(
+	{
+		master = process_master(),
+		mirrors = process_mirrors(),
+		date = os.time(),
+		version = app_version
+	}
+)
diff --git a/index.tpl b/index.tpl
new file mode 100644
index 0000000..ac381af
--- /dev/null
+++ b/index.tpl
@@ -0,0 +1,123 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Mirror health</title>
+    <link rel="stylesheet" href="https://unpkg.com/purecss@1.0.0/build/pure-min.css">    <!--[if lte IE 8]>
+        <link rel="stylesheet" href="https://unpkg.com/purecss@1.0.0/build/grids-responsive-old-ie-min.css">
+    <![endif]-->
+    <!--[if gt IE 8]><!-->
+        <link rel="stylesheet" href="https://unpkg.com/purecss@1.0.0/build/grids-responsive-min.css">
+    <!--<![endif]-->
+	<style>
+	html, body {
+		margin: 0;
+		padding: 0;
+		height: 100%;
+		font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial,
+			sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
+		color: #526066;
+		font-size: 1.0em;
+	}
+	div#wrapper {
+		min-height: 100%;
+		position: relative;
+	}
+	header {
+		border-bottom: 1px solid #eaecef;
+	}
+	.logo {
+	    padding:1em;
+	}
+	#content {
+	    margin: 0 auto;
+	    padding: 0em 1em 2em 1em;
+	    max-width: 1080px;
+	    padding-bottom: 1em;
+	}
+	footer {
+	    background: #111;
+	    color: #888;
+	    text-align: center;
+	    position: absolute;
+	    bottom: 0;
+	    width: 100%;
+	    padding-top: 0.5em;
+	    padding-bottom: 0.5em;
+	    font-size: 0.8em;
+	}
+	.last-updated {
+		color: #ccc;
+		margin: 1em;
+		margin-bottom: 2em;
+	}
+	.mirror-meta {
+		margin: 1em;
+		margin-left: 0;
+	}
+	.status-ok {
+		color: #228B22;
+		font-weight: bold;
+	}
+	.status-na {
+		color: #ccc;
+	}
+	.status-warn {
+		color: #DAA520;
+		font-weight: bold;
+	}
+	.status-error, .status-unk {
+		color: #8B0000;
+		font-weight: bold;
+	}
+	</style>
+</head>
+
+<body>
+	<div id="wrapper">
+		<header class="pure-g">
+			<div class="pure-u-1 pure-u-lg-4-24">
+				<div class="logo">
+					<a href="/"><img src="https://alpinelinux.org/alpinelinux-logo.svg" alt="Alpine Logo" class="pure-img"></a>
+				</div>
+			</div>
+		</header>
+		<div id="content">
+			<div class="pure-g">
+				{{#mirrors}}
+				<div class="pure-u-1">
+					<h3>{{url}}</h3>
+					<div class="mirror-meta">
+						<ul class="mirror-meta">
+							<li>Generated in {{duration}} seconds.</li>
+							<li>Found {{count}} indexes.</li>
+						</ul>
+					</div>
+					<table class="pure-table pure-table-striped">
+						<thead>
+							<tr>
+								{{#thead}}
+								<th>{{.}}</th>
+								{{/thead}}
+							</tr>
+						</thead>
+						<tbody>
+							{{#tbody}}
+							<tr>
+								{{#row}}
+								<td class="{{class}}">{{text}}</td>
+								{{/row}}
+							</tr>
+							{{/tbody}}
+						</tbody>
+					</table>
+				</div>
+				{{/mirrors}}
+			</div>
+			<div class="last-updated">Last updated: <span>{{lupdate}}</span> UTC</div>
+		</div>
+		<footer>© Copyright 2017 Alpine Linux Development Team all rights reserved</footer>
+	</div>
+</body>
+</html>
\ No newline at end of file
diff --git a/utils.lua b/utils.lua
new file mode 100644
index 0000000..d53100a
--- /dev/null
+++ b/utils.lua
@@ -0,0 +1,69 @@
+local M = {}
+
+
+M.allowed = {
+	archs = { x86=1, x86_64=2, armhf=3, aarch64=4, ppc64le=5, s390x=6 },
+	repos = { main=1, community=2, testing=3, backports=4 }
+}
+
+function M.in_array(t, value)
+	for k,v in ipairs(t) do
+		if v == value then
+			return true
+		end
+	end
+end
+
+function M.read_file(file)
+	local f = assert(io.open(file))
+	local file = f:read("*all")
+	f:close()
+	return file
+end
+
+function M.write_file(file, string)
+	file = assert(io.open(file, "w"))
+	file:write(string)
+	file:close()
+end
+
+----
+-- table iterator which sorts on keys
+function M.kpairs(t, f)
+    local keys = {}
+    for k in pairs(t) do keys[#keys + 1] = k end
+    table.sort(keys,f)
+    local i = 0
+    return function()
+        i = i + 1
+        return keys[i], t[keys[i]]
+    end
+end
+
+----
+-- branch sort function for kpairs
+function M.sort_branch(a,b)
+	if a == "edge" then a = "z" end
+	if b == "edge" then b = "z" end
+	if a < b then return true end
+end
+
+----
+-- repo sort function for kpairs
+function M.sort_repo(a,b)
+	local repos = M.allowed.repos
+	if type(repos[a]) == "number" and type(repos[b]) == "number" then
+		if repos[a] < repos[b] then return true end
+	end
+end
+
+----
+-- arch sort function for kpairs
+function M.sort_arch(a,b)
+	local archs = M.allowed.archs
+	if type(archs[a]) == "number" and type(archs[b]) == "number" then 
+		if archs[a] < archs[b] then return true end
+	end
+end
+
+return M
\ No newline at end of file