From 44b7203dc9fe828e917eb10e8af98ca366ac53b0 Mon Sep 17 00:00:00 2001 From: Carlo Landmeter Date: Wed, 29 Nov 2017 09:28:44 +0100 Subject: Initial commit --- .gitignore | 1 + LICENSE | 21 ++++++++ README.md | 2 + apkindex.list | 86 ++++++++++++++++++++++++++++++ generate-html.lua | 157 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ generate-json.lua | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++ index.tpl | 123 ++++++++++++++++++++++++++++++++++++++++++ utils.lua | 69 ++++++++++++++++++++++++ 8 files changed, 609 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 apkindex.list create mode 100755 generate-html.lua create mode 100755 generate-json.lua create mode 100644 index.tpl create mode 100644 utils.lua diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c1d18d8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +_out diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3c6177f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Carlo Landmeter + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..682dcf2 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# alpine-mirror-status +Scripts to generate Alpine mirror statistics diff --git a/apkindex.list b/apkindex.list new file mode 100644 index 0000000..49c2b89 --- /dev/null +++ b/apkindex.list @@ -0,0 +1,86 @@ +alpine/edge/community/aarch64/APKINDEX.tar.gz +alpine/edge/community/all/APKINDEX.tar.gz +alpine/edge/community/armhf/APKINDEX.tar.gz +alpine/edge/community/ppc64le/APKINDEX.tar.gz +alpine/edge/community/s390x/APKINDEX.tar.gz +alpine/edge/community/x86/APKINDEX.tar.gz +alpine/edge/community/x86_64/APKINDEX.tar.gz +alpine/edge/main/aarch64/APKINDEX.tar.gz +alpine/edge/main/armhf/APKINDEX.tar.gz +alpine/edge/main/ppc64le/APKINDEX.tar.gz +alpine/edge/main/s390x/APKINDEX.tar.gz +alpine/edge/main/x86/APKINDEX.tar.gz +alpine/edge/main/x86_64/APKINDEX.tar.gz +alpine/edge/testing/aarch64/APKINDEX.tar.gz +alpine/edge/testing/armhf/APKINDEX.tar.gz +alpine/edge/testing/ppc64le/APKINDEX.tar.gz +alpine/edge/testing/s390x/APKINDEX.tar.gz +alpine/edge/testing/x86/APKINDEX.tar.gz +alpine/edge/testing/x86_64/APKINDEX.tar.gz +alpine/v2.4/main/x86/APKINDEX.tar.gz +alpine/v2.4/main/x86_64/APKINDEX.tar.gz +alpine/v2.4/testing/x86/APKINDEX.tar.gz +alpine/v2.4/testing/x86_64/APKINDEX.tar.gz +alpine/v2.5/main/x86/APKINDEX.tar.gz +alpine/v2.5/main/x86_64/APKINDEX.tar.gz +alpine/v2.6/main/x86/APKINDEX.tar.gz +alpine/v2.6/main/x86_64/APKINDEX.tar.gz +alpine/v2.7/backports/x86/APKINDEX.tar.gz +alpine/v2.7/backports/x86_64/APKINDEX.tar.gz +alpine/v2.7/main/x86/APKINDEX.tar.gz +alpine/v2.7/main/x86_64/APKINDEX.tar.gz +alpine/v3.0/main/x86/APKINDEX.tar.gz +alpine/v3.0/main/x86_64/APKINDEX.tar.gz +alpine/v3.0/testing/x86/APKINDEX.tar.gz +alpine/v3.0/testing/x86_64/APKINDEX.tar.gz +alpine/v3.1/main/armhf/APKINDEX.tar.gz +alpine/v3.1/main/x86/APKINDEX.tar.gz +alpine/v3.1/main/x86_64/APKINDEX.tar.gz +alpine/v3.1/testing/armhf/APKINDEX.tar.gz +alpine/v3.2/main/armhf/APKINDEX.tar.gz +alpine/v3.2/main/x86/APKINDEX.tar.gz +alpine/v3.2/main/x86_64/APKINDEX.tar.gz +alpine/v3.3/community/armhf/APKINDEX.tar.gz +alpine/v3.3/community/x86/APKINDEX.tar.gz +alpine/v3.3/community/x86_64/APKINDEX.tar.gz +alpine/v3.3/main/armhf/APKINDEX.tar.gz +alpine/v3.3/main/x86/APKINDEX.tar.gz +alpine/v3.3/main/x86_64/APKINDEX.tar.gz +alpine/v3.4/community/armhf/APKINDEX.tar.gz +alpine/v3.4/community/x86/APKINDEX.tar.gz +alpine/v3.4/community/x86_64/APKINDEX.tar.gz +alpine/v3.4/main/armhf/APKINDEX.tar.gz +alpine/v3.4/main/x86/APKINDEX.tar.gz +alpine/v3.4/main/x86_64/APKINDEX.tar.gz +alpine/v3.5/community/aarch64/APKINDEX.tar.gz +alpine/v3.5/community/armhf/APKINDEX.tar.gz +alpine/v3.5/community/x86/APKINDEX.tar.gz +alpine/v3.5/community/x86_64/APKINDEX.tar.gz +alpine/v3.5/main/aarch64/APKINDEX.tar.gz +alpine/v3.5/main/armhf/APKINDEX.tar.gz +alpine/v3.5/main/x86/APKINDEX.tar.gz +alpine/v3.5/main/x86_64/APKINDEX.tar.gz +alpine/v3.6/community/aarch64/APKINDEX.tar.gz +alpine/v3.6/community/armhf/APKINDEX.tar.gz +alpine/v3.6/community/ppc64le/APKINDEX.tar.gz +alpine/v3.6/community/s390x/APKINDEX.tar.gz +alpine/v3.6/community/x86/APKINDEX.tar.gz +alpine/v3.6/community/x86_64/APKINDEX.tar.gz +alpine/v3.6/main/aarch64/APKINDEX.tar.gz +alpine/v3.6/main/armhf/APKINDEX.tar.gz +alpine/v3.6/main/ppc64le/APKINDEX.tar.gz +alpine/v3.6/main/s390x/APKINDEX.tar.gz +alpine/v3.6/main/x86/APKINDEX.tar.gz +alpine/v3.6/main/x86_64/APKINDEX.tar.gz +alpine/v3.7/community/aarch64/APKINDEX.tar.gz +alpine/v3.7/community/armhf/APKINDEX.tar.gz +alpine/v3.7/community/ppc64le/APKINDEX.tar.gz +alpine/v3.7/community/s390x/APKINDEX.tar.gz +alpine/v3.7/community/x86/APKINDEX.tar.gz +alpine/v3.7/community/x86_64/APKINDEX.tar.gz +alpine/v3.7/main/aarch64/APKINDEX.tar.gz +alpine/v3.7/main/armhf/APKINDEX.tar.gz +alpine/v3.7/main/ppc64le/APKINDEX.tar.gz +alpine/v3.7/main/s390x/APKINDEX.tar.gz +alpine/v3.7/main/x86/APKINDEX.tar.gz +alpine/v3.7/main/x86_64/APKINDEX.tar.gz diff --git a/generate-html.lua b/generate-html.lua new file mode 100755 index 0000000..4decea2 --- /dev/null +++ b/generate-html.lua @@ -0,0 +1,157 @@ +#!/usr/bin/lua5.3 + +local json = require("cjson") +local inspect = require("inspect") +local lustache = require("lustache") +local utils = require("utils") + +local outdir = "_out" +local mirrors_file = outdir.."/mirror-status.json" + +function get_branches(indexes) + local res = {} + for k,v in ipairs(indexes.master.branch) do + table.insert(res, v.name) + end + return res +end + +---- +-- need to create one flipped table with both mirros and master +function flip_branches(branches) + local res = {} + for _,b in ipairs(branches) do + for _,r in ipairs(b.repo) do + for _,a in ipairs(r.arch) do + if type(res[b.name]) == "nil" then res[b.name] = {} end + if type(res[b.name][r.name]) == "nil" then + res[b.name][r.name] = {} + end + res[b.name][r.name][a.name] = a + end + end + end + return res +end + +---- +-- convert table to array (values to keys) +function get_repo_arch(indexes) + local t = {} + for _,b in ipairs(indexes.master.branch) do + for _,r in ipairs(b.repo) do + for _,a in ipairs(r.arch) do + if type(t[r.name]) == "nil" then t[r.name] = {} end + t[r.name][a.name] = 1 + end + end + end + local res = {} + for repo in utils.kpairs(t, utils.sort_repo) do + for arch in utils.kpairs(t[repo], utils.sort_arch) do + table.insert(res, repo.."/"..arch) + end + end + return res +end + +---- +-- convert table to array (values to keys) for each mirror +function flip_mirrors(mirrors) + local res = {} + for _,m in ipairs(mirrors) do + res[m.url] = flip_branches(m.branch) + end + return res +end + +---- +-- format timestamp difference +function format_age(ts) + local res = {} + if ts < 3600 then + res.text = "OK" + res.class = "status-ok" + elseif ts < 86400 then + res.text = ("%dh"):format(math.ceil(ts/3600)) + res.class = "status-warn" + elseif ts > 86400 then + res.text = ("%dd"):format(math.ceil(ts/86400)) + res.class = "status-error" + end + return res +end + +---- +-- format status based on http status message +function format_status(status, age) + local res = {} + if status ~= "failed" then + if status == "200" then + res = format_age(age) + elseif status == "404" then + res.class = "status-na" + res.text = "N/A" + else + res.class = "status-unk" + res.text = status + end + else + res.class = "status-na" + res.text = "N/A" + end + return res +end + +---- +-- get status and format it based on http status and modified time +function get_status(fm, fb, mirror, branch, repo, arch) + local res = { text = "N/A", class = "status-na" } + if type(fm[mirror]) == "table" and + type(fm[mirror][branch]) == "table" and + type(fm[mirror][branch][repo]) == "table" and + type(fm[mirror][branch][repo][arch]) == "table" then + if type(fm[mirror][branch][repo][arch]) == "table" then + local status = fm[mirror][branch][repo][arch] + local age + if type(status.modified) == "number" then + age = fb[branch][repo][arch].modified - status.modified + end + res = format_status(status.status, age) + end + end + return res +end + +---- +-- build the html table +function build_tables(indexes) + local res = {} + local fm = flip_mirrors(indexes.mirrors) + local fb = flip_branches(indexes.master.branch) + for idx,mirror in ipairs(indexes.mirrors) do + local rows = {} + for _,ra in ipairs(get_repo_arch(indexes)) do + local repo, arch = ra:match("(.*)/(.*)") + local row = {} + table.insert(row, {text = ra}) + for _,branch in ipairs(get_branches(indexes)) do + local status = get_status(fm, fb, mirror.url, branch, repo, arch) + table.insert(row, status) + end + table.insert(rows, { row = row }) + end + res[idx] = { + url = mirror.url, tbody = rows, duration = mirror.duration, + count = mirror.count + } + end + return res +end + +local indexes = json.decode(utils.read_file(mirrors_file)) +local thead = get_branches(indexes) +table.insert(thead, 1, "branch/release") +local view = { lupdate = os.date("%c", indexes.date), mirrors = build_tables(indexes), thead = thead } +local tpl = utils.read_file("index.tpl") +utils.write_file(outdir.."/output.html", lustache:render(tpl, view)) diff --git a/generate-json.lua b/generate-json.lua new file mode 100755 index 0000000..1af51cb --- /dev/null +++ b/generate-json.lua @@ -0,0 +1,150 @@ +#!/usr/bin/lua5.3 + +local inspect = require("inspect") +local request = require("http.request") +local yaml = require("yaml") +local json = require("cjson") +local utils = require("utils") + +local app_version = "v0.0.1" +local apkindex_list = "apkindex.list" +local mirrors_yaml = "https://git.alpinelinux.org/cgit/aports/plain/main/alpine-mirrors/mirrors.yaml" +local master = "http://rsync.alpinelinux.org/alpine/" +local output = "_out/mirror-status.json" +local http_timeout = 3 + +---- +-- convert apkindex list to a table +function get_apkindexes() + local res = {} + local qty = 0 + for line in io.lines(apkindex_list) do + branch, repo, arch = line:match("^alpine/(.*)/(.*)/(.*)/APKINDEX.tar.gz") + if type(res[branch]) == "nil" then res[branch] = {} end + if type(res[branch][repo]) == "nil" then res[branch][repo] = {} end + res[branch][repo][arch] = 1 + qty = qty + 1 + end + return res, qty +end + +---- +-- convert last-modified header date to timestamp +function rfc2616_date_to_ts(s) + local day,month,year,hour,min,sec + local m = { Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, + Oct=10,Nov=11,Dec=12 } + local format = "%a+, (%d+) (%a+) (%d+) (%d+):(%d+):(%d+) GMT" + local day,month,year,hour,min,sec = s:match(format) + return os.time({day=day,month=m[month],year=year,hour=hour,min=min,sec=sec}) +end + +---- +-- get a list of http urls from mirrors yaml +function get_mirrors(uri) + local res = {} + local headers, stream = assert(request.new_from_uri(uri):go()) + if headers:get(":status") ~= "200" then + error("Failed to get mirrors yaml!") + end + local y = assert(stream:get_body_as_string()) + local mirrors = yaml.load(y) + for idx, mirror in ipairs(mirrors) do + for _,url in ipairs(mirror.urls) do + if url:match("http://") then + table.insert(res, url) + end + end + end + return res +end + +function get_index_status(uri) + local res = {} + local status, modified + local headers = request.new_from_uri(uri):go(http_timeout) + if headers then + status = headers:get(":status") + else + return "failed" + end + if status == "200" then + modified = headers:get("last-modified") + modified = rfc2616_date_to_ts(modified) + end + return status, modified +end + +--- write results to json file on disk +function write_json(t) + local f = assert(io.open(output, "w")) + local json = assert(json.encode(t)) + f:write(json) + f:close() +end + +--- show a process indicator on stdout +function progress(num) + num = (num < 10) and "0"..num or num + io.write(("Indexes left: %s\r"):format(num)) + io.flush() +end + +-- check all apkindex for specific mirror +function check_apkindexes(mirror) + local indexes, num_indexes = get_apkindexes() + local branches = {} + local qty = 0 + local cnt = 0 + for branch in utils.kpairs(indexes, utils.sort_branch) do + local repos = {} + for repo in utils.kpairs(indexes[branch], utils.sort_repo) do + local archs = {} + for arch in utils.kpairs(indexes[branch][repo], utils.sort_arch) do + if type(utils.allowed.archs[arch]) == "number" then + local uri = ("%s/%s/%s/%s/APKINDEX.tar.gz"):format(mirror, branch, repo, arch) + status, modified = get_index_status(uri) + table.insert(archs, {name=arch, status=status, modified=modified}) + if status == "200" then qty = qty+1 end + end + cnt = cnt + 1 + progress(num_indexes-cnt) + end + table.insert(repos, {name=repo, arch=archs}) + end + table.insert(branches, {name=branch, repo=repos}) + end + return branches, qty +end + +function process_mirrors() + local res = {} + local mirrors = get_mirrors(mirrors_yaml) + for idx,mirror in ipairs(mirrors) do + local start_time = os.time() + res[idx] = {} + res[idx].url = mirror + print(("[%s/%s] Getting indexes from mirror: %s"):format(idx, + #mirrors, mirror)) + res[idx].branch, res[idx].count = check_apkindexes(mirror) + res[idx].duration = os.difftime(os.time(),start_time) + end + return res +end + +function process_master() + print(("Getting indexes from master: %s"):format(master)) + local res = {} + res.url = master + res.branch = check_apkindexes(master) + return res +end + +write_json( + { + master = process_master(), + mirrors = process_mirrors(), + date = os.time(), + version = app_version + } +) diff --git a/index.tpl b/index.tpl new file mode 100644 index 0000000..ac381af --- /dev/null +++ b/index.tpl @@ -0,0 +1,123 @@ + + + + + + Mirror health + + + + + + + + +
+
+
+ +
+
+
+
+ {{#mirrors}} +
+

{{url}}

+
+
    +
  • Generated in {{duration}} seconds.
    • +
  • Found {{count}} indexes.
    • +
+
+ + + + {{#thead}} + + {{/thead}} + + + + {{#tbody}} + + {{#row}} + + {{/row}} + + {{/tbody}} + +
{{.}}
{{text}}
+
+ {{/mirrors}} +
+
Last updated: {{lupdate}} UTC
+
+ +
+ + \ No newline at end of file diff --git a/utils.lua b/utils.lua new file mode 100644 index 0000000..d53100a --- /dev/null +++ b/utils.lua @@ -0,0 +1,69 @@ +local M = {} + + +M.allowed = { + archs = { x86=1, x86_64=2, armhf=3, aarch64=4, ppc64le=5, s390x=6 }, + repos = { main=1, community=2, testing=3, backports=4 } +} + +function M.in_array(t, value) + for k,v in ipairs(t) do + if v == value then + return true + end + end +end + +function M.read_file(file) + local f = assert(io.open(file)) + local file = f:read("*all") + f:close() + return file +end + +function M.write_file(file, string) + file = assert(io.open(file, "w")) + file:write(string) + file:close() +end + +---- +-- table iterator which sorts on keys +function M.kpairs(t, f) + local keys = {} + for k in pairs(t) do keys[#keys + 1] = k end + table.sort(keys,f) + local i = 0 + return function() + i = i + 1 + return keys[i], t[keys[i]] + end +end + +---- +-- branch sort function for kpairs +function M.sort_branch(a,b) + if a == "edge" then a = "z" end + if b == "edge" then b = "z" end + if a < b then return true end +end + +---- +-- repo sort function for kpairs +function M.sort_repo(a,b) + local repos = M.allowed.repos + if type(repos[a]) == "number" and type(repos[b]) == "number" then + if repos[a] < repos[b] then return true end + end +end + +---- +-- arch sort function for kpairs +function M.sort_arch(a,b) + local archs = M.allowed.archs + if type(archs[a]) == "number" and type(archs[b]) == "number" then + if archs[a] < archs[b] then return true end + end +end + +return M \ No newline at end of file -- cgit v1.2.3