From 3829c9b7962f11e760361deb764fc96af1345c62 Mon Sep 17 00:00:00 2001
From: onefang
Date: Mon, 2 Dec 2019 06:28:30 +1000
Subject: Merge email and web report scripts.

---
 apt-panopticon-report-email-web.lua | 525 ++++++++++++++++++++++++++++++++++++
 1 file changed, 525 insertions(+)
 create mode 100755 apt-panopticon-report-email-web.lua

(limited to 'apt-panopticon-report-email-web.lua')

diff --git a/apt-panopticon-report-email-web.lua b/apt-panopticon-report-email-web.lua
new file mode 100755
index 0000000..60027d0
--- /dev/null
+++ b/apt-panopticon-report-email-web.lua
@@ -0,0 +1,525 @@
+#!/usr/bin/env luajit
+
+local args = {...}
+
+verbosity = -1
+local logFile
+local html = false
+
+
+--[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table.
+    From http://lua-users.org/wiki/SortedIteration
+  ]]
+function __genOrderedIndex( t )
+    local orderedIndex = {}
+    for key in pairs(t) do
+        table.insert( orderedIndex, key )
+    end
+    table.sort( orderedIndex )
+    return orderedIndex
+end
+function orderedNext(t, state)
+    -- Equivalent of the next function, but returns the keys in the alphabetic
+    -- order. We use a temporary ordered key table that is stored in the
+    -- table being iterated.
+
+    local key = nil
+    --print("orderedNext: state = "..tostring(state) )
+    if state == nil then
+        -- the first time, generate the index
+        t.__orderedIndex = __genOrderedIndex( t )
+        key = t.__orderedIndex[1]
+    else
+        -- fetch the next value
+        for i = 1,table.getn(t.__orderedIndex) do
+            if t.__orderedIndex[i] == state then
+                key = t.__orderedIndex[i+1]
+            end
+        end
+    end
+
+    if key then
+        return key, t[key]
+    end
+
+    -- no more value to return, cleanup
+    t.__orderedIndex = nil
+    return
+end
+function orderedPairs(t)
+    -- Equivalent of the pairs() function on tables. Allows to iterate
+    -- in order
+    return orderedNext, t, nil
+end
+
+-- Use this to dump a table to a string, with HTML.
+dumpTableHTML = function (table, space, name)
+    local r = name .. "\n"
+    r = r .. dumpTableHTMLSub(table, space .. " ")
+    r = r .. space .. ""
+    return r
+end
+dumpTableHTMLSub = function (table, space)
+    local r = ""
+    for k, v in orderedPairs(table) do 
+	if type(v) == "table" then
+	    if " " == space then
+		r = r ..           space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul>\n"
+	    else
+		r = r .. "<li>" .. space .. dumpTableHTML(v, space, k .. "<ul>") .. "</ul></li>\n"
+	    end
+	else
+	    r = r .. space .. "<li>" .. k .. "</li>\n"
+	end
+    end
+    return r
+end
+
+local plurals = function(e, w)
+    local result = ""
+    if 1 == e then
+	result = e .. " error"
+    elseif e ~= 0 then
+	result = e .. " errors"
+    end
+    if ("" ~= result) and html then result = "<font color='red'><b>" .. result .. "</b></font>" end
+--    result = " " .. result
+    if 0 < w then
+	if 0 < e then result = result .. ", " end
+	if 1 == w then 
+	    result = result .. w .. " warning"
+	else
+	    result = result .. w .. " warnings"
+	end
+	if ("" ~= result) and html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
+--	result = " " .. result
+    end
+    if "" ~= result then result = " (" .. result .. ")" end
+    return result
+end
+
+local results = {}
+
+local log = function(v, t, s, prot, test, host)
+    local x = ""
+    if nil == prot then prot = "" end
+    if nil ~= test then x = x .. test else test = "" end
+    if nil ~= host then
+	if #x > 0 then x = x .. " " end
+	x = x .. host
+    end
+    if #x > 0 then
+	t = t .. "(" .. x ..  ")"
+    end
+    if v <= verbosity then
+	if 3 <= verbosity then t = os.date() .. " " .. t end
+	print(t .. ": " .. s)
+    end
+    if nil ~= logFile then
+	logFile:write(os.date() .. " " .. t .. ": " .. s .. "\n")
+	logFile:flush()
+    end
+end
+local D = function(s) log(3,  "DEBUG   ", s) end
+local I = function(s) log(2,  "INFO    ", s) end
+local W = function(s, p, t, h) log(1,  "WARNING ", s, p, t, h) end
+local E = function(s, p, t, h) log(0,  "ERROR   ", s, p, t, h) end
+local C = function(s) log(-1, "CRITICAL", s) end
+
+local mirrors = loadfile("results/mirrors.lua")()
+
+local revDNS = function(dom, IP)
+    if "deb.devuan.org" ~= dom then
+	if nil ~= mirrors["deb.devuan.org"] then
+	    if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
+		if html then
+		    return "<font color='purple'><b>DNS-RR</b></font>"
+		else
+		    return "DNS-RR"
+		end
+	    end
+	end
+    else
+	for k, v in pairs(mirrors) do
+	    if "deb.devuan.org" ~= k then
+		local IPs = v.IPs
+		for i, u in pairs(IPs) do
+		    if "table" == type(u) then
+			for h, t in pairs(u) do
+			    if IP == h then return k end
+			end
+		    else
+			if IP == i then return k end
+		    end
+		end
+	    end
+	end
+    end
+    return ""
+end
+
+local faulty = ""
+local status = function(host, results, typ)
+    local result = ""
+    local e = 0
+    local w = 0
+    local s = nil ~= mirrors[host].Protocols[typ]
+    local to = false
+    if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end
+    if nil ~= results[typ] then
+	e = results[typ].errors
+	w = results[typ].warnings
+--[[
+	for k, v in pairs(results[typ]) do
+	    if "table" == type(v) then
+		if 0 <= v.errors   then e = e + v.errors   else to = true end
+		if 0 <= v.warnings then w = w + v.warnings else to = true end
+	    end
+	end
+]]
+    else
+	for k, v in pairs(results) do
+	    if "table" == type(v) then
+		for i, u in pairs(v) do
+		    if "table" == type(u) then
+			if typ == i then
+			    if 0 <= u.errors   then e = e + u.errors   else to = true end
+			    if 0 <= u.warnings then w = w + u.warnings else to = true end
+			end
+		    end
+		end
+	    end
+	end
+    end
+
+    if to then
+	result = "[TIMEOUT"
+	if not s then result = result .. "*" end
+	if html then
+	    if s then
+		result = "[<font color='blue'><b>TIMEOUT</b></font>"
+	    else
+		result = "[<font color='darkblue'><b>TIMEOUT*</b></font>"
+	    end
+	end
+    elseif 0 < e then
+	result = "[FAILED"
+	if not s then result = result .. "*" end
+	if html then
+	    if s then
+		result = "[<font color='red'><b>FAILED</b></font>"
+	    else
+		result = "[<font color='darkred'><b>FAILED*</b></font>"
+	    end
+	end
+	if html then
+	    faulty = faulty .. host .. " (" .. typ .. ")<br>\n"
+	else
+	    faulty = faulty .. host .. " (" .. typ .. ")\n"
+	end
+    else
+	result = "[OK"
+	if not s then result = result .. "*" end
+	if html then
+	    if s then
+		result = "[<font color='lime'><b>OK</b></font>"
+	    else
+		result = "[<font color='darkgreen'><b>OK*</b></font>"
+	    end
+	end
+    end
+    return result .. plurals(e, w) .. "]"
+end
+
+local collate = function(host, ip, results)
+    local f = "results/" .. host .. "_" .. ip .. ".lua"
+    local rfile, e = io.open(f, "r")
+    if nil == rfile then I("opening " .. f .. " file - " .. e) else
+	rfile:close()
+	local rs = loadfile(f)()
+	for k, v in pairs(rs) do
+	    if "table" == type(v) then
+		for i, u in pairs(v) do
+		    if "table" == type(u) then
+			for h, t in pairs(u) do
+			    local a = results[k][h]
+			    if nil == a then a = 0 end
+			    results[k][h] = a + t
+			end
+		    else
+			local a = results[k][i]
+			if nil == a then a = 0 end
+			results[k][i] = a + u
+		    end
+		end
+	    else
+		local a = results[k]
+		if nil == a then a = 0 end
+		results[k] = a + v
+	    end
+	end
+    end
+    return results
+end
+
+local m = {}
+
+local logCount = function(domain, ip)
+    local nm = "LOG_" .. domain
+    local log = ""
+    local extra = ""
+    if nil ~= ip then nm = nm .. "_" .. ip end
+    nm = nm .. ".html"
+    local rfile, e = io.open("results/" .. nm, "r")
+    if nil ~= rfile then
+	local errors = 0
+	local warnings = 0
+	for l in rfile:lines() do
+	    if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
+	    if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
+	end
+	rfile:close()
+	if html then
+	    if nil == ip then
+		log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
+	    else
+		log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
+	    end
+	end
+	log = log .. plurals(errors, warnings)
+    end
+    return log
+end
+
+
+html = false
+local email, e = io.open("results/Report-email.txt", "w+")
+if nil == email then C("opening mirrors file - " .. e) else
+    email:write( "Dear Mirror Admins,\n\n" ..
+		"This is the status of the mirror servers in the Devuan package mirror network.\n\n" ..
+		"The full list of Devuan package mirrors is available at the URL:\n\n" ..
+		" https://pkgmaster.devuan.org/mirror_list.txt\n\n" ..
+		'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' ..
+		"The full results of the mirror checking is available at the URL:\n\n" ..
+		" https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" ..
+		"Due to the nature of the tests, some errors or warnings will be \ncounted several times.  " ..
+		"Refer to the logs on the web page for details.\n\n" ..
+		"Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" ..
+		"==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" ..
+		"[skip] means that the test hasn't been written yet.\n\n")
+    for k, v in orderedPairs(mirrors) do
+	local results = loadfile("results/" .. k .. ".lua")()
+	email:write(k .. "....\n")
+	local IPs = v.IPs
+	for i, u in pairs(IPs) do
+	    if "table" == type(u) then
+		for h, t in pairs(u) do
+		    results = collate(k, h, results)
+		end
+	    else
+		results = collate(k, i, results)
+	    end
+	end
+	local ftp	= "[skip]"
+	local http	= status(k, results, "http")
+	local https	= status(k, results, "https")
+	local rsync	= "[skip]"
+	local dns	= ""
+	local protocol	= status(k, results, "Protocol")
+	local sanity	= "[skip]"
+	local integrity = status(k, results, "Integrity")
+	local updated	= status(k, results, "Updated")
+
+	-- DNS-RR test.
+	if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then
+	    for l, w in pairs(mirrors[k].IPs) do
+		if type(w) == "table" then
+		    for i, u in pairs(w) do
+			if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
+			    local log = logCount("deb.devuan.org", i)
+			    if "" ~= log then
+				if "" == dns then dns = " " else dns = dns .. "  " end
+				dns = dns .. logCount("deb.devuan.org", i)
+			    else
+				if "" == dns then dns = " " else dns = dns .. "  " end
+				dns = dns .. i
+			    end
+			end
+		    end
+		else
+		    if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
+			local log = logCount("deb.devuan.org", l)
+			if "" ~= log then
+			    if "" == dns then dns = " " else dns = dns .. "  " end
+			    dns = dns .. log
+			else
+			    if "" == dns then dns = " " else dns = dns .. "  " end
+			    dns = dns .. l
+			end
+		    end
+		end
+	    end
+	    if "" == dns then dns = "[no]" end
+	    dns = " DNS-RR: " .. dns
+	end
+
+	email:write(	"      ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" ..
+			"     " .. dns .. "\n" ..
+			"      Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" ..
+			"      Updated: " .. updated .. "\n")
+    end
+    email:write( "\n==== faulty mirrors: ====\n" .. faulty)
+    email:write( "\n-------------------------\n\n" ..
+		"* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" ..
+		"Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" ..
+		"You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ .\n\n" ..
+		"Love\n\n" ..
+		"The Dev1Devs\n\n")
+    email:close()
+end
+
+
+results = {}
+m = {}
+faulty = ""
+html = true
+local web, e = io.open("results/Report-web.html", "w+")
+if nil == web then C("opening mirrors file - " .. e) else
+    web:write( "<html><head><title>apt-panopticon results</title>\n" ..
+		'</head><body bgcolor="black" text="white">' ..
+		"<h1>Welcome to the apt-panopticon results page.</h1>\n" ..
+		"<p>This is the status of the mirror servers in the Devuan package mirror network.</p>\n" ..
+		"<p>The full list of Devuan package mirrors is available at the URL: " ..
+		"<a href='https://pkgmaster.devuan.org/mirror_list.txt'>https://pkgmaster.devuan.org/mirror_list.txt</a></p>\n" ..
+		"<p>Due to the nature of the tests, some errors or warnings will be counted several times. &nbsp; " ..
+		"The links in the table and DNS list go to the detailed testing logs.</p>\n\n" ..
+		"<hr>\n<h2>==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====</h2>\n<table>\n" ..
+
+		"<p>[<font color='red'><b>FAILED</b></font>] or [<font color='lime'><b>OK</b></font>]" ..
+		" means the tested thing is supported for that mirror.</p>\n" ..
+		"<p>[<font color='darkred'><b>FAILED*</b></font>] or [<font color='darkgreen'><b>OK*</b></font>]" ..
+		" means the tested thing is unsupported for that mirror, but might have been tested anyway.</p>\n" ..
+		"<p>[<font color='blue'><b>TIMEOUT</b></font>] or [<font color='darkblue'><b>TIMEOUT</b></font>]" ..
+		" means the server had too many timeouts, and tests where aborted, so there is no result for this test.</p>" ..
+		"<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [<font color='grey'><b>no</b></font>] if it isn't part of the DNS-RR. &nbsp; " ..
+		"The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " ..
+		"deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>\n" ..
+		"<p>The time in the Updated column is how often the mirror updates itself.</p>" ..
+		"<p>Mirrors with a <font style='background-color:dimgrey'>grey background</font> are not active (though may be usable as part of the DNS-RR).</p>\n" ..
+		"<p>[<font color='grey'><b>skip</b></font>] means that the test hasn't been written yet.</p>\n" ..
+		"<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th><th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th></tr>\n"
+		)
+    for k, v in orderedPairs(mirrors) do
+	local results = loadfile("results/" .. k .. ".lua")()
+	local active = ""
+	if "yes" == v.Active then
+	    web:write("     <tr><th>" .. k .. "</th> ")
+	else
+	    if nil == v.Active then active = 'nil' else active = v.Active end
+	    web:write("     <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
+	end
+	local IPs = v.IPs
+	for i, u in pairs(IPs) do
+	    if "table" == type(u) then
+		for h, t in pairs(u) do
+		    results = collate(k, h, results)
+		end
+	    else
+		results = collate(k, i, results)
+	    end
+	end
+	local ftp	= "[<font color='grey'><b>skip</b></font>]"
+	local http	= status(k, results, "http")
+	local https	= status(k, results, "https")
+	local rsync	= "[<font color='grey'><b>skip</b></font>]"
+	local dns	= ""
+	local protocol	= status(k, results, "Protocol")
+	local sanity	= "[<font color='grey'><b>skip</b></font>]"
+	local integrity = status(k, results, "Integrity")
+	local updated	= status(k, results, "Updated")
+	local rate = v.Rate
+	if nil ~= rate then updated = updated .. ' ' .. rate end
+
+	-- DNS-RR test.
+	if ("deb.devuan.org" ~= k) and (nil ~= mirrors["deb.devuan.org"]) then
+	    for l, w in pairs(mirrors[k].IPs) do
+		if type(w) == "table" then
+		    for i, u in pairs(w) do
+			if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
+			    local log = logCount("deb.devuan.org", i)
+			    if "" ~= log then
+				if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+				dns = dns .. logCount("deb.devuan.org", i)
+			    else
+				if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+				dns = dns .. "<font color='maroon'><b>" .. i .. "</b></font>"
+			    end
+			end
+		    end
+		else
+		    if nil ~= mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
+			local log = logCount("deb.devuan.org", l)
+			if "" ~= log then
+			    if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+			    dns = dns .. log
+			else
+			    if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
+			    dns = dns .. "<font color='maroon'><b>" .. l .. "</b></font>"
+			end
+		    end
+		end
+	    end
+	    if "" == dns then dns = "[<font color='grey'><b>no</b></font>]" end
+	end
+
+	web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
+		    "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity ..
+		    "&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. "</td></tr>\n")
+	if "" ~= v.Active then
+	    web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
+	end
+    end
+    web:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty)
+    web:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n")
+
+    for k, v in pairs(mirrors) do
+	local log = k
+	local n = {}
+	log = logCount(k)
+	mirrors[k].Protocols = nil
+	mirrors[k].FQDN = nil
+	mirrors[k].Active = nil
+	mirrors[k].Rate = nil
+	mirrors[k].BaseURL = nil
+	mirrors[k].Country = nil
+	mirrors[k].Bandwidth = nil
+	for l, w in pairs(mirrors[k].IPs) do
+	    if type(w) == "table" then
+		n[l] = {}
+		for i, u in pairs(w) do
+		    local log = logCount(k, i)
+		    if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end
+		end
+	    else
+		local log = logCount(k, l)
+		if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end
+	    end
+	end
+	m[log .. " DNS entries -"] = n
+    end
+    web:write(	"<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " ..
+		"The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. &nbsp; " ..
+		"If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. &nbsp; " ..
+		"deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR. &nbsp; " ..
+		"If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>' &nbsp; " ..
+		"pkgmaster.devuan.org is the master mirror, all the others sync to it. &nbsp; " ..
+		"</p>\n"
+		)
+    web:write(dumpTableHTML(m, "", ""))
+    web:write( "\n<br>\n<hr>\n\n" ..
+		"<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
+		"All <a href='../results'>the logs and other output</a>. &nbsp; " ..
+		"You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>" ..
+		"</body></html>\n")
+    web:close()
+end
-- 
cgit v1.1