diff options
| -rw-r--r-- | weblog-model.lua | 155 |
1 files changed, 97 insertions, 58 deletions
diff --git a/weblog-model.lua b/weblog-model.lua index d98d92d..9d1c152 100644 --- a/weblog-model.lua +++ b/weblog-model.lua @@ -6,6 +6,8 @@ fs = require("acf.fs") format = require("acf.format") validator = require("acf.validator") require("luasql.postgres") +require("posix") +require("subprocess") local DatabaseName = "webproxylog" local DatabaseOwner = "weblogowner" @@ -68,10 +70,7 @@ end -- List the postgres databases on this system local listdatabases = function() local dbs = {} - local cmd = path.."psql -U postgres -tl 2>&1" - local f = io.popen(cmd) - local result = f:read("*a") or "" - f:close() + local result = modelfunctions.run_executable({"psql", "-U", "postgres", "-tl"}) for line in string.gmatch(result, "[^\n]+") do dbs[#dbs+1] = string.match(line, "^ (%S+)") end @@ -83,20 +82,21 @@ local createdatabase = function(password) local result = {} -- First, create the users - local cmd = path..'psql -U postgres -c "CREATE USER '..DatabaseOwner..' WITH PASSWORD \''..password..'\'" 2>&1' - local f = io.popen(cmd) - table.insert(result, f:read("*a")) - f:close() - cmd = path..'psql -U postgres -c "CREATE USER '..DatabaseUser..'" 2>&1' - f = io.popen(cmd) - table.insert(result, f:read("*a")) - f:close() + local cmd = "CREATE USER "..DatabaseOwner.." WITH PASSWORD '"..password.."'" + local cmdresult, errtxt = modelfunctions.run_executable({"psql", "-U", "postgres", "-c", cmd}, true) + table.insert(result, errtxt) + table.insert(result, cmdresult) + + cmd = "CREATE USER "..DatabaseUser + cmdresult, errtxt = modelfunctions.run_executable({"psql", "-U", "postgres", "-c", cmd}, true) + table.insert(result, errtxt) + table.insert(result, cmdresult) -- Create the database - cmd = path..'psql -U postgres -c "CREATE DATABASE '..DatabaseName..' WITH OWNER '..DatabaseOwner..'" 2>&1' - f = io.popen(cmd) - table.insert(result, f:read("*a")) - f:close() + cmd = "CREATE DATABASE "..DatabaseName.." WITH OWNER "..DatabaseOwner + cmdresult, errtxt = modelfunctions.run_executable({"psql", "-U", "postgres", "-c", cmd}, true) + table.insert(result, errtxt) + table.insert(result, cmdresult) return table.concat(result, "\n") end @@ -105,32 +105,28 @@ end local deletedatabase = function() local result = {} - local cmd = path..'psql -U postgres -c "DROP DATABASE '..DatabaseName..'" 2>&1' - local f = io.popen(cmd) - table.insert(result, f:read("*a")) - f:close() - cmd = path..'psql -U postgres -c "DROP ROLE '..DatabaseUser..'" 2>&1' - f = io.popen(cmd) - table.insert(result, f:read("*a")) - f:close() - cmd = path..'psql -U postgres -c "DROP ROLE '..DatabaseOwner..'" 2>&1' - f = io.popen(cmd) - table.insert(result, f:read("*a")) - f:close() + local cmd = "DROP DATABASE "..DatabaseName + local cmdresult, errtxt = modelfunctions.run_executable({"psql", "-U", "postgres", "-c", cmd}, true) + table.insert(result, errtxt) + table.insert(result, cmdresult) + + cmd = "DROP ROLE "..DatabaseUser + cmdresult, errtxt = modelfunctions.run_executable({"psql", "-U", "postgres", "-c", cmd}, true) + table.insert(result, errtxt) + table.insert(result, cmdresult) + + cmd = "DROP ROLE "..DatabaseOwner + cmdresult, errtxt = modelfunctions.run_executable({"psql", "-U", "postgres", "-c", cmd}, true) + table.insert(result, errtxt) + table.insert(result, cmdresult) return table.concat(result, "\n") end -- Run an SQL script local runSQLscript = function(filename) - -- Create the database - local cmd = path..'psql -U postgres -f "'..filename..'" '..DatabaseName..' 2>&1' - local f = io.popen(cmd) - local result = f:read("*a") or "" - f:close() - -- Create the tables - print (result) - return result + local result, errtxt = modelfunctions.run_executable({"psql", "-U", "postgres", "-f", filename, DatabaseName}, true) + return errtxt or result end -- Create the database and tables @@ -620,11 +616,8 @@ local connecttosource = function(source, cookiesfile) local success = false logme("Connecting to source "..source.sourcename) if source.method == "http" or source.method == "https" then - fs.write_file(cookiesfile, "password="..source.passwd.."&userid="..source.userid.."&Logon=Logon") - local cmd = "wget -O - --no-check-certificate --save-cookies "..cookiesfile.." --keep-session-cookies --post-file '"..cookiesfile.."' '"..source.method.."://"..format.escapespecialcharacters(source.source).."/cgi-bin/acf/acf-util/logon/logon' 2>/dev/null" - local f = io.popen(cmd) - local resultpage = f:read("*a") - f:close() + fs.write_file(cookiesfile, "password="..source.passwd.."&userid="..source.userid.."&Logon=Logon&submit=true") + local resultpage, errtxt = modelfunctions.run_executable({"wget", "-O", "-", "--no-check-certificate", "--save-cookies", cookiesfile, "--keep-session-cookies", "--post-file", cookiesfile, source.method.."://"..source.source.."/cgi-bin/acf/acf-util/logon/logon"}) if resultpage == "" then logme("Failed to connect to "..source.sourcename) elseif string.find(resultpage, "Log in") then @@ -641,11 +634,8 @@ end local getlogcandidates = function(source, cookiesfile) local candidates = {} if source.method == "http" or source.method == "https" then - local cmd = "wget -O - --no-check-certificate --load-cookies "..cookiesfile.." '"..source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/status' 2>/dev/null" - local f = io.popen(cmd) - local resultpage = f:read("*a") - f:close() - for file in string.gmatch(resultpage, "download%?name=([^\"]+)") do + local resultpage = modelfunctions.run_executable({"wget", "-O", "-", "--no-check-certificate", "--load-cookies", cookiesfile, source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/status"}) + for file in string.gmatch(resultpage, "download%?[^\"]*name=([^\"]+)") do candidates[#candidates+1] = file end elseif source.method == "local" then @@ -655,29 +645,71 @@ local getlogcandidates = function(source, cookiesfile) end local openlogfile = function(source, cookiesfile, logfile) - local handle + local handle, handle2, errtxt if source.method == "http" or source.method == "https" then - local cmd = "wget -O - --no-check-certificate --load-cookies "..cookiesfile.." --post-data 'name="..logfile.."' '"..source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/download' 2>/dev/null" - if string.find(logfile, "%.gz$") then - cmd = cmd.." | gunzip -c" + local cmd = {"wget", "-O", "-", "--no-check-certificate", "--load-cookies", cookiesfile, "--post-data", "submit=true&viewtype=stream&name="..logfile.."&filename="..logfile, source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/download"} + cmd.stdin = "/dev/null" + cmd.stdout = subprocess.PIPE + cmd.stderr = "/dev/null" + local res, err = pcall(function() + -- For security, set the path + posix.setenv("PATH", "/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin") + local proc, errmsg, errno = subprocess.popen(cmd) + if proc then + if string.find(logfile, "%.gz$") then + local cmd2 = {"gunzip", "-c"} + -- Pipe the output of wget into gunzip + cmd2.stdin = proc.stdout + cmd2.stdout = subprocess.PIPE + cmd2.stderr = "/dev/null" + local proc2, errmsg2, errno2 = subprocess.popen(cmd2) + if proc2 then + handle = proc2.stdout + handle2 = proc.stdout + else + proc.stdout:close() + errtxt = errmsg2 or "Unknown failure" + end + else + handle = proc.stdout + end + + else + errtxt = errmsg or "Unknown failure" + end + end) + if not res or err then + errtxt = err or "Unknown failure" end - handle = io.popen(cmd) elseif source.method == "local" then if string.find(logfile, "%.gz$") then - local cmd = "gunzip -c "..logfile - handle = io.popen(cmd) + local res, err = pcall(function() + -- For security, set the path + posix.setenv("PATH", "/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin") + local cmd = {"gunzip", "-c", logfile} + cmd.stdin = "/dev/null" + cmd.stdout = subprocess.PIPE + cmd.stderr = "/dev/null" + local proc, errmsg, errno = subprocess.popen(cmd) + if proc then + handle = proc.stdout + else + errtxt = errmsg or "Unknown failure" + end + end) + if not res or err then + errtxt = err or "Unknown failure" + end else handle = io.open(logfile) end end - return handle + return handle, handle2 end local deletelogfile = function(source, cookiesfile, logfile) if source.method == "http" or source.method == "https" then - local cmd = "wget -O - --no-check-certificate --load-cookies "..cookiesfile.." --post-data 'name="..logfile.."' '"..source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/delete' 2>/dev/null" - local f = io.popen(cmd) - f:close() + modelfunctions.run_executable({"wget", "-O", "-", "--no-check-certificate", "--load-cookies", cookiesfile, "--post-data", "submit=true&name="..logfile.."&filename="..logfile, source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/delete"}) elseif source.method == "local" then os.remove(logfile) end @@ -886,7 +918,11 @@ end -- import a logfile and delete logfile after local function importlogfile(source, cookiesfile, file, parselog_func) logme("Getting " .. file ) - local loghandle = openlogfile(source, cookiesfile, file) + local loghandle, loghandle2 = openlogfile(source, cookiesfile, file) + if not loghandle then + logme("Failed to get " .. file ) + return + end logme("Processing " .. file ) local res, err = pcall(function() con:execute("START TRANSACTION") @@ -925,6 +961,9 @@ local function importlogfile(source, cookiesfile, file, parselog_func) end end loghandle:close() + if loghandle2 then + loghandle2:close() + end if res then logme("Deleting " .. file ) deletelogfile(source, cookiesfile, file) |