save ram by parse logfiles line by line

rather than read everything into a big buffer, we open and parse the log
using a file handle. This should reduce the memory usage dramatically.

1 files changed, 14 insertions, 17 deletions

diff --git a/weblog-model.lua b/weblog-model.lua
index aaa8a30..4a2e16a 100644
--- a/weblog-model.lua
+++ b/weblog-model.lua
@@ -552,9 +552,9 @@ end
 -- ################################################################################
 -- LOG FILE FUNCTIONS
 
-local parsesquidlog = function(logdata)
+local parsesquidlog = function(f)
 	local logentries = {}
-	for line in string.gmatch(logdata, "[^\n]+") do
+	for line in f:lines() do
 		-- Format of squid log (space separated):
 		-- time elapsed remotehost code/status bytes method URL rfc931 peerstatus/peerhost
 		local words = {}
@@ -571,9 +571,9 @@ local parsesquidlog = function(logdata)
 	return logentries
 end
 
-local parsedglog = function(logdata)
+local parsedglog = function(f)
 	local logentries = {}
-	for line in string.gmatch(logdata, "[^\n]+") do
+	for line in f:lines() do
 		local words = format.string_to_table(line, "\t")
 		local logentry = {logdatetime=words[1], clientuserid=words[2], clientip=words[3], URL=words[4], reason=words[5], method=words[6], bytes=words[7], shortreason=words[9]}
 		if logentry.reason ~= "" then
@@ -630,27 +630,23 @@ local getlogcandidates = function(source, cookiesfile)
 	return candidates
 end
 
-local getlogfile = function(source, cookiesfile, logfile)
-	local filecontent
+local openlogfile = function(source, cookiesfile, logfile)
+	local handle
 	if source.method == "http" or source.method == "https" then
 		local cmd = "wget -O - --no-check-certificate --load-cookies "..cookiesfile.." --post-data 'name="..logfile.."' '"..source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/download' 2>/dev/null"
 		if string.find(logfile, "%.gz$") then
 			cmd = cmd.." | gunzip -c"
 		end
-		local f = io.popen(cmd)
-		filecontent = f:read("*a")
-		f:close()
+		handle = io.popen(cmd)
 	elseif source.method == "local" then
 		if string.find(logfile, "%.gz$") then
 			local cmd = "gunzip -c "..logfile
-			local f = io.popen(cmd)
-			filecontent = f:read("*a")
-			f:close()
+			handle = io.popen(cmd)
 		else
-			filecontent = fs.read_file(logfile)
+			handle = io.open(logfile)
 		end
 	end
-	return filecontent
+	return handle
 end
 
 local deletelogfile = function(source, cookiesfile, logfile)
@@ -853,9 +849,10 @@ end
 function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
 	logme("Processing " .. file )
 	logme("Getting " .. file )
-	logcontent = getlogfile(source, cookiesfile, file)
-	logentries = parselog_func(logcontent)
-	importlog(logentries, source.sourcename)
+	loghandle = openlogfile(source, cookiesfile, file)
+	logentries = parselog_func(loghandle)
+	importlog_func(logentries, source.sourcename)
+	loghandle:close()
 	logme("Deleting " .. file )
 	deletelogfile(source, cookiesfile, file)
 end