Fixed http/https download typo, simplified importing code.

Changed parse and import functions to work on one line at a time, removed iterator functions.

1 files changed, 28 insertions, 53 deletions

diff --git a/weblog-model.lua b/weblog-model.lua
index e28464b..26cdf92 100644
--- a/weblog-model.lua
+++ b/weblog-model.lua
@@ -178,26 +178,22 @@ local listhistorylogentries = function()
 	return entries
 end
 
-local importsquidlog = function(logentries, sourcename)
-	con:execute("START TRANSACTION")
-	for entry in logentries do
+local importsquidlog = function(entry, sourcename)
+	if entry then
 		local sql = string.format("INSERT INTO weblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s')",
 			escape(sourcename), escape(entry.clientip), escape(entry.clientuserid:lower()),
 			escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes))
 		local res = assert (con:execute(sql))
 	end
-	con:execute("COMMIT")
 end
 
-local importdglog = function(logentries, sourcename)
-	con:execute("START TRANSACTION")
-	for entry in logentries do
+local importdglog = function(entry, sourcename)
+	if entry then
 		local sql = string.format("INSERT INTO blocklog VALUES ('%s', '0.0.0.0', '%s', '%s', '%s', '%s', '%s', '%s', '%s')",
 			escape(sourcename), escape(entry.clientuserid:lower()), escape(entry.logdatetime), escape(entry.URL),
 			escape(entry.bytes), escape(entry.reason), escape(entry.score or "0"), escape(entry.shortreason))
 		local res = assert (con:execute(sql))
 	end
-	con:execute("COMMIT")
 end
 
 local listsourceentries = function(sourcename)
@@ -552,7 +548,7 @@ end
 -- ################################################################################
 -- LOG FILE FUNCTIONS
 
-local function parsesquidlog_line(line)
+local function parsesquidlog(line)
 	-- Format of squid log (space separated):
 	-- time elapsed remotehost code/status bytes method URL rfc931 peerstatus/peerhost
 	local words = {}
@@ -572,51 +568,27 @@ local function parsesquidlog_line(line)
 		peerhost=string.match(words[9], "[^/]*$")}
 
 	logentry.logdatetime = os.date("%Y-%m-%d %H:%M:%S", logentry.logdatetime)..string.match(logentry.logdatetime, "%..*")
-	return logentry
-end
-
-local function parsesquidlog_iter(f)
-	return function()
-		while true do
-			line = f:read("*line")
-			if line == nil then
-				return nil
-			end
-			local logentry = parsesquidlog_line(line)
-			-- Don't care about local requests (from DG)
-			if logentry.clientip ~= "127.0.0.1" then
-				return logentry
-			end
-		end
+	-- Don't care about local requests (from DG)
+	if logentry.clientip ~= "127.0.0.1" then
+		return logentry
 	end
+	return nil
 end
 
-local function parsedglog_line(line)
+local function parsedglog(line)
 	local words = format.string_to_table(line, "\t")
-	return { logdatetime=words[1], clientuserid=words[2], clientip=words[3],
+	local logentry = { logdatetime=words[1], clientuserid=words[2], clientip=words[3],
 		URL=words[4], reason=words[5], method=words[6], bytes=words[7],
 		shortreason=words[9]}
-end
-
-local function parsedglog_iter(f)
-	return function()
-		while true do
-			line = f:read("*line")
-			if line == nil then
-				return nil
-			end
-			local logentry = parsedglog_line(line)
-
-			if logentry.reason ~= "" then
-				if logentry.shortreason == "" then
-					logentry.shortreason = logentry.reason
-				end
-				logentry.score = string.match(logentry.reason, "^.*: ([0-9]+) ")
-				logentry.logdatetime = string.gsub(logentry.logdatetime, "%.", "-")
-				return logentry
-			end
+	if logentry.reason ~= "" then
+		if logentry.shortreason == "" then
+			logentry.shortreason = logentry.reason
 		end
+		logentry.score = string.match(logentry.reason, "^.*: ([0-9]+) ")
+		logentry.logdatetime = string.gsub(logentry.logdatetime, "%.", "-")
+		return logentry
 	end
+	return nil
 end
 
 -- ################################################################################
@@ -878,13 +850,16 @@ end
 
 -- import either squid or dg log file.
 -- delete logfile after
-function importlogfile(source, cookiesfile, file, parselog_iter, importlog_func)
-	local logentries
+local function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
 	logme("Processing " .. file )
 	logme("Getting " .. file )
-	loghandle = openlogfile(source, cookiesfile, file)
-	logentries = parselog_iter(loghandle)
-	importlog_func(logentries, source.sourcename)
+	local loghandle = openlogfile(source, cookiesfile, file)
+	con:execute("START TRANSACTION")
+	for line in loghandle:lines() do
+		local logentry = parselog_func(line)
+		importlog_func(logentry, source.sourcename)
+	end
+	con:execute("COMMIT")
 	loghandle:close()
 	logme("Deleting " .. file )
 	deletelogfile(source, cookiesfile, file)
@@ -909,10 +884,10 @@ function importlogs()
 					for j,file in ipairs(files) do
 						if string.match(file, "dansguardian/access%.log[%.%-]") then
 							count = count + 1
-							importlogfile(source, cookeisfile, file, parsedglog_iter, importdglog)
+							importlogfile(source, cookiesfile, file, parsedglog, importdglog)
 						elseif string.match(file, "squid/access%.log[%.%-]") then
 							count = count + 1
-							importlogfile(source, cookeisfile, file, parsesquidlog_iter, importsquidlog)
+							importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog)
 						end
 					end
 				end