Modified import to better handle errors.

Better error reporting for the line that caused the error.
Continue to do the purging even if import fails.

1 files changed, 35 insertions, 15 deletions

diff --git a/weblog-model.lua b/weblog-model.lua
index d2dbc56..de837a5 100644
--- a/weblog-model.lua
+++ b/weblog-model.lua
@@ -855,23 +855,40 @@ end
 -- import either squid or dg log file.
 -- delete logfile after
 local function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
-	logme("Processing " .. file )
+	local tmp
 	logme("Getting " .. file )
 	local loghandle = openlogfile(source, cookiesfile, file)
-	con:execute("START TRANSACTION")
-	for line in loghandle:lines() do
-		local logentry = parselog_func(line)
-		importlog_func(logentry, source.sourcename)
+	logme("Processing " .. file )
+	local res, err = pcall(function()
+		con:execute("START TRANSACTION")
+		for line in loghandle:lines() do
+			tmp = line
+			local logentry = parselog_func(line)
+			importlog_func(logentry, source.sourcename)
+		end
+		con:execute("COMMIT")
+	end)
+	if not res then
+		pcall(function() con:execute("ROLLBACK") end)
+		if tmp then
+			pcall(function() logme("Exception on line:"..tmp) end)
+		end
+		if err then
+			pcall(function() logme(err) end)
+		end
 	end
-	con:execute("COMMIT")
 	loghandle:close()
-	logme("Deleting " .. file )
-	deletelogfile(source, cookiesfile, file)
+	if res then
+		logme("Deleting " .. file )
+		deletelogfile(source, cookiesfile, file)
+	end
+	return res
 end
 
 function importlogs()
 	local result = cfe({ label="Weblog Import Logs Result" })
 	local count = 0
+	local success = true
 
 	local res, err = pcall(function()
 		databaseconnect(DatabaseOwner, config.password)
@@ -888,10 +905,10 @@ function importlogs()
 					for j,file in ipairs(files) do
 						if string.match(file, "dansguardian/access%.log[%.%-]") then
 							count = count + 1
-							importlogfile(source, cookiesfile, file, parsedglog, importdglog)
+							success = importlogfile(source, cookiesfile, file, parsedglog, importdglog) and success
 						elseif string.match(file, "squid/access%.log[%.%-]") then
 							count = count + 1
-							importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog)
+							success = importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog) and success
 						end
 					end
 				end
@@ -900,23 +917,26 @@ function importlogs()
 		end
 
 		-- Process the logs
-		addtowatchlist()
-		updateusagestats()
-		importpubweblog()
+		if success then
+			addtowatchlist()
+			updateusagestats()
+			importpubweblog()
+			groomwatchlist()
+		end
 		-- Purge old database entries
-		groomwatchlist()
 		groomusagestat()
 		groomdbhistlog()
 		groompublogs()
 
 		databasedisconnect()
 	end)
-	if not res then
+	if not res or not success then
 		result.errtxt = "Import Logs Failure"
 		if err then
 			pcall(function() logme(err) end)
 			result.errtxt = result.errtxt .. "\n" .. err
 		end
+		pcall(function() databasedisconnect() end)
 	end
 
 	result.value = "Imported "..tostring(count).." logs"