Added option to stop commit on error and changed default behavior to log error and continue

1 files changed, 23 insertions, 7 deletions

diff --git a/weblog-model.lua b/weblog-model.lua
index 78b2ea0..a69f417 100644
--- a/weblog-model.lua
+++ b/weblog-model.lua
@@ -275,6 +275,8 @@ end
 -- Process weblog and blocklog, combine into pubweblog and pubblocklog
 -- empties weblog and blocklog
 local importpubweblog = function()
+	local sql = "ANALYZE"
+	res = assert (con:execute(sql))
 	-- Merge equal blocks into weblog
 	sql = "update weblog set reason=blocklog.reason, " ..
 		"score=blocklog.score, shortreason=blocklog.shortreason from blocklog where " ..
@@ -858,24 +860,36 @@ end
 -- import either squid or dg log file.
 -- delete logfile after
 local function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
-	local tmp
 	logme("Getting " .. file )
 	local loghandle = openlogfile(source, cookiesfile, file)
 	logme("Processing " .. file )
 	local res, err = pcall(function()
 		con:execute("START TRANSACTION")
 		for line in loghandle:lines() do
-			tmp = line
-			local logentry = parselog_func(line)
-			importlog_func(logentry, source.sourcename)
+			assert(con:execute("SAVEPOINT before_line"))
+			local res2, err2 = pcall(function()
+				local logentry = parselog_func(line)
+				importlog_func(logentry, source.sourcename)
+			end)
+			if not res2 then
+				if (config.stoponerror == "true") then
+					pcall(function() con:execute("ROLLBACK") end)
+				else
+					assert(con:execute("ROLLBACK TO before_line"))
+				end
+				pcall(function() logme("Exception on line:"..line) end)
+				if err2 then
+					pcall(function() logme(err2) end)
+				end
+				if (config.stoponerror == "true") then
+					assert(res2, "Import halted on exception")
+				end
+			end
 		end
 		con:execute("COMMIT")
 	end)
 	if not res then
 		pcall(function() con:execute("ROLLBACK") end)
-		if tmp then
-			pcall(function() logme("Exception on line:"..tmp) end)
-		end
 		if err then
 			pcall(function() logme(err) end)
 		end
@@ -1199,6 +1213,7 @@ function getconfig()
 	result.groupby = cfe({ type="select", value=config.groupby or "clientuserid", label="Group results by", option={"clientuserid", "clientip"} })
 	result.shorturi = cfe({ type="boolean", value=(config.shorturi == "true"), label="Truncate URLs", descr="You can limit the length of displayed URLs by enabling this option"})
 	result.shortreason = cfe({ type="boolean", value=(config.shortreason == "true"), label="Short Reason", descr="Display a short reason (without objectional words)"})
+	result.stoponerror = cfe({ type="boolean", value=(config.stoponerror == "true"), label="Stop on Error", descr="Stop import of logs if an error is encountered"})
 	return cfe({ type="group", value=result, label="Weblog Config" })
 end
 
@@ -1256,6 +1271,7 @@ function updateconfig(newconfig)
 		configcontent = format.update_ini_file(configcontent, "", "groupby", newconfig.value.groupby.value)
 		configcontent = format.update_ini_file(configcontent, "", "shorturi", tostring(newconfig.value.shorturi.value))
 		configcontent = format.update_ini_file(configcontent, "", "shortreason", tostring(newconfig.value.shortreason.value))
+		configcontent = format.update_ini_file(configcontent, "", "stoponerror", tostring(newconfig.value.stoponerror.value))
 
 		fs.write_file(configfile, configcontent)
 		config = format.parse_ini_file(configcontent, "") or {}