Only read word lists once for speed improvement and to avoid exception if missing

1 files changed, 20 insertions, 12 deletions

diff --git a/weblog-model.lua b/weblog-model.lua
index 529ada9..179c633 100644
--- a/weblog-model.lua
+++ b/weblog-model.lua
@@ -19,8 +19,11 @@ local configfile = "/etc/weblog/weblog.conf"
 local configcontent = fs.read_file(configfile) or ""
 local config = format.parse_ini_file(configcontent, "") or {}
 local goodwordslist = "/etc/weblog/goodwords"
+local goodwords
 local badwordslist = "/etc/weblog/badwords"
+local badwords
 local ignorewordslist = "/etc/weblog/ignorewords"
+local ignorewords
 local files = {badwordslist, goodwordslist, ignorewordslist, configfile}
 
 local database_creation_script = {
@@ -530,7 +533,7 @@ local function parsesquidlog(line)
 	local ignoreme=false
 	
 	--check for ignored records first
-	for thisline in io.lines(ignorewordslist) do
+	for i,thisline in ipairs(ignorewords) do
 		if not thisline then
 			break
 		end
@@ -543,7 +546,7 @@ local function parsesquidlog(line)
 
 	if ignoreme ~= true then
 	 --proceed with record analysis
-	  for thisline in io.lines(badwordslist) do
+	  for i,thisline in ipairs(badwords) do
 		if not thisline then
 			break
 		end
@@ -573,7 +576,7 @@ local function parsesquidlog(line)
 		   isbypass=1
 		end
 	  end
-	  for goodline in io.lines(goodwordslist) do
+	  for i,goodline in ipairs(goodwords) do
 		if not goodline then
 		  break
 		end
@@ -642,7 +645,7 @@ local function parsesquarklog(line)
 	local ignoreme=false
 	
 	--check for ignored records first
-	for thisline in io.lines(ignorewordslist) do
+	for i,thisline in ipairs(ignorewords) do
 		if not thisline then
 			break
 		end
@@ -654,7 +657,7 @@ local function parsesquarklog(line)
 	end
 
  	if ignoreme ~= true then	
-	  for thisline in io.lines(badwordslist) do
+	  for i,thisline in ipairs(badwords) do
 		if not thisline then
 			-- logme("This line is apparently empty...")
 			break
@@ -682,7 +685,7 @@ local function parsesquarklog(line)
 		   isbypass=1
 		end
 	end
-	for goodline in io.lines(goodwordslist) do
+	for i,goodline in ipairs(goodwords) do
 		if not goodline then
 		  -- logme("This line is apparently empty...")
 		  break
@@ -746,7 +749,7 @@ local function parsedglog(line)
 	local ignoreme=false
 
 	--check for ignored records first
-	for thisline in io.lines(ignorewordslist) do
+	for i,thisline in ipairs(ignorewords) do
 		if not thisline then
 			break
 		end
@@ -758,7 +761,7 @@ local function parsedglog(line)
 	end
 
 	if ignoreme ~= true then
-	  for thisline in io.lines(badwordslist) do
+	  for i,thisline in ipairs(badwords) do
 		if not thisline then
 			-- logme("This line is apparently empty...")
 			break
@@ -787,7 +790,7 @@ local function parsedglog(line)
 		   isbypass=1
 		end
 	  end
-	  for goodline in io.lines(goodwordslist) do
+	  for i,goodline in ipairs(goodwords) do
 		if not goodline then
 		  -- logme("This line is apparently empty...")
 		  break
@@ -847,7 +850,7 @@ local function parsedumplog(line)
 	isbad=0
 	isdenied=0
 	isbypass=0
-	for thisline in io.lines(badwordslist) do
+	for i,thisline in ipairs(badwords) do
 		if not thisline then
 			 logme("This line is apparently empty...")
 			break
@@ -873,7 +876,7 @@ local function parsedumplog(line)
 		   isbypass=1
 		end
 	end
-	for goodline in io.lines(goodwordslist) do
+	for i,goodline in ipairs(goodwords) do
 		if not goodline then
 		  -- logme("This line is apparently empty...")
 		  break
@@ -1254,7 +1257,12 @@ function importlogs()
 		logme("Analyzing...")
 		local sql = "ANALYZE"
 		res = assert (con:execute(sql))
-		
+
+		-- Get the word lists
+		goodwords = fs.read_file_as_array(goodwordslist) or {}
+		badwords = fs.read_file_as_array(badwordslist) or {}
+		ignorewords = fs.read_file_as_array(ignorewordslist) or {}
+
 		-- Determine sources
 		local sources = listsourceentries(sourcename)