Separated out testing logic into separate modules to make it easier to use.

Test/script/regression.py

@@ -26,6 +26,7 @@ import urllib2
 import re
 import zipfile
 import zlib
+import Emailer
 
 #
 # Please read me...
@@ -69,6 +70,7 @@ class Args:
 		self.exception_string = ""
 		self.contin = False
 		self.gold_creation = False
+		self.daily = False
 	
 	def parse(self):
 		global nxtproc 
@@ -134,6 +136,9 @@ class Args:
 			elif arg == "-g" or arg == "--gold":
 				printout("Creating gold standards")
 				self.gold_creation = True
+			elif arg == "-d" or arg == "--daily":
+				printout("Running daily")
+				self.daily = True
 			else:
 				printout(usage())
 				return False
@@ -148,9 +153,9 @@ class Args:
 class TestAutopsy:
 	def __init__(self):
 		# Paths:
-		self.input_dir = make_local_path("..","input")
+		self.input_dir = Emailer.make_local_path("..","input")
 		self.output_dir = ""
-		self.gold = make_local_path("..", "output", "gold", "tmp")
+		self.gold = Emailer.make_local_path("..", "output", "gold", "tmp")
 		# Logs:
 		self.antlog_dir = ""
 		self.common_log = ""
@@ -307,7 +312,7 @@ class Database:
 		
 	def generate_autopsy_artifacts(self):
 		if not self.autopsy_artifacts:
-			autopsy_db_file = make_path(case.output_dir, case.image_name,
+			autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name,
 										  "AutopsyTestCase", "autopsy.db")
 			autopsy_con = sqlite3.connect(autopsy_db_file)
 			autopsy_cur = autopsy_con.cursor()
@@ -325,7 +330,7 @@ class Database:
 				
 	def generate_autopsy_attributes(self):
 		if self.autopsy_attributes == 0:
-			autopsy_db_file = make_path(case.output_dir, case.image_name,
+			autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name,
 										  "AutopsyTestCase", "autopsy.db")
 			autopsy_con = sqlite3.connect(autopsy_db_file)
 			autopsy_cur = autopsy_con.cursor()
@@ -335,7 +340,7 @@ class Database:
 
 	def generate_autopsy_objects(self):
 		if self.autopsy_objects == 0:
-			autopsy_db_file = make_path(case.output_dir, case.image_name,
+			autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name,
 										  "AutopsyTestCase", "autopsy.db")
 			autopsy_con = sqlite3.connect(autopsy_db_file)
 			autopsy_cur = autopsy_con.cursor()
@@ -345,7 +350,7 @@ class Database:
 		
 	def generate_gold_artifacts(self):
 		if not self.gold_artifacts:
-			gold_db_file = make_path(case.gold, case.image_name, "autopsy.db")
+			gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
 			gold_con = sqlite3.connect(gold_db_file)
 			gold_cur = gold_con.cursor()
 			gold_cur.execute("SELECT COUNT(*) FROM blackboard_artifact_types")
@@ -361,7 +366,7 @@ class Database:
 				
 	def generate_gold_attributes(self):
 		if self.gold_attributes == 0:
-			gold_db_file = make_path(case.gold, case.image_name, "autopsy.db")
+			gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
 			gold_con = sqlite3.connect(gold_db_file)
 			gold_cur = gold_con.cursor()
 			gold_cur.execute("SELECT COUNT(*) FROM blackboard_attributes")
@@ -369,7 +374,7 @@ class Database:
 
 	def generate_gold_objects(self):
 		if self.gold_objects == 0:
-			gold_db_file = make_path(case.gold, case.image_name, "autopsy.db")
+			gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
 			gold_con = sqlite3.connect(gold_db_file)
 			gold_cur = gold_con.cursor()
 			gold_cur.execute("SELECT COUNT(*) FROM tsk_objects")
@@ -394,7 +399,7 @@ def run_config_test(config_file):
 			case.input_dir = parsed.getElementsByTagName("indir")[0].getAttribute("value").encode().decode("utf_8")
 		if parsed.getElementsByTagName("global_csv"):
 			case.global_csv = parsed.getElementsByTagName("global_csv")[0].getAttribute("value").encode().decode("utf_8")
-			case.global_csv = make_local_path(case.global_csv)
+			case.global_csv = Emailer.make_local_path(case.global_csv)
 		
 		# Generate the top navbar of the HTML for easy access to all images
 		values = []
@@ -403,7 +408,7 @@ def run_config_test(config_file):
 			if file_exists(value):
 				values.append(value)
 		count = len(values)
-		archives = make_path(case.gold, "..")
+		archives = Emailer.make_path(case.gold, "..")
 		arcount = 0
 		for file in os.listdir(archives):
 			if not(file == 'tmp'):
@@ -425,7 +430,6 @@ def run_config_test(config_file):
 			#Begin infiniloop
 			if(newDay()):
 				global daycount
-				compile()
 				if(daycount > 0):
 					print("starting process")
 					outputer = open("ScriptLog.txt", "a")
@@ -449,46 +453,6 @@ def run_config_test(config_file):
 		printerror(str(e) + "\n")
 		logging.critical(traceback.format_exc())
 
-def compile():
-	global redo
-	global tryredo
-	global daycount
-	global nxtproc
-	global failedbool
-	global errorem
-	global attachl
-	global passed
-	passed = True
-	tryredo = False
-	setDay()
-	redo = True
-	while(redo):
-		passed = True
-		if(passed):
-			gitPull("sleuthkit")
-		if(passed):
-			vsBuild()
-		if(passed):
-			gitPull("autopsy")
-		if(passed):
-			antBuild("datamodel", False)
-		if(passed):
-			antBuild("autopsy", True)
-		if(passed):
-			redo = False
-		else:
-			print("Compile Failed")
-			time.sleep(3600)
-	attachl = []
-	errorem = "The test standard didn't match the gold standard.\n"
-	failedbool = False
-	if(tryredo):
-		errorem += "Rebuilt properly.\n"
-		send_email()
-		attachl = []
-		errorem = "The test standard didn't match the gold standard.\n"	
-		passed = True
-
 # Runs the test on the single given file.
 # The path must be guarenteed to be a correct path.
 def run_test(image_file, count):
@@ -502,24 +466,24 @@ def run_test(image_file, count):
 	case.image_file = image_file
 	case.image_name = case.get_image_name(image_file) + "(" + str(count) + ")"
 	case.image = case.get_image_name(image_file)
-	case.common_log_path = make_local_path(case.output_dir, case.image_name, case.image_name+case.common_log)
+	case.common_log_path = Emailer.make_local_path(case.output_dir, case.image_name, case.image_name+case.common_log)
-	case.warning_log = make_local_path(case.output_dir, case.image_name, "AutopsyLogs.txt")
+	case.warning_log = Emailer.make_local_path(case.output_dir, case.image_name, "AutopsyLogs.txt")
-	case.antlog_dir = make_local_path(case.output_dir, case.image_name, "antlog.txt")
+	case.antlog_dir = Emailer.make_local_path(case.output_dir, case.image_name, "antlog.txt")
 	if(args.list):
 		element = parsed.getElementsByTagName("build")
 		if(len(element)<=0):
-			toval = make_path("..", "build.xml")
+			toval = Emailer.make_path("..", "build.xml")
 		else:
 			element = element[0]
 			toval = element.getAttribute("value").encode().decode("utf_8")
 			if(toval==None):
-				toval = make_path("..", "build.xml")
+				toval = Emailer.make_path("..", "build.xml")
 	else:
-		toval = make_path("..", "build.xml")
+		toval = Emailer.make_path("..", "build.xml")
 	case.build_path = toval	
-	case.known_bad_path = make_path(case.input_dir, "notablehashes.txt-md5.idx")
+	case.known_bad_path = Emailer.make_path(case.input_dir, "notablehashes.txt-md5.idx")
-	case.keyword_path = make_path(case.input_dir, "notablekeywords.xml")
+	case.keyword_path = Emailer.make_path(case.input_dir, "notablekeywords.xml")
-	case.nsrl_path = make_path(case.input_dir, "nsrl.txt-md5.idx")
+	case.nsrl_path = Emailer.make_path(case.input_dir, "nsrl.txt-md5.idx")
 	
 	logging.debug("--------------------")
 	logging.debug(case.image_name)
@@ -538,7 +502,7 @@ def run_test(image_file, count):
 		logging.critical(traceback.format_exc())
 	# If NOT keeping Solr index (-k)
 	if not args.keep:
-		solr_index = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "KeywordSearch")
+		solr_index = Emailer.make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "KeywordSearch")
 		if clear_dir(solr_index):
 			print_report([], "DELETE SOLR INDEX", "Solr index deleted.")
 	elif args.keep:
@@ -558,8 +522,8 @@ def run_test(image_file, count):
 	if not args.gold_creation:
 		try:
 			gold_path = case.gold
-			img_gold = make_path(case.gold, case.image_name)
+			img_gold = Emailer.make_path(case.gold, case.image_name)
-			img_archive = make_local_path("..", "output", "gold", case.image_name+"-archive.zip")
+			img_archive = Emailer.make_local_path("..", "output", "gold", case.image_name+"-archive.zip")
 			extrctr = zipfile.ZipFile(img_archive, 'r', compression=zipfile.ZIP_DEFLATED)
 			extrctr.extractall(gold_path)
 			extrctr.close
@@ -580,7 +544,7 @@ def run_test(image_file, count):
 	if args.rebuild or args.gold_creation:
 		rebuild()
 	# Reset the case and return the tests sucessfully finished
-	clear_dir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "ModuleOutput", "keywordsearch"))
+	clear_dir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "ModuleOutput", "keywordsearch"))
 	case.reset()
 	return True
 
@@ -604,8 +568,8 @@ def run_ant():
 	case.ant.append("-Dknown_bad_path=" + case.known_bad_path)
 	case.ant.append("-Dkeyword_path=" + case.keyword_path)
 	case.ant.append("-Dnsrl_path=" + case.nsrl_path)
-	case.ant.append("-Dgold_path=" + make_path(case.gold))
+	case.ant.append("-Dgold_path=" + Emailer.make_path(case.gold))
-	case.ant.append("-Dout_path=" + make_local_path(case.output_dir, case.image_name))
+	case.ant.append("-Dout_path=" + Emailer.make_local_path(case.output_dir, case.image_name))
 	case.ant.append("-Dignore_unalloc=" + "%s" % args.unallocated)
 	case.ant.append("-Dcontin_mode=" + str(args.contin))
 	case.ant.append("-Dtest.timeout=" + str(case.timeout))
@@ -613,7 +577,7 @@ def run_ant():
 	printout("Ingesting Image:\n" + case.image_file + "\n")
 	printout("CMD: " + " ".join(case.ant))
 	printout("Starting test...\n")
-	antoutpth = make_local_path(case.output_dir, "antRunOutput.txt")
+	antoutpth = Emailer.make_local_path(case.output_dir, "antRunOutput.txt")
 	antout = open(antoutpth, "a")
 	if SYS is OS.CYGWIN:
 		subprocess.call(case.ant, stdout=antout)
@@ -652,32 +616,32 @@ def rebuild():
 	# Errors to print
 	errors = []
 	# Delete the current gold standards
-	gold_dir = make_path(case.gold, case.image_name)
+	gold_dir = Emailer.make_path(case.gold, case.image_name)
 	print("here")
 	clear_dir(gold_dir)
 	print("here1")
-	dbinpth = make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db")
+	dbinpth = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db")
-	dboutpth = make_path(case.gold, case.image_name, "autopsy.db")
+	dboutpth = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
 	if not os.path.exists(case.gold):
 		os.makedirs(case.gold)
 	if not os.path.exists(gold_dir):
 		os.makedirs(gold_dir)
 	copy_file(dbinpth, dboutpth)
-	error_pth = make_path(case.gold, case.image_name, case.image_name+"SortedErrors.txt")
+	error_pth = Emailer.make_path(case.gold, case.image_name, case.image_name+"SortedErrors.txt")
 	copy_file(case.sorted_log, error_pth)
 	# Rebuild the HTML report
 	htmlfolder = ""
 	for fs in os.listdir(os.path.join(os.getcwd(),case.output_dir, case.image_name, "AutopsyTestCase", "Reports")):
 		if os.path.isdir(os.path.join(os.getcwd(), case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)):
 			htmlfolder = fs
-	autopsy_html_path = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)
+	autopsy_html_path = Emailer.make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)
 	
-	html_path = make_path(case.output_dir, case.image_name,
+	html_path = Emailer.make_path(case.output_dir, case.image_name,
 								 "AutopsyTestCase", "Reports")
 	try:
 		os.makedirs(os.path.join(case.gold, case.image_name, htmlfolder))
 		for file in os.listdir(autopsy_html_path):
-			html_to = make_path(case.gold, case.image_name, file.replace("HTML Report", "Report"))
+			html_to = Emailer.make_path(case.gold, case.image_name, file.replace("HTML Report", "Report"))
 			copy_dir(get_file_in_dir(autopsy_html_path, file), html_to)
 	except FileNotFoundException as e:
 		errors.append(e.error)
@@ -689,7 +653,7 @@ def rebuild():
 	os.chdir(zpdir)
 	img_gold = case.image_name
 	print(img_gold)
-	img_archive = make_path("..", case.image_name+"-archive.zip")
+	img_archive = Emailer.make_path("..", case.image_name+"-archive.zip")
 	comprssr = zipfile.ZipFile(img_archive, 'w',compression=zipfile.ZIP_DEFLATED)
 	zipdir(img_gold, comprssr)
 	comprssr.close()
@@ -709,8 +673,8 @@ def zipdir(path, zip):
 # from queries while comparing
 def compare_to_gold_db():
 	# SQLITE needs unix style pathing
-	gold_db_file = make_path(case.gold, case.image_name, "autopsy.db")
+	gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
-	autopsy_db_file = make_path(case.output_dir, case.image_name,
+	autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name,
 									  "AutopsyTestCase", "autopsy.db")
 	# Try to query the databases. Ignore any exceptions, the function will
 	# return an error later on if these do fail
@@ -764,12 +728,12 @@ def compare_to_gold_db():
 # Using the global case's variables, compare the html report file made by
 # the regression test against the gold standard html report
 def compare_to_gold_html():
-	gold_html_file = make_path(case.gold, case.image_name, "Report", "index.html")
+	gold_html_file = Emailer.make_path(case.gold, case.image_name, "Report", "index.html")
 	htmlfolder = ""
-	for fs in os.listdir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports")):
+	for fs in os.listdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports")):
-		if os.path.isdir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)):
+		if os.path.isdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)):
 			htmlfolder = fs
-	autopsy_html_path = make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, "HTML Report") #, "AutopsyTestCase", "Reports", htmlfolder)
+	autopsy_html_path = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, "HTML Report") #, "AutopsyTestCase", "Reports", htmlfolder)
 	
 	
 	try:
@@ -784,14 +748,14 @@ def compare_to_gold_html():
 			return
 		#Find all gold .html files belonging to this case
 		ListGoldHTML = []
-		for fs in os.listdir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)):
+		for fs in os.listdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)):
 			if(fs.endswith(".html")):
 				ListGoldHTML.append(os.path.join(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, fs))
 		#Find all new .html files belonging to this case
 		ListNewHTML = []
-		for fs in os.listdir(make_path(case.gold, case.image_name)):
+		for fs in os.listdir(Emailer.make_path(case.gold, case.image_name)):
 			if (fs.endswith(".html")):
-				ListNewHTML.append(make_path(case.gold, case.image_name, fs))
+				ListNewHTML.append(Emailer.make_path(case.gold, case.image_name, fs))
 		#ensure both reports have the same number of files and are in the same order
 		if(len(ListGoldHTML) != len(ListNewHTML)):
 			printerror("The reports did not have the same number of files. One of the reports may have been corrupted")
@@ -887,16 +851,16 @@ def compare_tsk_objects():
 # from each log file generated by Autopsy
 def generate_common_log():
 	try:
-		logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+		logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs")
 		common_log = codecs.open(case.common_log_path, "w", "utf_8")
 		warning_log = codecs.open(case.warning_log, "w", "utf_8")
 		common_log.write("--------------------------------------------------\n")
 		common_log.write(case.image_name + "\n")
 		common_log.write("--------------------------------------------------\n")
-		rep_path = make_local_path(case.output_dir)
+		rep_path = Emailer.make_local_path(case.output_dir)
 		rep_path = rep_path.replace("\\\\", "\\")
 		for file in os.listdir(logs_path):
-			log = codecs.open(make_path(logs_path, file), "r", "utf_8")
+			log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8")
 			for line in log:
 				line = line.replace(rep_path, "CASE")
 				if line.startswith("Exception"):
@@ -912,7 +876,7 @@ def generate_common_log():
 			log.close()
 		common_log.write("\n\n")
 		common_log.close()
-		case.sorted_log = make_local_path(case.output_dir, case.image_name, case.image_name + "SortedErrors.txt")
+		case.sorted_log = Emailer.make_local_path(case.output_dir, case.image_name, case.image_name + "SortedErrors.txt")
 		srtcmdlst = ["sort", case.common_log_path, "-o", case.sorted_log]
 		subprocess.call(srtcmdlst)
 	except Exception as e:
@@ -921,14 +885,14 @@ def generate_common_log():
 		logging.critical(traceback.format_exc())
 		
 def	compare_errors():
-	gold_dir = make_path(case.gold, case.image_name, case.image_name + "SortedErrors.txt")
+	gold_dir = Emailer.make_path(case.gold, case.image_name, case.image_name + "SortedErrors.txt")
 	common_log = codecs.open(case.sorted_log, "r", "utf_8")
 	gold_log = codecs.open(gold_dir, "r", "utf_8")
 	gold_dat = gold_log.read()
 	common_dat = common_log.read()
 	patrn = re.compile("\d")
 	if (not((re.sub(patrn, 'd', gold_dat)) == (re.sub(patrn, 'd', common_dat)))):
-		diff_dir = make_local_path(case.output_dir, case.image_name, "ErrorDiff.txt")
+		diff_dir = Emailer.make_local_path(case.output_dir, case.image_name, "ErrorDiff.txt")
 		diff_file = open(diff_dir, "w") 
 		dffcmdlst = ["diff", case.sorted_log, gold_dir]
 		subprocess.call(dffcmdlst, stdout = diff_file)
@@ -944,7 +908,7 @@ def	compare_errors():
 def fill_case_data():
 	try:
 		# Open autopsy.log.0
-		log_path = make_local_path(case.output_dir, case.image_name, "logs", "autopsy.log.0")
+		log_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs", "autopsy.log.0")
 		log = open(log_path)
 		
 		# Set the case starting time based off the first line of autopsy.log.0
@@ -1047,10 +1011,10 @@ def generate_csv(csv_path):
 		vars.append( str(database.autopsy_objects) )
 		vars.append( str(database.get_artifacts_count()) )
 		vars.append( str(database.autopsy_attributes) )
-		vars.append( make_local_path("gold", case.image_name, "autopsy.db") )
+		vars.append( Emailer.make_local_path("gold", case.image_name, "autopsy.db") )
 		vars.append( database.get_artifact_comparison() )
 		vars.append( database.get_attribute_comparison() )
-		vars.append( make_local_path("gold", case.image_name, "standard.html") )
+		vars.append( Emailer.make_local_path("gold", case.image_name, "standard.html") )
 		vars.append( str(case.report_passed) )
 		vars.append( case.ant_to_string() )
 		
@@ -1108,11 +1072,11 @@ def csv_header(csv_path):
 # Returns a list of all the exceptions listed in all the autopsy logs
 def get_exceptions():
 	exceptions = []
-	logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+	logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs")
 	results = []
 	for file in os.listdir(logs_path):
 		if "autopsy.log" in file:
-			log = codecs.open(make_path(logs_path, file), "r", "utf_8")
+			log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8")
 			ex = re.compile("\SException")
 			er = re.compile("\SError")
 			for line in log:
@@ -1143,10 +1107,10 @@ def report_all_errors():
 # Searched all the known logs for the given regex
 # The function expects regex = re.compile(...)
 def regex_search_logs(regex):
-	logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+	logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs")
 	results = []
 	for file in os.listdir(logs_path):
-		log = codecs.open(make_path(logs_path, file), "r", "utf_8")
+		log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8")
 		for line in log:
 			if regex.search(line):
 				results.append(line)
@@ -1157,10 +1121,10 @@ def regex_search_logs(regex):
 # Search through all the known log files for a specific string.
 # Returns a list of all lines with that string
 def search_logs(string):
-	logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+	logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs")
 	results = []
 	for file in os.listdir(logs_path):
-		log = codecs.open(make_path(logs_path, file), "r", "utf_8")
+		log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8")
 		for line in log:
 			if string in line:
 				results.append(line)
@@ -1180,7 +1144,7 @@ def search_common_log(string):
 # Searches the given log for the given string
 # Returns a list of all lines with that string
 def search_log(log, string):
-	logs_path = make_local_path(case.output_dir, case.image_name, "logs", log)
+	logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs", log)
 	try:
 		results = []
 		log = codecs.open(logs_path, "r", "utf_8")
@@ -1196,11 +1160,11 @@ def search_log(log, string):
 # Search through all the the logs of the given type
 # Types include autopsy, tika, and solr
 def search_log_set(type, string):
-	logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+	logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs")
 	results = []
 	for file in os.listdir(logs_path):
 		if type in file:
-			log = codecs.open(make_path(logs_path, file), "r", "utf_8")
+			log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8")
 			for line in log:
 				if string in line:
 					results.append(line)
@@ -1272,9 +1236,9 @@ def generate_html():
 		logs = "<div id='logs'>\
 				<h2><a name='" + case.image_name + "-logs'>Logs</a></h2>\
 				<hr color='#00a00f'>"
-		logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+		logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs")
 		for file in os.listdir(logs_path):
-			logs += "<p><a href='file:\\" + make_path(logs_path, file) + "' target='_blank'>" + file + "</a></p>"
+			logs += "<p><a href='file:\\" + Emailer.make_path(logs_path, file) + "' target='_blank'>" + file + "</a></p>"
 		logs += "</div>"
 		
 		# All the testing information
@@ -1420,100 +1384,6 @@ def getDay():
 def newDay():
 	return getLastDay() != getDay()
 
-#Pulls from git
-def gitPull(TskOrAutopsy):
-	global SYS
-	global errorem
-	global attachl
-	ccwd = ""
-	gppth = make_local_path(case.output_dir, "GitPullOutput" + TskOrAutopsy + ".txt")
-	attachl.append(gppth)
-	gpout = open(gppth, 'a')
-	toPull = "http://www.github.com/sleuthkit/" + TskOrAutopsy
-	call = ["git", "pull", toPull]
-	if TskOrAutopsy == "sleuthkit":
-		ccwd = os.path.join("..", "..", "..", "sleuthkit")
-	else:
-		ccwd = os.path.join("..", "..")
-	subprocess.call(call, stdout=gpout, cwd=ccwd)
-	gpout.close()
-	
-
-#Builds TSK as a win32 applicatiion
-def vsBuild():
-	global redo
-	global tryredo
-	global passed
-	#Please ensure that the current working directory is $autopsy/testing/script
-	vs = []
-	vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe")
-	vs.append(os.path.join("..", "..", "..","sleuthkit", "win32", "Tsk-win.sln"))
-	vs.append("/p:configuration=release")
-	vs.append("/p:platform=win32")
-	vs.append("/t:clean")
-	vs.append("/t:rebuild")
-	print(vs)
-	VSpth = make_local_path(case.output_dir, "VSOutput.txt")
-	VSout = open(VSpth, 'a')
-	subprocess.call(vs, stdout=VSout)
-	VSout.close()
-	chk = os.path.join("..", "..", "..","sleuthkit", "win32", "Release", "libtsk_jni.dll")
-	try:
-		open(chk)
-	except IOError as e:
-		global errorem
-		global attachl
-		if(not tryredo):
-			errorem += "LIBTSK C++ failed to build.\n"
-			attachl.append(VSpth)
-			send_email()
-		tryredo = True
-		passed = False
-		redo = True
-		
-	
- 
-#Builds Autopsy or the Datamodel
-def antBuild(which, Build):
-	global redo
-	global passed
-	global tryredo
-	directory = os.path.join("..", "..")
-	ant = []
-	if which == "datamodel":
-		directory = os.path.join("..", "..", "..", "sleuthkit", "bindings", "java")
-	ant.append("ant")
-	ant.append("-f")
-	ant.append(directory)
-	ant.append("clean")
-	if(Build):
-		ant.append("build")
-	else:
-		ant.append("dist")
-	antpth = make_local_path(case.output_dir, "ant" + which + "Output.txt")
-	antout = open(antpth, 'a')
-	succd = subprocess.call(ant, stdout=antout)
-	antout.close()
-	global errorem
-	global attachl
-	if which == "datamodel":
-		chk = os.path.join("..", "..", "..","sleuthkit",  "bindings", "java", "dist", "TSK_DataModel.jar")
-		try:
-			open(chk)
-		except IOError as e:
-			if(not tryredo):
-				errorem += "DataModel Java build failed.\n"
-				attachl.append(antpth)
-				send_email()
-			passed = False
-			tryredo = True
-	elif (succd != 0 and (not tryredo)):
-		errorem += "Autopsy build failed.\n"
-		attachl.append(antpth)
-		send_email()
-		tryredo = True
-	elif (succd != 0):
-		passed = False
 		
 	
 #Watches clock and waits for current ingest to be done
@@ -1533,39 +1403,11 @@ def dir_exists(dir):
 	except:
 		return False
 
-# Returns a Windows style path starting with the cwd and
-# ending with the list of directories given
-def make_local_path(*dirs):
-	path = wgetcwd()
-	for dir in dirs:
-		path += ("\\" + dir)
-	return path_fix(path)
-
-# Returns a Windows style path based only off the given directories
-def make_path(*dirs):
-	path = dirs[0]
-	for dir in dirs[1:]:
-		path += ("\\" + dir)
-	return path_fix(path)
-	
-# Fix a standard os.path by making it Windows format
-def path_fix(path):
-	return path.replace("/", "\\")
-
-# Gets the true current working directory instead of Cygwin's
-def wgetcwd():
-	if SYS is OS.CYGWIN:
-		proc = subprocess.Popen(("cygpath", "-m", os.getcwd()), stdout=subprocess.PIPE)
-		out,err = proc.communicate()
-		return out.rstrip()
-	elif SYS is OS.WIN:
-		return os.getcwd()
-
 # Copy the log files from Autopsy's default directory
 def copy_logs():
 	try:
 		log_dir = os.path.join("..", "..", "Testing","build","test","qa-functional","work","userdir0","var","log")
-		shutil.copytree(log_dir, make_local_path(case.output_dir, case.image_name, "logs"))
+		shutil.copytree(log_dir, Emailer.make_local_path(case.output_dir, case.image_name, "logs"))
 	except Exception as e:
 		printerror("Error: Failed to copy the logs.")
 		printerror(str(e) + "\n")
@@ -1614,7 +1456,7 @@ def get_file_in_dir(dir, ext):
 	try:
 		for file in os.listdir(dir):
 			if file.endswith(ext):
-				return make_path(dir, file)
+				return Emailer.make_path(dir, file)
 		# If nothing has been found, raise an exception
 		raise FileNotFoundException(dir)
 	except:
@@ -1625,7 +1467,7 @@ def find_file_in_dir(dir, name, ext):
 		for file in os.listdir(dir):
 			if file.startswith(name):
 				if file.endswith(ext):
-					return make_path(dir, file)
+					return Emailer.make_path(dir, file)
 		raise FileNotFoundException(dir)
 	except:
 		raise DirNotFoundException(dir)
@@ -1742,14 +1584,16 @@ class DirNotFoundException(Exception):
 
 #Executes the tests, makes continuous testing easier 
 def execute_test():
+	global parsed
+	global errorem
 	global failedbool
 	global html
 	global attachl
-	case.output_dir = make_path("..", "output", "results", time.strftime("%Y.%m.%d-%H.%M.%S"))
+	case.output_dir = Emailer.make_path("..", "output", "results", time.strftime("%Y.%m.%d-%H.%M.%S"))
 	os.makedirs(case.output_dir)
 	case.common_log = "AutopsyErrors.txt"
-	case.csv = make_local_path(case.output_dir, "CSV.txt")
+	case.csv = Emailer.make_local_path(case.output_dir, "CSV.txt")
-	case.html_log = make_local_path(case.output_dir, "AutopsyTestCase.html")
+	case.html_log = Emailer.make_local_path(case.output_dir, "AutopsyTestCase.html")
 	log_name = case.output_dir + "\\regression.log"
 	logging.basicConfig(filename=log_name, level=logging.DEBUG)
 	# If user wants to do a single file and a list (contradictory?)
@@ -1797,53 +1641,13 @@ def execute_test():
 		errorem += "There were no Errors.\n"
 		attachl = []
 	if not args.gold_creation:
-		send_email()
+		Emailer.send_email(parsed, errorem, attachl, html)
-
-
-def send_email():
-	global parsed
-	global errorem
-	global attachl
-	global html
-	if(not args.list):
-		sys.exit()
-	element = parsed.getElementsByTagName("email")
-	if(len(element)<=0):
-		return
-	element = element[0]
-	toval = element.getAttribute("value").encode().decode("utf_8")
-	if(toval==None):
-		return
-	element = parsed.getElementsByTagName("mail_server")[0]
-	serverval = element.getAttribute("value").encode().decode("utf_8")
-	# Create the container (outer) email message.
-	msg = MIMEMultipart()
-	msg['Subject'] = 'Email Test'
-	# me == the sender's email address
-	# family = the list of all recipients' email addresses
-	msg['From'] = 'AutopsyContinuousTest'
-	msg['To'] = toval
-	msg.preamble = 'This is a test'
-	container = MIMEText(errorem, 'plain')
-	msg.attach(container)
-	Build_email(msg)
-	s = smtplib.SMTP(serverval)
-	s.sendmail(msg['From'], msg['To'], msg.as_string())
-	s.quit()
-	
-def Build_email(msg):
-	global attachl
-	for file in attachl:
-		part = MIMEBase('application', "octet-stream")
-		atach = open(file, "rb")
-		attch = atach.read()
-		noml = file.split("\\")
-		nom = noml[len(noml)-1]
-		part.set_payload(attch)
-		Encoders.encode_base64(part)
-		part.add_header('Content-Disposition', 'attachment; filename="' + nom + '"')
-		msg.attach(part)
 		
+def secs_till_tommorow():
+	seconds = (23*3600)-(int(strftime("%H", localtime()))*3600)
+	seconds += (59*60)-(int(strftime("%M", localtime()))*60)
+	seconds += 60-(int(strftime("%S", localtime())))
+	return seconds+5
 #----------------------#
 #		 Main		 #
 #----------------------#
@@ -1887,7 +1691,10 @@ def main():
 		pass
 	# Otherwise test away!
 	else:
+		print(args.daily and args.contin)
 		execute_test()
+		if(args.daily and args.contin):
+			time.sleep(secs_till_tommorow())
 		while args.contin:
 			redo = False
 			attachl = []

test/script/Emailer.py

@@ -0,0 +1,83 @@
+import smtplib
+from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.MIMEBase import MIMEBase
+from email import Encoders
+import urllib2
+import xml
+from time import localtime, strftime
+from xml.dom.minidom import parse, parseString
+import subprocess
+import sys
+import os
+
+def send_email(parsed, errorem, attachl):
+	if(not args.list):
+		sys.exit()
+	element = parsed.getElementsByTagName("email")
+	if(len(element)<=0):
+		return
+	element = element[0]
+	toval = element.getAttribute("value").encode().decode("utf_8")
+	if(toval==None):
+		return
+	element = parsed.getElementsByTagName("mail_server")[0]
+	serverval = element.getAttribute("value").encode().decode("utf_8")
+	# Create the container (outer) email message.
+	msg = MIMEMultipart()
+	msg['Subject'] = 'Email Test'
+	# me == the sender's email address
+	# family = the list of all recipients' email addresses
+	msg['From'] = 'AutopsyContinuousTest'
+	msg['To'] = toval
+	msg.preamble = 'This is a test'
+	container = MIMEText(errorem, 'plain')
+	msg.attach(container)
+	Build_email(msg, attachl)
+	s = smtplib.SMTP(serverval)
+	s.sendmail(msg['From'], msg['To'], msg.as_string())
+	s.quit()
+	
+def Build_email(msg, attachl):
+	for file in attachl:
+		part = MIMEBase('application', "octet-stream")
+		atach = open(file, "rb")
+		attch = atach.read()
+		noml = file.split("\\")
+		nom = noml[len(noml)-1]
+		part.set_payload(attch)
+		Encoders.encode_base64(part)
+		part.add_header('Content-Disposition', 'attachment; filename="' + nom + '"')
+		msg.attach(part)
+		
+# Returns a Windows style path starting with the cwd and
+# ending with the list of directories given
+def make_local_path(*dirs):
+	path = wgetcwd()
+	for dir in dirs:
+		path += ("\\" + dir)
+	return path_fix(path)
+
+# Returns a Windows style path based only off the given directories
+def make_path(*dirs):
+	path = dirs[0]
+	for dir in dirs[1:]:
+		path += ("\\" + dir)
+	return path_fix(path)
+	
+# Fix a standard os.path by making it Windows format
+def path_fix(path):
+	return path.replace("/", "\\")
+	
+# Gets the true current working directory instead of Cygwin's
+def wgetcwd():
+	proc = subprocess.Popen(("cygpath", "-m", os.getcwd()), stdout=subprocess.PIPE)
+	out,err = proc.communicate()
+	tst = out.rstrip()
+	if os.getcwd == tst:
+		return os.getcwd
+	else:
+		proc = subprocess.Popen(("cygpath", "-m", os.getcwd()), stdout=subprocess.PIPE)
+		out,err = proc.communicate()
+		return out.rstrip()

test/script/srcupdater.py

@@ -0,0 +1,186 @@
+import codecs
+import datetime
+import logging
+import os
+import re
+import shutil
+import socket
+import sqlite3
+import subprocess
+import sys
+from sys import platform as _platform
+import time
+import traceback
+import xml
+from xml.dom.minidom import parse, parseString
+import Emailer
+
+def compile():
+	global redo
+	global tryredo
+	global failedbool
+	global errorem
+	global attachl
+	global passed
+	global parsed
+	passed = True
+	tryredo = False
+	redo = True
+	while(redo):
+		passed = True
+		if(passed):
+			gitPull("sleuthkit")
+		if(passed):
+			vsBuild()
+		if(passed):
+			gitPull("autopsy")
+		if(passed):
+			antBuild("datamodel", False)
+		if(passed):
+			antBuild("autopsy", True)
+		if(passed):
+			redo = False
+		else:
+			print("Compile Failed")
+			time.sleep(3600)
+	attachl = []
+	errorem = "The test standard didn't match the gold standard.\n"
+	failedbool = False
+	if(tryredo):
+		errorem += "Rebuilt properly.\n"
+		send_email(parsed, errorem, attachl)
+		attachl = []
+		errorem = "The test standard didn't match the gold standard.\n"	
+		passed = True
+		
+#Pulls from git
+def gitPull(TskOrAutopsy):
+	global SYS
+	global errorem
+	global attachl
+	ccwd = ""
+	gppth = Emailer.make_local_path("..", "GitPullOutput" + TskOrAutopsy + ".txt")
+	attachl.append(gppth)
+	gpout = open(gppth, 'a')
+	toPull = "http://www.github.com/sleuthkit/" + TskOrAutopsy
+	call = ["git", "pull", toPull]
+	if TskOrAutopsy == "sleuthkit":
+		ccwd = os.path.join("..", "..", "..", "sleuthkit")
+	else:
+		ccwd = os.path.join("..", "..")
+	subprocess.call(call, stdout=gpout, cwd=ccwd)
+	gpout.close()
+	
+
+#Builds TSK as a win32 applicatiion
+def vsBuild():
+	global redo
+	global tryredo
+	global passed
+	global parsed
+	#Please ensure that the current working directory is $autopsy/testing/script
+	vs = []
+	vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe")
+	vs.append(os.path.join("..", "..", "..","sleuthkit", "win32", "Tsk-win.sln"))
+	vs.append("/p:configuration=release")
+	vs.append("/p:platform=win32")
+	vs.append("/t:clean")
+	vs.append("/t:rebuild")
+	print(vs)
+	VSpth = Emailer.make_local_path("..", "VSOutput.txt")
+	VSout = open(VSpth, 'a')
+	subprocess.call(vs, stdout=VSout)
+	VSout.close()
+	chk = os.path.join("..", "..", "..","sleuthkit", "win32", "Release", "libtsk_jni.dll")
+	try:
+		open(chk)
+	except IOError as e:
+		global errorem
+		global attachl
+		if(not tryredo):
+			errorem += "LIBTSK C++ failed to build.\n"
+			attachl.append(VSpth)
+			send_email(parsed, errorem, attachl)
+		tryredo = True
+		passed = False
+		redo = True
+		
+	
+ 
+#Builds Autopsy or the Datamodel
+def antBuild(which, Build):
+	global redo
+	global passed
+	global tryredo
+	global parsed
+	directory = os.path.join("..", "..")
+	ant = []
+	if which == "datamodel":
+		directory = os.path.join("..", "..", "..", "sleuthkit", "bindings", "java")
+	ant.append("ant")
+	ant.append("-f")
+	ant.append(directory)
+	ant.append("clean")
+	if(Build):
+		ant.append("build")
+	else:
+		ant.append("dist")
+	antpth = Emailer.make_local_path("..", "ant" + which + "Output.txt")
+	antout = open(antpth, 'a')
+	succd = subprocess.call(ant, stdout=antout)
+	antout.close()
+	global errorem
+	global attachl
+	if which == "datamodel":
+		chk = os.path.join("..", "..", "..","sleuthkit",  "bindings", "java", "dist", "TSK_DataModel.jar")
+		try:
+			open(chk)
+		except IOError as e:
+			if(not tryredo):
+				errorem += "DataModel Java build failed.\n"
+				attachl.append(antpth)
+				send_email()
+			passed = False
+			tryredo = True
+	elif (succd != 0 and (not tryredo)):
+		errorem += "Autopsy build failed.\n"
+		attachl.append(antpth)
+		send_email(parsed, errorem, attachl)
+		tryredo = True
+	elif (succd != 0):
+		passed = False
+
+
+def main():
+	global parsed
+	global errorem
+	global attachl
+	print("hello")
+	errorem = ""
+	attachl = []
+	config_file = ""
+	print("hi")
+	arg = sys.argv.pop(0)
+	arg = sys.argv.pop(0)
+	config_file = arg
+	print(config_file)
+	parsed = parse(config_file)
+	compile()
+	
+class OS:
+  LINUX, MAC, WIN, CYGWIN = range(4)	  
+if __name__ == "__main__":
+	global SYS
+	if _platform == "linux" or _platform == "linux2":
+		SYS = OS.LINUX
+	elif _platform == "darwin":
+		SYS = OS.MAC
+	elif _platform == "win32":
+		SYS = OS.WIN
+	elif _platform == "cygwin":
+		SYS = OS.CYGWIN
+		
+	if SYS is OS.WIN or SYS is OS.CYGWIN:
+		main()
+	else:
+		print("We only support Windows and Cygwin at this time.")