diff --git a/Test/script/regression.py b/Test/script/regression.py index a7da6c8557..d23541e0f1 100644 --- a/Test/script/regression.py +++ b/Test/script/regression.py @@ -26,6 +26,7 @@ import urllib2 import re import zipfile import zlib +import Emailer # # Please read me... @@ -69,6 +70,7 @@ class Args: self.exception_string = "" self.contin = False self.gold_creation = False + self.daily = False def parse(self): global nxtproc @@ -134,6 +136,9 @@ class Args: elif arg == "-g" or arg == "--gold": printout("Creating gold standards") self.gold_creation = True + elif arg == "-d" or arg == "--daily": + printout("Running daily") + self.daily = True else: printout(usage()) return False @@ -148,9 +153,9 @@ class Args: class TestAutopsy: def __init__(self): # Paths: - self.input_dir = make_local_path("..","input") + self.input_dir = Emailer.make_local_path("..","input") self.output_dir = "" - self.gold = make_local_path("..", "output", "gold", "tmp") + self.gold = Emailer.make_local_path("..", "output", "gold", "tmp") # Logs: self.antlog_dir = "" self.common_log = "" @@ -307,7 +312,7 @@ class Database: def generate_autopsy_artifacts(self): if not self.autopsy_artifacts: - autopsy_db_file = make_path(case.output_dir, case.image_name, + autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db") autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() @@ -325,7 +330,7 @@ class Database: def generate_autopsy_attributes(self): if self.autopsy_attributes == 0: - autopsy_db_file = make_path(case.output_dir, case.image_name, + autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db") autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() @@ -335,7 +340,7 @@ class Database: def generate_autopsy_objects(self): if self.autopsy_objects == 0: - autopsy_db_file = make_path(case.output_dir, case.image_name, + autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db") autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() @@ -345,7 +350,7 @@ class Database: def generate_gold_artifacts(self): if not self.gold_artifacts: - gold_db_file = make_path(case.gold, case.image_name, "autopsy.db") + gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db") gold_con = sqlite3.connect(gold_db_file) gold_cur = gold_con.cursor() gold_cur.execute("SELECT COUNT(*) FROM blackboard_artifact_types") @@ -361,7 +366,7 @@ class Database: def generate_gold_attributes(self): if self.gold_attributes == 0: - gold_db_file = make_path(case.gold, case.image_name, "autopsy.db") + gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db") gold_con = sqlite3.connect(gold_db_file) gold_cur = gold_con.cursor() gold_cur.execute("SELECT COUNT(*) FROM blackboard_attributes") @@ -369,7 +374,7 @@ class Database: def generate_gold_objects(self): if self.gold_objects == 0: - gold_db_file = make_path(case.gold, case.image_name, "autopsy.db") + gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db") gold_con = sqlite3.connect(gold_db_file) gold_cur = gold_con.cursor() gold_cur.execute("SELECT COUNT(*) FROM tsk_objects") @@ -394,7 +399,7 @@ def run_config_test(config_file): case.input_dir = parsed.getElementsByTagName("indir")[0].getAttribute("value").encode().decode("utf_8") if parsed.getElementsByTagName("global_csv"): case.global_csv = parsed.getElementsByTagName("global_csv")[0].getAttribute("value").encode().decode("utf_8") - case.global_csv = make_local_path(case.global_csv) + case.global_csv = Emailer.make_local_path(case.global_csv) # Generate the top navbar of the HTML for easy access to all images values = [] @@ -403,7 +408,7 @@ def run_config_test(config_file): if file_exists(value): values.append(value) count = len(values) - archives = make_path(case.gold, "..") + archives = Emailer.make_path(case.gold, "..") arcount = 0 for file in os.listdir(archives): if not(file == 'tmp'): @@ -425,7 +430,6 @@ def run_config_test(config_file): #Begin infiniloop if(newDay()): global daycount - compile() if(daycount > 0): print("starting process") outputer = open("ScriptLog.txt", "a") @@ -448,46 +452,6 @@ def run_config_test(config_file): printerror("Error: There was an error running with the configuration file.") printerror(str(e) + "\n") logging.critical(traceback.format_exc()) - -def compile(): - global redo - global tryredo - global daycount - global nxtproc - global failedbool - global errorem - global attachl - global passed - passed = True - tryredo = False - setDay() - redo = True - while(redo): - passed = True - if(passed): - gitPull("sleuthkit") - if(passed): - vsBuild() - if(passed): - gitPull("autopsy") - if(passed): - antBuild("datamodel", False) - if(passed): - antBuild("autopsy", True) - if(passed): - redo = False - else: - print("Compile Failed") - time.sleep(3600) - attachl = [] - errorem = "The test standard didn't match the gold standard.\n" - failedbool = False - if(tryredo): - errorem += "Rebuilt properly.\n" - send_email() - attachl = [] - errorem = "The test standard didn't match the gold standard.\n" - passed = True # Runs the test on the single given file. # The path must be guarenteed to be a correct path. @@ -502,24 +466,24 @@ def run_test(image_file, count): case.image_file = image_file case.image_name = case.get_image_name(image_file) + "(" + str(count) + ")" case.image = case.get_image_name(image_file) - case.common_log_path = make_local_path(case.output_dir, case.image_name, case.image_name+case.common_log) - case.warning_log = make_local_path(case.output_dir, case.image_name, "AutopsyLogs.txt") - case.antlog_dir = make_local_path(case.output_dir, case.image_name, "antlog.txt") + case.common_log_path = Emailer.make_local_path(case.output_dir, case.image_name, case.image_name+case.common_log) + case.warning_log = Emailer.make_local_path(case.output_dir, case.image_name, "AutopsyLogs.txt") + case.antlog_dir = Emailer.make_local_path(case.output_dir, case.image_name, "antlog.txt") if(args.list): element = parsed.getElementsByTagName("build") if(len(element)<=0): - toval = make_path("..", "build.xml") + toval = Emailer.make_path("..", "build.xml") else: element = element[0] toval = element.getAttribute("value").encode().decode("utf_8") if(toval==None): - toval = make_path("..", "build.xml") + toval = Emailer.make_path("..", "build.xml") else: - toval = make_path("..", "build.xml") + toval = Emailer.make_path("..", "build.xml") case.build_path = toval - case.known_bad_path = make_path(case.input_dir, "notablehashes.txt-md5.idx") - case.keyword_path = make_path(case.input_dir, "notablekeywords.xml") - case.nsrl_path = make_path(case.input_dir, "nsrl.txt-md5.idx") + case.known_bad_path = Emailer.make_path(case.input_dir, "notablehashes.txt-md5.idx") + case.keyword_path = Emailer.make_path(case.input_dir, "notablekeywords.xml") + case.nsrl_path = Emailer.make_path(case.input_dir, "nsrl.txt-md5.idx") logging.debug("--------------------") logging.debug(case.image_name) @@ -538,7 +502,7 @@ def run_test(image_file, count): logging.critical(traceback.format_exc()) # If NOT keeping Solr index (-k) if not args.keep: - solr_index = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "KeywordSearch") + solr_index = Emailer.make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "KeywordSearch") if clear_dir(solr_index): print_report([], "DELETE SOLR INDEX", "Solr index deleted.") elif args.keep: @@ -558,8 +522,8 @@ def run_test(image_file, count): if not args.gold_creation: try: gold_path = case.gold - img_gold = make_path(case.gold, case.image_name) - img_archive = make_local_path("..", "output", "gold", case.image_name+"-archive.zip") + img_gold = Emailer.make_path(case.gold, case.image_name) + img_archive = Emailer.make_local_path("..", "output", "gold", case.image_name+"-archive.zip") extrctr = zipfile.ZipFile(img_archive, 'r', compression=zipfile.ZIP_DEFLATED) extrctr.extractall(gold_path) extrctr.close @@ -580,7 +544,7 @@ def run_test(image_file, count): if args.rebuild or args.gold_creation: rebuild() # Reset the case and return the tests sucessfully finished - clear_dir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "ModuleOutput", "keywordsearch")) + clear_dir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "ModuleOutput", "keywordsearch")) case.reset() return True @@ -604,8 +568,8 @@ def run_ant(): case.ant.append("-Dknown_bad_path=" + case.known_bad_path) case.ant.append("-Dkeyword_path=" + case.keyword_path) case.ant.append("-Dnsrl_path=" + case.nsrl_path) - case.ant.append("-Dgold_path=" + make_path(case.gold)) - case.ant.append("-Dout_path=" + make_local_path(case.output_dir, case.image_name)) + case.ant.append("-Dgold_path=" + Emailer.make_path(case.gold)) + case.ant.append("-Dout_path=" + Emailer.make_local_path(case.output_dir, case.image_name)) case.ant.append("-Dignore_unalloc=" + "%s" % args.unallocated) case.ant.append("-Dcontin_mode=" + str(args.contin)) case.ant.append("-Dtest.timeout=" + str(case.timeout)) @@ -613,7 +577,7 @@ def run_ant(): printout("Ingesting Image:\n" + case.image_file + "\n") printout("CMD: " + " ".join(case.ant)) printout("Starting test...\n") - antoutpth = make_local_path(case.output_dir, "antRunOutput.txt") + antoutpth = Emailer.make_local_path(case.output_dir, "antRunOutput.txt") antout = open(antoutpth, "a") if SYS is OS.CYGWIN: subprocess.call(case.ant, stdout=antout) @@ -652,32 +616,32 @@ def rebuild(): # Errors to print errors = [] # Delete the current gold standards - gold_dir = make_path(case.gold, case.image_name) + gold_dir = Emailer.make_path(case.gold, case.image_name) print("here") clear_dir(gold_dir) print("here1") - dbinpth = make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db") - dboutpth = make_path(case.gold, case.image_name, "autopsy.db") + dbinpth = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db") + dboutpth = Emailer.make_path(case.gold, case.image_name, "autopsy.db") if not os.path.exists(case.gold): os.makedirs(case.gold) if not os.path.exists(gold_dir): os.makedirs(gold_dir) copy_file(dbinpth, dboutpth) - error_pth = make_path(case.gold, case.image_name, case.image_name+"SortedErrors.txt") + error_pth = Emailer.make_path(case.gold, case.image_name, case.image_name+"SortedErrors.txt") copy_file(case.sorted_log, error_pth) # Rebuild the HTML report htmlfolder = "" for fs in os.listdir(os.path.join(os.getcwd(),case.output_dir, case.image_name, "AutopsyTestCase", "Reports")): if os.path.isdir(os.path.join(os.getcwd(), case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)): htmlfolder = fs - autopsy_html_path = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder) + autopsy_html_path = Emailer.make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder) - html_path = make_path(case.output_dir, case.image_name, + html_path = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports") try: os.makedirs(os.path.join(case.gold, case.image_name, htmlfolder)) for file in os.listdir(autopsy_html_path): - html_to = make_path(case.gold, case.image_name, file.replace("HTML Report", "Report")) + html_to = Emailer.make_path(case.gold, case.image_name, file.replace("HTML Report", "Report")) copy_dir(get_file_in_dir(autopsy_html_path, file), html_to) except FileNotFoundException as e: errors.append(e.error) @@ -689,7 +653,7 @@ def rebuild(): os.chdir(zpdir) img_gold = case.image_name print(img_gold) - img_archive = make_path("..", case.image_name+"-archive.zip") + img_archive = Emailer.make_path("..", case.image_name+"-archive.zip") comprssr = zipfile.ZipFile(img_archive, 'w',compression=zipfile.ZIP_DEFLATED) zipdir(img_gold, comprssr) comprssr.close() @@ -709,8 +673,8 @@ def zipdir(path, zip): # from queries while comparing def compare_to_gold_db(): # SQLITE needs unix style pathing - gold_db_file = make_path(case.gold, case.image_name, "autopsy.db") - autopsy_db_file = make_path(case.output_dir, case.image_name, + gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db") + autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db") # Try to query the databases. Ignore any exceptions, the function will # return an error later on if these do fail @@ -764,12 +728,12 @@ def compare_to_gold_db(): # Using the global case's variables, compare the html report file made by # the regression test against the gold standard html report def compare_to_gold_html(): - gold_html_file = make_path(case.gold, case.image_name, "Report", "index.html") + gold_html_file = Emailer.make_path(case.gold, case.image_name, "Report", "index.html") htmlfolder = "" - for fs in os.listdir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports")): - if os.path.isdir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)): + for fs in os.listdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports")): + if os.path.isdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)): htmlfolder = fs - autopsy_html_path = make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, "HTML Report") #, "AutopsyTestCase", "Reports", htmlfolder) + autopsy_html_path = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, "HTML Report") #, "AutopsyTestCase", "Reports", htmlfolder) try: @@ -784,14 +748,14 @@ def compare_to_gold_html(): return #Find all gold .html files belonging to this case ListGoldHTML = [] - for fs in os.listdir(make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)): + for fs in os.listdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)): if(fs.endswith(".html")): ListGoldHTML.append(os.path.join(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, fs)) #Find all new .html files belonging to this case ListNewHTML = [] - for fs in os.listdir(make_path(case.gold, case.image_name)): + for fs in os.listdir(Emailer.make_path(case.gold, case.image_name)): if (fs.endswith(".html")): - ListNewHTML.append(make_path(case.gold, case.image_name, fs)) + ListNewHTML.append(Emailer.make_path(case.gold, case.image_name, fs)) #ensure both reports have the same number of files and are in the same order if(len(ListGoldHTML) != len(ListNewHTML)): printerror("The reports did not have the same number of files. One of the reports may have been corrupted") @@ -887,16 +851,16 @@ def compare_tsk_objects(): # from each log file generated by Autopsy def generate_common_log(): try: - logs_path = make_local_path(case.output_dir, case.image_name, "logs") + logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs") common_log = codecs.open(case.common_log_path, "w", "utf_8") warning_log = codecs.open(case.warning_log, "w", "utf_8") common_log.write("--------------------------------------------------\n") common_log.write(case.image_name + "\n") common_log.write("--------------------------------------------------\n") - rep_path = make_local_path(case.output_dir) + rep_path = Emailer.make_local_path(case.output_dir) rep_path = rep_path.replace("\\\\", "\\") for file in os.listdir(logs_path): - log = codecs.open(make_path(logs_path, file), "r", "utf_8") + log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8") for line in log: line = line.replace(rep_path, "CASE") if line.startswith("Exception"): @@ -912,7 +876,7 @@ def generate_common_log(): log.close() common_log.write("\n\n") common_log.close() - case.sorted_log = make_local_path(case.output_dir, case.image_name, case.image_name + "SortedErrors.txt") + case.sorted_log = Emailer.make_local_path(case.output_dir, case.image_name, case.image_name + "SortedErrors.txt") srtcmdlst = ["sort", case.common_log_path, "-o", case.sorted_log] subprocess.call(srtcmdlst) except Exception as e: @@ -921,14 +885,14 @@ def generate_common_log(): logging.critical(traceback.format_exc()) def compare_errors(): - gold_dir = make_path(case.gold, case.image_name, case.image_name + "SortedErrors.txt") + gold_dir = Emailer.make_path(case.gold, case.image_name, case.image_name + "SortedErrors.txt") common_log = codecs.open(case.sorted_log, "r", "utf_8") gold_log = codecs.open(gold_dir, "r", "utf_8") gold_dat = gold_log.read() common_dat = common_log.read() patrn = re.compile("\d") if (not((re.sub(patrn, 'd', gold_dat)) == (re.sub(patrn, 'd', common_dat)))): - diff_dir = make_local_path(case.output_dir, case.image_name, "ErrorDiff.txt") + diff_dir = Emailer.make_local_path(case.output_dir, case.image_name, "ErrorDiff.txt") diff_file = open(diff_dir, "w") dffcmdlst = ["diff", case.sorted_log, gold_dir] subprocess.call(dffcmdlst, stdout = diff_file) @@ -944,7 +908,7 @@ def compare_errors(): def fill_case_data(): try: # Open autopsy.log.0 - log_path = make_local_path(case.output_dir, case.image_name, "logs", "autopsy.log.0") + log_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs", "autopsy.log.0") log = open(log_path) # Set the case starting time based off the first line of autopsy.log.0 @@ -1047,10 +1011,10 @@ def generate_csv(csv_path): vars.append( str(database.autopsy_objects) ) vars.append( str(database.get_artifacts_count()) ) vars.append( str(database.autopsy_attributes) ) - vars.append( make_local_path("gold", case.image_name, "autopsy.db") ) + vars.append( Emailer.make_local_path("gold", case.image_name, "autopsy.db") ) vars.append( database.get_artifact_comparison() ) vars.append( database.get_attribute_comparison() ) - vars.append( make_local_path("gold", case.image_name, "standard.html") ) + vars.append( Emailer.make_local_path("gold", case.image_name, "standard.html") ) vars.append( str(case.report_passed) ) vars.append( case.ant_to_string() ) @@ -1108,11 +1072,11 @@ def csv_header(csv_path): # Returns a list of all the exceptions listed in all the autopsy logs def get_exceptions(): exceptions = [] - logs_path = make_local_path(case.output_dir, case.image_name, "logs") + logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs") results = [] for file in os.listdir(logs_path): if "autopsy.log" in file: - log = codecs.open(make_path(logs_path, file), "r", "utf_8") + log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8") ex = re.compile("\SException") er = re.compile("\SError") for line in log: @@ -1143,10 +1107,10 @@ def report_all_errors(): # Searched all the known logs for the given regex # The function expects regex = re.compile(...) def regex_search_logs(regex): - logs_path = make_local_path(case.output_dir, case.image_name, "logs") + logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs") results = [] for file in os.listdir(logs_path): - log = codecs.open(make_path(logs_path, file), "r", "utf_8") + log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8") for line in log: if regex.search(line): results.append(line) @@ -1157,10 +1121,10 @@ def regex_search_logs(regex): # Search through all the known log files for a specific string. # Returns a list of all lines with that string def search_logs(string): - logs_path = make_local_path(case.output_dir, case.image_name, "logs") + logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs") results = [] for file in os.listdir(logs_path): - log = codecs.open(make_path(logs_path, file), "r", "utf_8") + log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8") for line in log: if string in line: results.append(line) @@ -1180,7 +1144,7 @@ def search_common_log(string): # Searches the given log for the given string # Returns a list of all lines with that string def search_log(log, string): - logs_path = make_local_path(case.output_dir, case.image_name, "logs", log) + logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs", log) try: results = [] log = codecs.open(logs_path, "r", "utf_8") @@ -1196,11 +1160,11 @@ def search_log(log, string): # Search through all the the logs of the given type # Types include autopsy, tika, and solr def search_log_set(type, string): - logs_path = make_local_path(case.output_dir, case.image_name, "logs") + logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs") results = [] for file in os.listdir(logs_path): if type in file: - log = codecs.open(make_path(logs_path, file), "r", "utf_8") + log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8") for line in log: if string in line: results.append(line) @@ -1272,9 +1236,9 @@ def generate_html(): logs = "