diff --git a/Testing/README.txt b/Testing/README.txt new file mode 100644 index 0000000000..a100bfd731 --- /dev/null +++ b/Testing/README.txt @@ -0,0 +1,5 @@ +This folder contains the Test module that is used to drive Autopsy. It +relies on NetBeans libraries to drive the UI. The test folder in the +root directory contains scripts and gold standard files to compare +against. That folder is where you should run regression tests from. + diff --git a/test/README.txt b/test/README.txt new file mode 100644 index 0000000000..70bf4b75ab --- /dev/null +++ b/test/README.txt @@ -0,0 +1,7 @@ +This folder contains the data and scripts required to run regression tests +for Autopsy. There is a 'Testing' folder in the root directory that contains +the Java code that drives Autopsy to perform the tests. + +To run these tests: +- Download the input images by typing 'ant test-download-imgs' in the root Autopsy folder. This will place images in 'test/input'. +- Run 'regression.py' from inside of the 'test/scripts' folder. diff --git a/test/script/regression.py b/test/script/regression.py index 3234c827cb..f062b62414 100644 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -221,7 +221,7 @@ class Database: self.autopsy_objects = 0 self.artifact_comparison = [] self.attribute_comparison = [] - self.test_img = case + self.test_data = case def clear(self): self.gold_artifacts = [] @@ -265,7 +265,7 @@ class Database: def _generate_autopsy_artifacts(self): if not self.autopsy_artifacts: - autopsy_db_file = Emailer.make_path(test_case.output_dir, self.test_img.image_name, + autopsy_db_file = Emailer.make_path(test_case.output_dir, self.test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() @@ -277,7 +277,7 @@ class Database: def _generate_autopsy_attributes(self): if self.autopsy_attributes == 0: - autopsy_db_file = Emailer.make_path(test_case.output_dir, self.test_img.image_name, + autopsy_db_file = Emailer.make_path(test_case.output_dir, self.test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() @@ -285,9 +285,12 @@ class Database: autopsy_attributes = autopsy_cur.fetchone()[0] self.autopsy_attributes = autopsy_attributes + # Counts number of objects and saves them into database. + # @@@ Should have better name (count_output_objects) and does not need to connect again. Should be storing connection in Database + # See also for _generate_autopsy_attributes def _generate_autopsy_objects(self): if self.autopsy_objects == 0: - autopsy_db_file = Emailer.make_path(test_case.output_dir, self.test_img.image_name, + autopsy_db_file = Emailer.make_path(test_case.output_dir, self.test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() @@ -295,11 +298,13 @@ class Database: autopsy_objects = autopsy_cur.fetchone()[0] self.autopsy_objects = autopsy_objects + # @@@ see _generate_autopsy_objects comment about better name, saving connections, etc. Or could have design where connection + # is passed in so that we do not need separate methods for gold and output. def _generate_gold_artifacts(self): if not self.gold_artifacts: - gold_db_file = Emailer.make_path(test_case.img_gold, self.test_img.image_name, test_case.test_db_file) + gold_db_file = Emailer.make_path(test_case.img_gold, self.test_data.image_name, test_case.test_db_file) if(not Emailer.file_exists(gold_db_file)): - gold_db_file = Emailer.make_path(test_case.img_gold_parse, self.test_img.image_name, test_case.test_db_file) + gold_db_file = Emailer.make_path(test_case.img_gold_parse, self.test_data.image_name, test_case.test_db_file) gold_con = sqlite3.connect(gold_db_file) gold_cur = gold_con.cursor() gold_cur.execute("SELECT COUNT(*) FROM blackboard_artifact_types") @@ -315,9 +320,9 @@ class Database: def _generate_gold_attributes(self): if self.gold_attributes == 0: - gold_db_file = Emailer.make_path(test_case.img_gold, self.test_img.image_name, test_case.test_db_file) + gold_db_file = Emailer.make_path(test_case.img_gold, self.test_data.image_name, test_case.test_db_file) if(not Emailer.file_exists(gold_db_file)): - gold_db_file = Emailer.make_path(test_case.img_gold_parse, self.test_img.image_name, test_case.test_db_file) + gold_db_file = Emailer.make_path(test_case.img_gold_parse, self.test_data.image_name, test_case.test_db_file) gold_con = sqlite3.connect(gold_db_file) gold_cur = gold_con.cursor() gold_cur.execute("SELECT COUNT(*) FROM blackboard_attributes") @@ -325,21 +330,24 @@ class Database: def _generate_gold_objects(self): if self.gold_objects == 0: - gold_db_file = Emailer.make_path(test_case.img_gold, self.test_img.image_name, test_case.test_db_file) + gold_db_file = Emailer.make_path(test_case.img_gold, self.test_data.image_name, test_case.test_db_file) if(not Emailer.file_exists(gold_db_file)): - gold_db_file = Emailer.make_path(test_case.img_gold_parse, self.test_img.image_name, test_case.test_db_file) + gold_db_file = Emailer.make_path(test_case.img_gold_parse, self.test_data.image_name, test_case.test_db_file) gold_con = sqlite3.connect(gold_db_file) gold_cur = gold_con.cursor() gold_cur.execute("SELECT COUNT(*) FROM tsk_objects") self.gold_objects = gold_cur.fetchone()[0] - def _retrieve_data(data_file, autopsy_con,autopsy_db_file, test_img): + # smart method that deals with blackboard comparison to avoid issues with different IDs based on when artifacts were created. + # Dumps sorted text results to output location stored in test_data. + # autopsy_db_file: Output database file + def _retrieve_data(autopsy_con, autopsy_db_file, test_data): autopsy_cur2 = autopsy_con.cursor() global errorem global attachl global failedbool autopsy_cur2.execute("SELECT tsk_files.parent_path, tsk_files.name, blackboard_artifact_types.display_name, blackboard_artifacts.artifact_id FROM blackboard_artifact_types INNER JOIN blackboard_artifacts ON blackboard_artifact_types.artifact_type_id = blackboard_artifacts.artifact_type_id INNER JOIN tsk_files ON tsk_files.obj_id = blackboard_artifacts.obj_id") - database_log = codecs.open(data_file, "wb", "utf_8") + database_log = codecs.open(test_data.autopsy_data_file, "wb", "utf_8") rw = autopsy_cur2.fetchone() appnd = False counter = 0 @@ -351,7 +359,7 @@ class Database: database_log.write(rw[1] + ' ') autopsy_cur1 = autopsy_con.cursor() looptry = True - test_img.artifact_count += 1 + test_data.artifact_count += 1 try: key = "" key = str(rw[3]) @@ -359,14 +367,14 @@ class Database: autopsy_cur1.execute("SELECT blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double FROM blackboard_attributes INNER JOIN blackboard_attribute_types ON blackboard_attributes.attribute_type_id = blackboard_attribute_types.attribute_type_id WHERE artifact_id =? ORDER BY blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double", key) attributes = autopsy_cur1.fetchall() except Exception as e: - printerror(test_img, str(e)) - printerror(test_img, str(rw[3])) - print(test_img.image_name) - errorem += test_img.image_name + ":Artifact with id#" + str(rw[3]) + " encountered an error.\n" + printerror(test_data, str(e)) + printerror(test_data, str(rw[3])) + print(test_data.image_name) + errorem += test_data.image_name + ":Artifact with id#" + str(rw[3]) + " encountered an error.\n" looptry = False - print(test_img.artifact_fail) - test_img.artifact_fail += 1 - print(test_img.artifact_fail) + print(test_data.artifact_fail) + test_data.artifact_fail += 1 + print(test_data.artifact_fail) if(looptry == True): src = attributes[0][0] for attr in attributes: @@ -376,15 +384,15 @@ class Database: if(attr[x] != None): numvals += 1 if(numvals > 1): - errorem += test_img.image_name + ":There were too many values for attribute type: " + attr[1] + " for artifact with id #" + str(rw[3]) + ".\n" - printerror(test_img, "There were too many values for attribute type: " + attr[1] + " for artifact with id #" + str(rw[3]) + " for image " + test_img.image_name + ".") + errorem += test_data.image_name + ":There were too many values for attribute type: " + attr[1] + " for artifact with id #" + str(rw[3]) + ".\n" + printerror(test_data, "There were too many values for attribute type: " + attr[1] + " for artifact with id #" + str(rw[3]) + " for image " + test_data.image_name + ".") failedbool = True if(not appnd): attachl.append(autopsy_db_file) appnd = True if(not attr[0] == src): - errorem += test_img.image_name + ":There were inconsistents sources for artifact with id #" + str(rw[3]) + ".\n" - printerror(test_img, "There were inconsistents sources for artifact with id #" + str(rw[3]) + " for image " + test_img.image_name + ".") + errorem += test_data.image_name + ":There were inconsistents sources for artifact with id #" + str(rw[3]) + ".\n" + printerror(test_data, "There were inconsistents sources for artifact with id #" + str(rw[3]) + " for image " + test_data.image_name + ".") failedbool = True if(not appnd): attachl.append(autopsy_db_file) @@ -399,30 +407,32 @@ class Database: try: database_log.write(inpval) except Exception as e: - printerror(test_img, "Inner exception" + outp) + printerror(test_data, "Inner exception" + outp) except Exception as e: - printerror(test_img, str(e)) + printerror(test_data, str(e)) database_log.write('" />') database_log.write(' \n') rw = autopsy_cur2.fetchone() - srtcmdlst = ["sort", test_img.autopsy_data_file, "-o", test_img.sorted_data_file] + srtcmdlst = ["sort", test_data.autopsy_data_file, "-o", test_data.sorted_data_file] subprocess.call(srtcmdlst) - print(test_img.artifact_fail) - if(test_img.artifact_fail > 0): - errorem += test_img.image_name + ":There were " + str(test_img.artifact_count) + " artifacts and " + str(test_img.artifact_fail) + " threw an exception while loading.\n" + print(test_data.artifact_fail) + if(test_data.artifact_fail > 0): + errorem += test_data.image_name + ":There were " + str(test_data.artifact_count) + " artifacts and " + str(test_data.artifact_fail) + " threw an exception while loading.\n" except Exception as e: - printerror(test_img, 'outer exception: ' + str(e)) + printerror(test_data, 'outer exception: ' + str(e)) - def _dbDump(test_img): - autopsy_db_file = Emailer.make_path(test_case.output_dir, test_img.image_name, + # Dumps a database (minus the artifact and attributes) to a text file. + # + def _dbDump(test_data): + autopsy_db_file = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) - backup_db_file = Emailer.make_path(test_case.output_dir, test_img.image_name, + backup_db_file = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "autopsy_backup.db") copy_file(autopsy_db_file,backup_db_file) autopsy_con = sqlite3.connect(backup_db_file) autopsy_con.execute("DROP TABLE blackboard_artifacts") autopsy_con.execute("DROP TABLE blackboard_attributes") - dump_file = Emailer.make_path(test_case.output_dir, test_img.image_name, test_img.image_name + "Dump.txt") + dump_file = Emailer.make_path(test_case.output_dir, test_data.image_name, test_data.image_name + "Dump.txt") database_log = codecs.open(dump_file, "wb", "utf_8") dump_list = autopsy_con.iterdump() try: @@ -430,83 +440,92 @@ class Database: try: database_log.write(line + "\n") except Exception as e: - printerror(test_img, "Inner dump Exception:" + str(e)) + printerror(test_data, "Inner dump Exception:" + str(e)) except Exception as e: - printerror(test_img, "Outer dump Exception:" + str(e)) + printerror(test_data, "Outer dump Exception:" + str(e)) - # Using the global test_case's variables, compare the database file made by the - # regression test to the gold standard database file - # Initializes the global database, which stores the information retrieved - # from queries while comparing - def compare_to_gold_db(test_img, database): + # Basic test between output and gold databases. Compares only counts of objects and blackboard items + # + # test_data: TestData object for current test + # database: Database object to use for comparison + # @@@ This is currently being called statically with databaes being passed in. Could be called as database.compare... and not pass database in. + # @@@ Rename + def compare_to_gold_db(test_data, database): # SQLITE needs unix style pathing - autopsy_db_file = Emailer.make_path(test_case.output_dir, test_img.image_name, + + # Get connection to output database from current run + autopsy_db_file = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() - gold_db_file = Emailer.make_path(test_case.img_gold, test_img.image_name, test_case.test_db_file) + + # Get connection to gold DB and count artifacts, etc. + gold_db_file = Emailer.make_path(test_case.img_gold, test_data.image_name, test_case.test_db_file) if(not Emailer.file_exists(gold_db_file)): - gold_db_file = Emailer.make_path(test_case.img_gold_parse, test_img.image_name, test_case.test_db_file) + gold_db_file = Emailer.make_path(test_case.img_gold_parse, test_data.image_name, test_case.test_db_file) try: database._generate_gold_objects() database._generate_gold_artifacts() database._generate_gold_attributes() except Exception as e: - printerror(test_img, "Way out:" + str(e)) + printerror(test_data, "Way out:" + str(e)) # This is where we return if a file doesn't exist, because we don't want to # compare faulty databases, but we do however want to try to run all queries # regardless of the other database if not Emailer.file_exists(autopsy_db_file): - printerror(test_img, "Error: Database file does not exist at:") - printerror(test_img, autopsy_db_file + "\n") + printerror(test_data, "Error: Database file does not exist at:") + printerror(test_data, autopsy_db_file + "\n") return if not Emailer.file_exists(gold_db_file): - printerror(test_img, "Error: Gold database file does not exist at:") - printerror(test_img, gold_db_file + "\n") + printerror(test_data, "Error: Gold database file does not exist at:") + printerror(test_data, gold_db_file + "\n") return + # compare size of bb artifacts, attributes, and tsk objects gold_con = sqlite3.connect(gold_db_file) gold_cur = gold_con.cursor() exceptions = [] - autopsy_db_file = Emailer.make_path(test_case.output_dir, test_img.image_name, + autopsy_db_file = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) + # Connect again and count things autopsy_con = sqlite3.connect(autopsy_db_file) try: database._generate_autopsy_objects() database._generate_autopsy_artifacts() database._generate_autopsy_attributes() except Exception as e: - printerror(test_img, "Way out:" + str(e)) - # Testing tsk_objects - exceptions.append(Database._compare_tsk_objects(test_img, database)) - # Testing blackboard_artifacts - exceptions.append(Database._count_bb_artifacts(test_img, database)) - # Testing blackboard_attributes - exceptions.append(Database._compare_bb_attributes(test_img, database)) + printerror(test_data, "Way out:" + str(e)) + + # Compare counts + exceptions.append(Database._compare_tsk_objects(test_data, database)) + exceptions.append(Database._count_bb_artifacts(test_data, database)) + exceptions.append(Database._compare_bb_attributes(test_data, database)) database.artifact_comparison = exceptions[1] database.attribute_comparison = exceptions[2] okay = "All counts match." - print_report(test_img, exceptions[0], "COMPARE TSK OBJECTS", okay) - print_report(test_img, exceptions[1], "COMPARE ARTIFACTS", okay) - print_report(test_img, exceptions[2], "COMPARE ATTRIBUTES", okay) + print_report(test_data, exceptions[0], "COMPARE TSK OBJECTS", okay) + print_report(test_data, exceptions[1], "COMPARE ARTIFACTS", okay) + print_report(test_data, exceptions[2], "COMPARE ATTRIBUTES", okay) - def get_Data(test_img): - autopsy_db_file = Emailer.make_path(test_case.output_dir, test_img.image_name, + # Dumps the given database to text files for later comparison + # @@@ Rename and follow-on methods + def get_Data(test_data): + autopsy_db_file = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() # Try to query the databases. Ignore any exceptions, the function will # return an error later on if these do fail - Database._retrieve_data(test_img.autopsy_data_file, autopsy_con,autopsy_db_file, test_img) - Database._dbDump(test_img) + Database._retrieve_data(autopsy_con,autopsy_db_file, test_data) + Database._dbDump(test_data) # Compares the blackboard artifact counts of two databases # given the two database cursors - def _count_bb_artifacts(test_img, database): + def _count_bb_artifacts(test_data, database): exceptions = [] try: global failedbool @@ -515,7 +534,7 @@ class Database: failedbool = True global imgfail imgfail = True - errorem += test_img.image + ":There was a TestDifference in the number of artifacts.\n" + errorem += test_data.image + ":There was a TestDifference in the number of artifacts.\n" rner = len(database.gold_artifacts) for type_id in range(1, rner): if database.gold_artifacts[type_id] != database.autopsy_artifacts[type_id]: @@ -526,13 +545,13 @@ class Database: exceptions.append(error) return exceptions except Exception as e: - printerror(test_img, str(e)) + printerror(test_data, str(e)) exceptions.append("Error: Unable to compare blackboard_artifacts.\n") return exceptions # Compares the blackboard atribute counts of two databases # given the two database cursors - def _compare_bb_attributes(test_img, database): + def _compare_bb_attributes(test_data, database): exceptions = [] try: if database.gold_attributes != database.autopsy_attributes: @@ -544,7 +563,7 @@ class Database: failedbool = True global imgfail imgfail = True - errorem += test_img.image + ":There was a TestDifference in the number of attributes.\n" + errorem += test_data.image + ":There was a TestDifference in the number of attributes.\n" return exceptions except Exception as e: exceptions.append("Error: Unable to compare blackboard_attributes.\n") @@ -552,7 +571,7 @@ class Database: # Compares the tsk object counts of two databases # given the two database cursors - def _compare_tsk_objects(test_img,database): + def _compare_tsk_objects(test_data,database): exceptions = [] try: if database.gold_objects != database.autopsy_objects: @@ -564,7 +583,7 @@ class Database: failedbool = True global imgfail imgfail = True - errorem += test_img.image + ":There was a TestDifference between the tsk object counts.\n" + errorem += test_data.image + ":There was a TestDifference between the tsk object counts.\n" return exceptions except Exception as e: exceptions.append("Error: Unable to compare tsk_objects.\n") @@ -575,18 +594,29 @@ class Database: #-------------------------------------------------# class TestDiffer: # - def run_diff(test_img): - TestDiffer._compare_errors(test_img) + def run_diff(test_data): + TestDiffer._compare_errors(test_data) + + # Compare smart blackboard stuff results gold_nm = "SortedData" - TestDiffer._compare_data(test_img.sorted_data_file, gold_nm, test_img) + TestDiffer._compare_data(test_data.sorted_data_file, gold_nm, test_data) + + # Compare the rest of the database (non-BB) gold_nm = "DBDump" - TestDiffer._compare_data(test_img.test_dbdump, gold_nm, test_img) - TestDiffer._compare_to_gold_html(test_img) + TestDiffer._compare_data(test_data.test_dbdump, gold_nm, test_data) + + # Compare html output + TestDiffer._compare_to_gold_html(test_data) - def _compare_data(aut, gld,test_img): - gold_dir = Emailer.make_path(test_case.img_gold, test_img.image_name, test_img.image_name + gld + ".txt") + # Compares database dump files. + # aut: output text file + # gld: gold text file + # test_data: Test being performed + # @@@ Could be renamed to be more generic about testing text files. + def _compare_data(aut, gld,test_data): + gold_dir = Emailer.make_path(test_case.img_gold, test_data.image_name, test_data.image_name + gld + ".txt") if(not Emailer.file_exists(gold_dir)): - gold_dir = Emailer.make_path(test_case.img_gold_parse, test_img.image_name, test_img.image_name + gld + ".txt") + gold_dir = Emailer.make_path(test_case.img_gold_parse, test_data.image_name, test_data.image_name + gld + ".txt") if(not Emailer.file_exists(aut)): return srtd_data = codecs.open(aut, "r", "utf_8") @@ -594,87 +624,88 @@ class TestDiffer: gold_dat = gold_data.read() srtd_dat = srtd_data.read() if (not(gold_dat == srtd_dat)): - diff_dir = Emailer.make_local_path(test_case.output_dir, test_img.image_name, test_img.image_name+gld+"-Diff.txt") + diff_dir = Emailer.make_local_path(test_case.output_dir, test_data.image_name, test_data.image_name+gld+"-Diff.txt") diff_file = codecs.open(diff_dir, "wb", "utf_8") - dffcmdlst = ["diff", test_img.sorted_data_file, gold_dir] + dffcmdlst = ["diff", test_data.sorted_data_file, gold_dir] subprocess.call(dffcmdlst, stdout = diff_file) global attachl global errorem global failedbool attachl.append(diff_dir) - errorem += test_img.image_name + ":There was a database TestDifference in the file " + gld + ".\n" - printerror(test_img, "There was a TestDifference in the Database data for " + test_img.image_name + " for the file " + gld + ".\n") + errorem += test_data.image_name + ":There was a database TestDifference in the file " + gld + ".\n" + printerror(test_data, "There was a TestDifference in the Database data for " + test_data.image_name + " for the file " + gld + ".\n") failedbool = True global imgfail imgfail = True - def _compare_errors(test_img): - gold_dir = Emailer.make_path(test_case.img_gold, test_img.image_name, test_img.image_name + "SortedErrors.txt") + # Compare merged error log files + def _compare_errors(test_data): + gold_dir = Emailer.make_path(test_case.img_gold, test_data.image_name, test_data.image_name + "SortedErrors.txt") if(not Emailer.file_exists(gold_dir)): - gold_dir = Emailer.make_path(test_case.img_gold_parse, test_img.image_name, test_img.image_name + "SortedErrors.txt") - common_log = codecs.open(test_img.sorted_log, "r", "utf_8") + gold_dir = Emailer.make_path(test_case.img_gold_parse, test_data.image_name, test_data.image_name + "SortedErrors.txt") + common_log = codecs.open(test_data.sorted_log, "r", "utf_8") gold_log = codecs.open(gold_dir, "r", "utf_8") gold_dat = gold_log.read() common_dat = common_log.read() patrn = re.compile("\d") if (not((re.sub(patrn, 'd', gold_dat)) == (re.sub(patrn, 'd', common_dat)))): - diff_dir = Emailer.make_local_path(test_case.output_dir, test_img.image_name, test_img.image_name+"AutopsyErrors-Diff.txt") + diff_dir = Emailer.make_local_path(test_case.output_dir, test_data.image_name, test_data.image_name+"AutopsyErrors-Diff.txt") diff_file = open(diff_dir, "w") - dffcmdlst = ["diff", test_img.sorted_log, gold_dir] + dffcmdlst = ["diff", test_data.sorted_log, gold_dir] subprocess.call(dffcmdlst, stdout = diff_file) global attachl global errorem global failedbool - attachl.append(test_img.sorted_log) + attachl.append(test_data.sorted_log) attachl.append(diff_dir) - errorem += test_img.image_name + ":There was a TestDifference in the exceptions Log.\n" - printerror(test_img, "Exceptions didn't match.\n") + errorem += test_data.image_name + ":There was a TestDifference in the exceptions Log.\n" + printerror(test_data, "Exceptions didn't match.\n") failedbool = True global imgfail imgfail = True - # Using the global test_case's variables, compare the html report file made by + # Compare the html report file made by # the regression test against the gold standard html report - def _compare_to_gold_html(test_img): - gold_html_file = Emailer.make_path(test_case.img_gold, test_img.image_name, "Report", "index.html") + def _compare_to_gold_html(test_data): + gold_html_file = Emailer.make_path(test_case.img_gold, test_data.image_name, "Report", "index.html") if(not Emailer.file_exists(gold_html_file)): - gold_html_file = Emailer.make_path(test_case.img_gold_parse, test_img.image_name, "Report", "index.html") + gold_html_file = Emailer.make_path(test_case.img_gold_parse, test_data.image_name, "Report", "index.html") htmlfolder = "" - for fs in os.listdir(Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports")): - if os.path.isdir(Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports", fs)): + for fs in os.listdir(Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports")): + if os.path.isdir(Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports", fs)): htmlfolder = fs - autopsy_html_path = Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder, "HTML Report") + autopsy_html_path = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder, "HTML Report") try: autopsy_html_file = get_file_in_dir(autopsy_html_path, "index.html") if not Emailer.file_exists(gold_html_file): - printerror(test_img, "Error: No gold html report exists at:") - printerror(test_img, gold_html_file + "\n") + printerror(test_data, "Error: No gold html report exists at:") + printerror(test_data, gold_html_file + "\n") return if not Emailer.file_exists(autopsy_html_file): - printerror(test_img, "Error: No test_case html report exists at:") - printerror(test_img, autopsy_html_file + "\n") + printerror(test_data, "Error: No test_case html report exists at:") + printerror(test_data, autopsy_html_file + "\n") return #Find all gold .html files belonging to this test_case ListGoldHTML = [] - for fs in os.listdir(Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder)): + for fs in os.listdir(Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder)): if(fs.endswith(".html")): - ListGoldHTML.append(Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder, fs)) + ListGoldHTML.append(Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder, fs)) #Find all new .html files belonging to this test_case ListNewHTML = [] - if(os.path.exists(Emailer.make_path(test_case.img_gold, test_img.image_name))): - for fs in os.listdir(Emailer.make_path(test_case.img_gold, test_img.image_name)): + if(os.path.exists(Emailer.make_path(test_case.img_gold, test_data.image_name))): + for fs in os.listdir(Emailer.make_path(test_case.img_gold, test_data.image_name)): if (fs.endswith(".html")): - ListNewHTML.append(Emailer.make_path(test_case.img_gold, test_img.image_name, fs)) + ListNewHTML.append(Emailer.make_path(test_case.img_gold, test_data.image_name, fs)) if(not test_case.img_gold_parse == "" or test_case.img_gold == test_case.img_gold_parse): - if(Emailer.file_exists(Emailer.make_path(test_case.img_gold_parse, test_img.image_name))): - for fs in os.listdir(Emailer.make_path(test_case.img_gold_parse,test_img.image_name)): + if(Emailer.file_exists(Emailer.make_path(test_case.img_gold_parse, test_data.image_name))): + for fs in os.listdir(Emailer.make_path(test_case.img_gold_parse,test_data.image_name)): if (fs.endswith(".html")): - ListNewHTML.append(Emailer.make_path(test_case.img_gold_parse, test_img.image_name, fs)) + ListNewHTML.append(Emailer.make_path(test_case.img_gold_parse, test_data.image_name, fs)) #ensure both reports have the same number of files and are in the same order if(len(ListGoldHTML) != len(ListNewHTML)): - printerror(test_img, "The reports did not have the same number of files. One of the reports may have been corrupted") + printerror(test_data, "The reports did not have the same number of files. One of the reports may have been corrupted") else: ListGoldHTML.sort() ListNewHTML.sort() @@ -686,18 +717,18 @@ class TestDiffer: total["New"]+=count[1] okay = "The test report matches the gold report." errors=["Gold report had " + str(total["Gold"]) +" errors", "New report had " + str(total["New"]) + " errors."] - print_report(test_img, errors, "REPORT COMPARISON", okay) + print_report(test_data, errors, "REPORT COMPARISON", okay) if total["Gold"] == total["New"]: - test_img.report_passed = True + test_data.report_passed = True else: - printerror(test_img, "The reports did not match each other.\n " + errors[0] +" and the " + errors[1]) + printerror(test_data, "The reports did not match each other.\n " + errors[0] +" and the " + errors[1]) except FileNotFoundException as e: e.print_error() except DirNotFoundException as e: e.print_error() except Exception as e: - printerror(test_img, "Error: Unknown fatal error comparing reports.") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unknown fatal error comparing reports.") + printerror(test_data, str(e) + "\n") logging.critical(traceback.format_exc()) class TestData: @@ -748,15 +779,15 @@ class TestData: self.printout = [] class Reports: - def generate_reports(csv_path, database, test_img): - Reports._generate_html(database, test_img) + def generate_reports(csv_path, database, test_data): + Reports._generate_html(database, test_data) if test_case.global_csv: - Reports._generate_csv(test_case.global_csv, database, test_img) + Reports._generate_csv(test_case.global_csv, database, test_data) else: - Reports._generate_csv(csv_path, database, test_img) + Reports._generate_csv(csv_path, database, test_data) # Generates the HTML log file - def _generate_html(database, test_img): + def _generate_html(database, test_data): # If the file doesn't exist yet, this is the first test_case to run for # this test, so we need to make the start of the html log global imgfail @@ -766,13 +797,13 @@ class Reports: global html html = open(test_case.html_log, "a") # The image title - title = "

" + test_img.image_name + " \ + title = "

" + test_data.image_name + " \ tested on " + socket.gethostname() + "

\

\ - Errors and Warnings |\ - Information |\ - General Output |\ - Logs\ + Errors and Warnings |\ + Information |\ + General Output |\ + Logs\

" # The script errors found if imgfail: @@ -780,10 +811,10 @@ class Reports: else: ids = 'errors' errors = "
\ -

Errors and Warnings

\ +

Errors and Warnings

\
" # For each error we have logged in the test_case - for error in test_img.printerror: + for error in test_data.printerror: # Replace < and > to avoid any html display errors errors += "

" + error.replace("<", "<").replace(">", ">") + "

" # If there is a \n, we probably want a
in the html @@ -793,51 +824,51 @@ class Reports: # Links to the logs logs = "
\ -

Logs

\ +

Logs

\
" - logs_path = Emailer.make_local_path(test_case.output_dir, test_img.image_name, "logs") + logs_path = Emailer.make_local_path(test_case.output_dir, test_data.image_name, "logs") for file in os.listdir(logs_path): logs += "

" + file + "

" logs += "
" # All the testing information info = "
\ -

Information

\ +

Information

\
\ " # The individual elements info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" info += "" info += "" info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" - info += "" + info += "" info += "" info += "" info += "" @@ -846,7 +877,7 @@ class Reports: info += "" info += "" - info += "" + info += "" info += "" info += "" info += "" @@ -857,10 +888,10 @@ class Reports: " # For all the general print statements in the test_case output = "
\ -

General Output

\ +

General Output

\
" # For each printout in the test_case's list - for out in test_img.printout: + for out in test_data.printout: output += "

" + out + "

" # If there was a \n it probably means we want a
in the html if "\n" in out: @@ -874,9 +905,9 @@ class Reports: html.write(output) html.close() except Exception as e: - printerror(test_img, "Error: Unknown fatal error when creating HTML log at:") - printerror(test_img, test_case.html_log) - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unknown fatal error when creating HTML log at:") + printerror(test_data, test_case.html_log) + printerror(test_data, str(e) + "\n") logging.critical(traceback.format_exc()) # Writed the top of the HTML log file @@ -927,7 +958,7 @@ class Reports: html.write("

" + (" | ".join(links)) + "

") # Generate the CSV log file - def _generate_csv(csv_path, database, test_img): + def _generate_csv(csv_path, database, test_data): try: # If the CSV file hasn't already been generated, this is the # first run, and we need to add the column names @@ -938,35 +969,35 @@ class Reports: # Variables that need to be written vars = [] - vars.append( test_img.image_file ) - vars.append( test_img.image_name ) + vars.append( test_data.image_file ) + vars.append( test_data.image_name ) vars.append( test_case.output_dir ) vars.append( socket.gethostname() ) vars.append( test_case.autopsy_version ) - vars.append( test_img.heap_space ) - vars.append( test_img.start_date ) - vars.append( test_img.end_date ) - vars.append( test_img.total_test_time ) - vars.append( test_img.total_ingest_time ) - vars.append( test_img.service_times ) - vars.append( str(len(get_exceptions(test_img))) ) - vars.append( str(Reports._get_num_memory_errors("autopsy", test_img)) ) - vars.append( str(Reports._get_num_memory_errors("tika", test_img)) ) - vars.append( str(Reports._get_num_memory_errors("solr", test_img)) ) - vars.append( str(len(search_log_set("autopsy", "TskCoreException", test_img))) ) - vars.append( str(len(search_log_set("autopsy", "TskDataException", test_img))) ) + vars.append( test_data.heap_space ) + vars.append( test_data.start_date ) + vars.append( test_data.end_date ) + vars.append( test_data.total_test_time ) + vars.append( test_data.total_ingest_time ) + vars.append( test_data.service_times ) + vars.append( str(len(get_exceptions(test_data))) ) + vars.append( str(Reports._get_num_memory_errors("autopsy", test_data)) ) + vars.append( str(Reports._get_num_memory_errors("tika", test_data)) ) + vars.append( str(Reports._get_num_memory_errors("solr", test_data)) ) + vars.append( str(len(search_log_set("autopsy", "TskCoreException", test_data))) ) + vars.append( str(len(search_log_set("autopsy", "TskDataException", test_data))) ) vars.append( str(test_case.ingest_messages) ) vars.append( str(test_case.indexed_files) ) vars.append( str(test_case.indexed_chunks) ) - vars.append( str(len(search_log_set("autopsy", "Stopping ingest due to low disk space on disk", test_img))) ) + vars.append( str(len(search_log_set("autopsy", "Stopping ingest due to low disk space on disk", test_data))) ) vars.append( str(database.autopsy_objects) ) vars.append( str(database.get_artifacts_count()) ) vars.append( str(database.autopsy_attributes) ) - vars.append( Emailer.make_local_path("gold", test_img.image_name, test_case.test_db_file) ) + vars.append( Emailer.make_local_path("gold", test_data.image_name, test_case.test_db_file) ) vars.append( database.get_artifact_comparison() ) vars.append( database.get_attribute_comparison() ) - vars.append( Emailer.make_local_path("gold", test_img.image_name, "standard.html") ) - vars.append( str(test_img.report_passed) ) + vars.append( Emailer.make_local_path("gold", test_data.image_name, "standard.html") ) + vars.append( str(test_data.report_passed) ) vars.append( test_case.ant_to_string() ) # Join it together with a ", " output = "|".join(vars) @@ -975,9 +1006,9 @@ class Reports: csv.write(output) csv.close() except Exception as e: - printerror(test_img, "Error: Unknown fatal error when creating CSV file at:") - printerror(test_img, csv_path) - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unknown fatal error when creating CSV file at:") + printerror(test_data, csv_path) + printerror(test_data, str(e) + "\n") print(traceback.format_exc()) logging.critical(traceback.format_exc()) @@ -1022,40 +1053,40 @@ class Reports: # Returns the number of OutOfMemoryErrors and OutOfMemoryExceptions # for a certain type of log - def _get_num_memory_errors(type, test_img): - return (len(search_log_set(type, "OutOfMemoryError", test_img)) + - len(search_log_set(type, "OutOfMemoryException", test_img))) + def _get_num_memory_errors(type, test_data): + return (len(search_log_set(type, "OutOfMemoryError", test_data)) + + len(search_log_set(type, "OutOfMemoryException", test_data))) class Logs: - def generate_log_data(test_img): - Logs._generate_common_log(test_img) + def generate_log_data(test_data): + Logs._generate_common_log(test_data) try: - Logs._fill_test_case_data(test_img) + Logs._fill_test_case_data(test_data) except Exception as e: - printerror(test_img, "Error: Unknown fatal error when filling test_case data.") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unknown fatal error when filling test_case data.") + printerror(test_data, str(e) + "\n") logging.critical(traceback.format_exc()) # If running in verbose mode (-v) if test_case.args.verbose: errors = Logs._report_all_errors() okay = "No warnings or errors in any log files." - print_report(test_img, errors, "VERBOSE", okay) + print_report(test_data, errors, "VERBOSE", okay) # Generate the "common log": a log of all exceptions and warnings # from each log file generated by Autopsy - def _generate_common_log(test_img): + def _generate_common_log(test_data): try: - logs_path = Emailer.make_local_path(test_case.output_dir, test_img.image_name, "logs") + logs_path = Emailer.make_local_path(test_case.output_dir, test_data.image_name, "logs") common_log = codecs.open(test_case.common_log_path, "w", "utf_8") - warning_log = codecs.open(test_img.warning_log, "w", "utf_8") + warning_log = codecs.open(test_data.warning_log, "w", "utf_8") common_log.write("--------------------------------------------------\n") - common_log.write(test_img.image_name + "\n") + common_log.write(test_data.image_name + "\n") common_log.write("--------------------------------------------------\n") rep_path = Emailer.make_local_path(test_case.output_dir) rep_path = rep_path.replace("\\\\", "\\") for file in os.listdir(logs_path): log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8") for line in log: - line = line.replace(rep_path, "test_img") + line = line.replace(rep_path, "test_data") if line.startswith("Exception"): common_log.write(file +": " + line) elif line.startswith("Error"): @@ -1067,66 +1098,66 @@ class Logs: log.close() common_log.write("\n") common_log.close() - print(test_img.sorted_log) - srtcmdlst = ["sort", test_case.common_log_path, "-o", test_img.sorted_log] + print(test_data.sorted_log) + srtcmdlst = ["sort", test_case.common_log_path, "-o", test_data.sorted_log] subprocess.call(srtcmdlst) except Exception as e: - printerror(test_img, "Error: Unable to generate the common log.") - printerror(test_img, str(e) + "\n") - printerror(test_img, traceback.format_exc()) + printerror(test_data, "Error: Unable to generate the common log.") + printerror(test_data, str(e) + "\n") + printerror(test_data, traceback.format_exc()) logging.critical(traceback.format_exc()) # Fill in the global test_case's variables that require the log files - def _fill_test_case_data(test_img): + def _fill_test_case_data(test_data): try: # Open autopsy.log.0 - log_path = Emailer.make_path(test_case.output_dir, test_img.image_name, "logs", "autopsy.log.0") + log_path = Emailer.make_path(test_case.output_dir, test_data.image_name, "logs", "autopsy.log.0") log = open(log_path) # Set the test_case starting time based off the first line of autopsy.log.0 # *** If logging time format ever changes this will break *** - test_img.start_date = log.readline().split(" org.")[0] + test_data.start_date = log.readline().split(" org.")[0] # Set the test_case ending time based off the "create" time (when the file was copied) - test_img.end_date = time.ctime(os.path.getmtime(log_path)) + test_data.end_date = time.ctime(os.path.getmtime(log_path)) except Exception as e: - printerror(test_img, "Error: Unable to open autopsy.log.0.") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unable to open autopsy.log.0.") + printerror(test_data, str(e) + "\n") logging.warning(traceback.format_exc()) # Set the test_case total test time # Start date must look like: "Jul 16, 2012 12:57:53 PM" # End date must look like: "Mon Jul 16 13:02:42 2012" # *** If logging time format ever changes this will break *** - start = datetime.datetime.strptime(test_img.start_date, "%b %d, %Y %I:%M:%S %p") - end = datetime.datetime.strptime(test_img.end_date, "%a %b %d %H:%M:%S %Y") - test_img.total_test_time = str(end - start) + start = datetime.datetime.strptime(test_data.start_date, "%b %d, %Y %I:%M:%S %p") + end = datetime.datetime.strptime(test_data.end_date, "%a %b %d %H:%M:%S %Y") + test_data.total_test_time = str(end - start) try: # Set Autopsy version, heap space, ingest time, and service times - version_line = search_logs("INFO: Application name: Autopsy, version:", test_img)[0] + version_line = search_logs("INFO: Application name: Autopsy, version:", test_data)[0] test_case.autopsy_version = Emailer.get_word_at(version_line, 5).rstrip(",") - test_img.heap_space = search_logs("Heap memory usage:", test_img)[0].rstrip().split(": ")[1] + test_data.heap_space = search_logs("Heap memory usage:", test_data)[0].rstrip().split(": ")[1] - ingest_line = search_logs("Ingest (including enqueue)", test_img)[0] - test_img.total_ingest_time = Emailer.get_word_at(ingest_line, 6).rstrip() + ingest_line = search_logs("Ingest (including enqueue)", test_data)[0] + test_data.total_ingest_time = Emailer.get_word_at(ingest_line, 6).rstrip() - message_line = search_log_set("autopsy", "Ingest messages count:", test_img)[0] + message_line = search_log_set("autopsy", "Ingest messages count:", test_data)[0] test_case.ingest_messages = int(message_line.rstrip().split(": ")[2]) - files_line = search_log_set("autopsy", "Indexed files count:", test_img)[0] + files_line = search_log_set("autopsy", "Indexed files count:", test_data)[0] test_case.indexed_files = int(files_line.rstrip().split(": ")[2]) - chunks_line = search_log_set("autopsy", "Indexed file chunks count:", test_img)[0] + chunks_line = search_log_set("autopsy", "Indexed file chunks count:", test_data)[0] test_case.indexed_chunks = int(chunks_line.rstrip().split(": ")[2]) except Exception as e: - printerror(test_img, "Error: Unable to find the required information to fill test_case data.") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unable to find the required information to fill test_case data.") + printerror(test_data, str(e) + "\n") logging.critical(traceback.format_exc()) print(traceback.format_exc()) try: - service_lines = search_log("autopsy.log.0", "to process()", test_img) + service_lines = search_log("autopsy.log.0", "to process()", test_data) service_list = [] for line in service_lines: words = line.split(" ") @@ -1139,10 +1170,10 @@ class Logs: times += words[i-1] + " " times += words[i] service_list.append(times) - test_img.service_times = "; ".join(service_list) + test_data.service_times = "; ".join(service_list) except Exception as e: - printerror(test_img, "Error: Unknown fatal error when finding service times.") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unknown fatal error when finding service times.") + printerror(test_data, str(e) + "\n") logging.critical(traceback.format_exc()) # Returns all the errors found in the common log in a list @@ -1150,11 +1181,11 @@ class Logs: try: return get_warnings() + get_exceptions() except Exception as e: - printerror(test_img, "Error: Unknown fatal error when reporting all errors.") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: Unknown fatal error when reporting all errors.") + printerror(test_data, str(e) + "\n") logging.warning(traceback.format_exc()) # Searches the common log for any instances of a specific string. - def search_common_log(string, test_img): + def search_common_log(string, test_data): results = [] log = codecs.open(test_case.common_log_path, "r", "utf_8") for line in log: @@ -1183,8 +1214,8 @@ def image_type(image_file): # Search through all the known log files for a specific string. # Returns a list of all lines with that string -def search_logs(string, test_img): - logs_path = Emailer.make_local_path(test_case.output_dir, test_img.image_name, "logs") +def search_logs(string, test_data): + logs_path = Emailer.make_local_path(test_case.output_dir, test_data.image_name, "logs") results = [] for file in os.listdir(logs_path): log = codecs.open(Emailer.make_path(logs_path, file), "r", "utf_8") @@ -1196,8 +1227,8 @@ def search_logs(string, test_img): # Searches the given log for the given string # Returns a list of all lines with that string -def search_log(log, string, test_img): - logs_path = Emailer.make_local_path(test_case.output_dir, test_img.image_name, "logs", log) +def search_log(log, string, test_data): + logs_path = Emailer.make_local_path(test_case.output_dir, test_data.image_name, "logs", log) try: results = [] log = codecs.open(logs_path, "r", "utf_8") @@ -1212,8 +1243,8 @@ def search_log(log, string, test_img): # Search through all the the logs of the given type # Types include autopsy, tika, and solr -def search_log_set(type, string, test_img): - logs_path = Emailer.make_local_path(test_case.output_dir, test_img.image_name, "logs") +def search_log_set(type, string, test_data): + logs_path = Emailer.make_local_path(test_case.output_dir, test_data.image_name, "logs") results = [] for file in os.listdir(logs_path): if type in file: @@ -1226,34 +1257,34 @@ def search_log_set(type, string, test_img): # Print a report for the given errors with the report name as name # and if no errors are found, print the okay message -def print_report(test_img, errors, name, okay): +def print_report(test_data, errors, name, okay): if errors: - printerror(test_img, "--------< " + name + " >----------") + printerror(test_data, "--------< " + name + " >----------") for error in errors: - printerror(test_img, str(error)) - printerror(test_img, "--------< / " + name + " >--------\n") + printerror(test_data, str(error)) + printerror(test_data, "--------< / " + name + " >--------\n") else: - printout(test_img, "-----------------------------------------------------------------") - printout(test_img, "< " + name + " - " + okay + " />") - printout(test_img, "-----------------------------------------------------------------\n") + printout(test_data, "-----------------------------------------------------------------") + printout(test_data, "< " + name + " - " + okay + " />") + printout(test_data, "-----------------------------------------------------------------\n") # Used instead of the print command when printing out an error -def printerror(test_img, string): +def printerror(test_data, string): print(string) - test_img.printerror.append(string) + test_data.printerror.append(string) # Used instead of the print command when printing out anything besides errors -def printout(test_img, string): +def printout(test_data, string): print(string) - test_img.printout.append(string) + test_data.printout.append(string) #----------------------------------# # Helper functions # #----------------------------------# # Returns a list of all the exceptions listed in all the autopsy logs -def get_exceptions(test_img): +def get_exceptions(test_data): exceptions = [] - logs_path = Emailer.make_path(test_case.output_dir, test_img.image_name, "logs") + logs_path = Emailer.make_path(test_case.output_dir, test_data.image_name, "logs") results = [] for file in os.listdir(logs_path): if "autopsy.log" in file: @@ -1267,22 +1298,22 @@ def get_exceptions(test_img): return exceptions # Returns a list of all the warnings listed in the common log -def get_warnings(test_img): +def get_warnings(test_data): warnings = [] - common_log = codecs.open(test_img.warning_log, "r", "utf_8") + common_log = codecs.open(test_data.warning_log, "r", "utf_8") for line in common_log: if "warning" in line.lower(): warnings.append(line) common_log.close() return warnings -def copy_logs(test_img): +def copy_logs(test_data): try: log_dir = os.path.join("..", "..", "Testing","build","test","qa-functional","work","userdir0","var","log") - shutil.copytree(log_dir, Emailer.make_local_path(test_case.output_dir, test_img.image_name, "logs")) + shutil.copytree(log_dir, Emailer.make_local_path(test_case.output_dir, test_data.image_name, "logs")) except Exception as e: - printerror(test_img,"Error: Failed to copy the logs.") - printerror(test_img,str(e) + "\n") + printerror(test_data,"Error: Failed to copy the logs.") + printerror(test_data,str(e) + "\n") logging.warning(traceback.format_exc()) # Clears all the files from a directory and remakes it def clear_dir(dir): @@ -1292,8 +1323,8 @@ def clear_dir(dir): os.makedirs(dir) return True; except Exception as e: - printerror(test_img,"Error: Cannot clear the given directory:") - printerror(test_img,dir + "\n") + printerror(test_data,"Error: Cannot clear the given directory:") + printerror(test_data,dir + "\n") print(str(e)) return False; @@ -1303,8 +1334,8 @@ def del_dir(dir): shutil.rmtree(dir) return True; except: - printerror(test_img,"Error: Cannot delete the given directory:") - printerror(test_img,dir + "\n") + printerror(test_data,"Error: Cannot delete the given directory:") + printerror(test_data,dir + "\n") return False; #Copies a given file from "ffrom" to "to" def copy_file(ffrom, to): @@ -1398,8 +1429,8 @@ class FileNotFoundException(Exception): self.strerror = "FileNotFoundException: " + file def print_error(self): - printerror(test_img,"Error: File could not be found at:") - printerror(test_img,self.file + "\n") + printerror(test_data,"Error: File could not be found at:") + printerror(test_data,self.file + "\n") def error(self): error = "Error: File could not be found at:\n" + self.file + "\n" return error @@ -1412,8 +1443,8 @@ class DirNotFoundException(Exception): self.strerror = "DirNotFoundException: " + dir def print_error(self): - printerror(test_img, "Error: Directory could not be found at:") - printerror(test_img, self.dir + "\n") + printerror(test_data, "Error: Directory could not be found at:") + printerror(test_data, self.dir + "\n") def error(self): error = "Error: Directory could not be found at:\n" + self.dir + "\n" return error @@ -1422,57 +1453,67 @@ class DirNotFoundException(Exception): # Main Testing Functions # ############################# class Test_Runner: + #Executes the tests, makes continuous testing easier + # Identifies the tests to run and runs the tests def execute_test(): global parsed global errorem global failedbool global html global attachl + + # Setup TestData object if(not Emailer.dir_exists(Emailer.make_path("..", "output", "results"))): os.makedirs(Emailer.make_path("..", "output", "results",)) test_case.output_dir = Emailer.make_path("..", "output", "results", time.strftime("%Y.%m.%d-%H.%M.%S")) os.makedirs(test_case.output_dir) test_case.csv = Emailer.make_local_path(test_case.output_dir, "CSV.txt") test_case.html_log = Emailer.make_path(test_case.output_dir, "AutopsyTestCase.html") - test_img = TestData() + test_data = TestData() log_name = test_case.output_dir + "\\regression.log" logging.basicConfig(filename=log_name, level=logging.DEBUG) + + + #Identify tests to run and populate test_case with list # If user wants to do a single file and a list (contradictory?) if test_case.args.single and test_case.args.list: - printerror(test_img, "Error: Cannot run both from config file and on a single file.") + printerror(test_data, "Error: Cannot run both from config file and on a single file.") return # If working from a configuration file if test_case.args.list: if not Emailer.file_exists(test_case.args.config_file): - printerror(test_img, "Error: Configuration file does not exist at:") - printerror(test_img, test_case.args.config_file) + printerror(test_data, "Error: Configuration file does not exist at:") + printerror(test_data, test_case.args.config_file) return - Test_Runner._fill_case_data(test_case.args.config_file,test_img) + Test_Runner._fill_case_data(test_case.args.config_file,test_data) # Else if working on a single file elif test_case.args.single: if not Emailer.file_exists(test_case.args.single_file): - printerror(test_img, "Error: Image file does not exist at:") - printerror(test_img, test_case.args.single_file) + printerror(test_data, "Error: Image file does not exist at:") + printerror(test_data, test_case.args.single_file) return - Test_case.images.append(test_case.args.single_file,) + Test_case.images.append(test_case.args.single_file) + # If user has not selected a single file, and does not want to ignore # the input directory, continue on to parsing ../input if (not test_case.args.single) and (not test_case.args.ignore) and (not test_case.args.list): test_case.args.config_file = "config.xml" if not Emailer.file_exists(test_case.args.config_file): - printerror(test_img, "Error: Configuration file does not exist at:") - printerror(test_img, test_case.args.config_file) + printerror(test_data, "Error: Configuration file does not exist at:") + printerror(test_data, test_case.args.config_file) return - Test_Runner._fill_case_data(test_case.args.config_file, test_img) + Test_Runner._fill_case_data(test_case.args.config_file, test_data) + + # Cycle through images in test_case and run tests logres =[] for img in test_case.images: if Emailer.file_exists(img): - logres.append(Test_Runner._run_ingest(str(img), 0, test_img)) + logres.append(Test_Runner._run_ingest(str(img), 0, test_data)) else: - printerror(test_img, "Warning: Image file listed in configuration does not exist:") - printrttot(value + "\n") - test_img.reset() + printerror(test_data, "Warning: Image file listed in configuration does not exist:") + printerror(value + "\n") + test_data.reset() Reports.write_html_foot() html.close() if (len(logres)>0): @@ -1494,8 +1535,10 @@ class Test_Runner: attachl = [] Emailer.send_email(parsed, errorem, attachl, passFail) - # Iterates through an XML configuration file to find all given elements - def _fill_case_data(config_file, test_img): + # Iterates through an XML configuration file to find all given elements. Populates global test_case object with tests to run + # config_file: Path to the config file + # test_data: TestData object (@@@ Only being passed in for print messages) + def _fill_case_data(config_file, test_data): try: global parsed global errorem @@ -1523,7 +1566,7 @@ class Test_Runner: if Emailer.file_exists(value): test_case.images.append(value) else: - printout(test_img, "File: " + value + " doesn't exist") + printout(test_data, "File: " + value + " doesn't exist") count = len(values) archives = Emailer.make_path(test_case.gold, "..") arcount = 0 @@ -1536,31 +1579,36 @@ class Test_Runner: print("******Alert: There are more gold standards than input images, this will not check all gold Standards.\n") Reports.html_add_images(values) except Exception as e: - printerror(test_img, "Error: There was an error running with the configuration file.") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Error: There was an error running with the configuration file.") + printerror(test_data, str(e) + "\n") logging.critical(traceback.format_exc()) print(traceback.format_exc()) - def _run_ingest(image_file, count, test_img): + # Run autopsy to generate output file and do comparision + # image_file: Path to image (@@@ We should be able to remove this since TestData should have this) + # count: Number of times this image has been tested (@@@ we think we can remove this) + # test_data: TestData object + def _run_ingest(image_file, count, test_data): global parsed global imgfail global failedbool imgfail = False if image_type(image_file) == IMGTYPE.UNKNOWN: - printerror(test_img, "Error: Image type is unrecognized:") - printerror(test_img, image_file + "\n") + printerror(test_data, "Error: Image type is unrecognized:") + printerror(test_data, image_file + "\n") return + # Set the test_case to work for this test - test_img.image_file = image_file - test_img.image_name = test_case.get_image_name(image_file) + "(" + str(count) + ")" - test_img.autopsy_data_file = Emailer.make_path(test_case.output_dir, test_img.image_name, test_img.image_name + "Autopsy_data.txt") - test_img.sorted_data_file = Emailer.make_path(test_case.output_dir, test_img.image_name, "Sorted_Autopsy_data.txt") - test_img.warning_log = Emailer.make_local_path(test_case.output_dir, test_img.image_name, "AutopsyLogs.txt") - test_img.antlog_dir = Emailer.make_local_path(test_case.output_dir, test_img.image_name, "antlog.txt") - test_img.test_dbdump = Emailer.make_path(test_case.output_dir, test_img.image_name, - test_img.image_name + "Dump.txt") - test_img.image = test_case.get_image_name(image_file) + test_data.image_file = image_file + test_data.image_name = test_case.get_image_name(image_file) + "(" + str(count) + ")" + test_data.autopsy_data_file = Emailer.make_path(test_case.output_dir, test_data.image_name, test_data.image_name + "Autopsy_data.txt") + test_data.sorted_data_file = Emailer.make_path(test_case.output_dir, test_data.image_name, "Sorted_Autopsy_data.txt") + test_data.warning_log = Emailer.make_local_path(test_case.output_dir, test_data.image_name, "AutopsyLogs.txt") + test_data.antlog_dir = Emailer.make_local_path(test_case.output_dir, test_data.image_name, "antlog.txt") + test_data.test_dbdump = Emailer.make_path(test_case.output_dir, test_data.image_name, + test_data.image_name + "Dump.txt") + test_data.image = test_case.get_image_name(image_file) if(test_case.args.list): element = parsed.getElementsByTagName("build") if(len(element)<=0): @@ -1577,71 +1625,96 @@ class Test_Runner: test_case.keyword_path = Emailer.make_path(test_case.input_dir, "notablekeywords.xml") test_case.nsrl_path = Emailer.make_path(test_case.input_dir, "nsrl.txt-md5.idx") logging.debug("--------------------") - logging.debug(test_img.image_name) + logging.debug(test_data.image_name) logging.debug("--------------------") - Test_Runner._run_ant(test_img) + Test_Runner._run_ant(test_data) time.sleep(2) # Give everything a second to process - test_case.common_log_path = Emailer.make_local_path(test_case.output_dir, test_img.image_name, test_img.image_name+test_case.common_log) - # After the java has ran: - Database.get_Data(test_img) - copy_logs(test_img) - test_img.sorted_log = Emailer.make_local_path(test_case.output_dir, test_img.image_name, test_img.image_name + "SortedErrors.txt") - Logs.generate_log_data(test_img) - logres = Logs.search_common_log("TskCoreException", test_img) - # If NOT keeping Solr index (-k) + + + # Autopsy has finished running, we will now process the results + test_case.common_log_path = Emailer.make_local_path(test_case.output_dir, test_data.image_name, test_data.image_name+test_case.common_log) + + # get data for the follow on db diff + Database.get_Data(test_data) + + # merges logs into a single log for later diff + copy_logs(test_data) + test_data.sorted_log = Emailer.make_local_path(test_case.output_dir, test_data.image_name, test_data.image_name + "SortedErrors.txt") + Logs.generate_log_data(test_data) + + # Look for existance of core exceptions (@@@ Should be moved to differ class) + logres = Logs.search_common_log("TskCoreException", test_data) + + # Cleanup SOLR: If NOT keeping Solr index (-k) if not test_case.args.keep: - solr_index = Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "ModuleOutput", "KeywordSearch") + solr_index = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "ModuleOutput", "KeywordSearch") if clear_dir(solr_index): - print_report(test_img, [], "DELETE SOLR INDEX", "Solr index deleted.") + print_report(test_data, [], "DELETE SOLR INDEX", "Solr index deleted.") elif test_case.args.keep: - print_report(test_img, [], "KEEP SOLR INDEX", "Solr index has been kept.") - # If running in exception mode (-e) + print_report(test_data, [], "KEEP SOLR INDEX", "Solr index has been kept.") + + # If running in exception mode, print exceptions to log if test_case.args.exception: - exceptions = search_logs(test_case.args.exception_string, test_img) + exceptions = search_logs(test_case.args.exception_string, test_data) okay = "No warnings or exceptions found containing text '" + test_case.args.exception_string + "'." - print_report(test_img, exceptions, "EXCEPTION", okay) - database = Database(test_img) - # Now test in comparison to the gold standards + print_report(test_data, exceptions, "EXCEPTION", okay) + + + database = Database(test_data) + + # Now either diff or rebuild if not test_case.args.rebuild: - Test_Runner._run_test(image_file, database, test_img) + Test_Runner._run_test(image_file, database, test_data) # If running in rebuild mode (-r) else: - Test_Runner.rebuild(test_img) + Test_Runner.rebuild(test_data) + + # @@@ COnsider if we want to do this for a rebuild. # Make the CSV log and the html log viewer - Reports.generate_reports(test_case.csv, database, test_img) + Reports.generate_reports(test_case.csv, database, test_data) # Reset the test_case and return the tests sucessfully finished - clear_dir(Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "ModuleOutput", "keywordsearch")) + clear_dir(Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "ModuleOutput", "keywordsearch")) if(failedbool): attachl.append(test_case.common_log_path) test_case.reset() return logres - # Runs the test on the single given file. - # The path must be guarenteed to be a correct path. - def _run_test(image_file, database, test_img): + # Compares results for a single test. Autopsy has already been run. + # This should all be moved into TestDiffer.run_diff() + # image_file: Path to image (@@@ Shoudl go away in favor of test_data) + # database: Database object created based on test_data + # test_data: TestData object + def _run_test(image_file, database, test_data): try: gold_path = test_case.gold - img_gold = Emailer.make_path(test_case.gold, "tmp", test_img.image_name) - img_archive = Emailer.make_path("..", "output", "gold", test_img.image_name+"-archive.zip") + # Tmp location to extract ZIP file into + img_gold = Emailer.make_path(test_case.gold, "tmp", test_data.image_name) + + # Open gold archive file + img_archive = Emailer.make_path("..", "output", "gold", test_data.image_name+"-archive.zip") if(not Emailer.file_exists(img_archive)): - img_archive = Emailer.make_path(test_case.gold_parse, test_img.image_name+"-archive.zip") + img_archive = Emailer.make_path(test_case.gold_parse, test_data.image_name+"-archive.zip") gold_path = test_case.gold_parse - img_gold = Emailer.make_path(gold_path, "tmp", test_img.image_name) + img_gold = Emailer.make_path(gold_path, "tmp", test_data.image_name) extrctr = zipfile.ZipFile(img_archive, 'r', compression=zipfile.ZIP_DEFLATED) extrctr.extractall(gold_path) extrctr.close time.sleep(2) - Database.compare_to_gold_db(test_img, database) - TestDiffer.run_diff(test_img) + + # Compare database count to gold (@@@ This should all be done as part of TestDiffer.run_diff()) + Database.compare_to_gold_db(test_data, database) + + # Compare other data to gold + TestDiffer.run_diff(test_data) del_dir(img_gold) except Exception as e: - printerror(test_img, "Tests failed due to an error, try rebuilding or creating gold standards.\n") - printerror(test_img, str(e) + "\n") + printerror(test_data, "Tests failed due to an error, try rebuilding or creating gold standards.\n") + printerror(test_data, str(e) + "\n") print(traceback.format_exc()) # Rebuilds the gold standards by copying the test-generated database - # and html report files into the gold directory - def rebuild(test_img): + # and html report files into the gold directory. Autopsy has already been run + def rebuild(test_data): # Errors to print errors = [] if(test_case.gold_parse == "" ): @@ -1650,12 +1723,12 @@ class Test_Runner: # Delete the current gold standards gold_dir = test_case.img_gold_parse clear_dir(test_case.img_gold_parse) - tmpdir = Emailer.make_path(gold_dir, test_img.image_name) - dbinpth = Emailer.make_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, test_case.test_db_file) + tmpdir = Emailer.make_path(gold_dir, test_data.image_name) + dbinpth = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, test_case.test_db_file) dboutpth = Emailer.make_path(tmpdir, test_case.test_db_file) - dataoutpth = Emailer.make_path(tmpdir, test_img.image_name + "SortedData.txt") - dbdumpinpth = test_img.test_dbdump - dbdumpoutpth = Emailer.make_path(tmpdir, test_img.image_name + "DBDump.txt") + dataoutpth = Emailer.make_path(tmpdir, test_data.image_name + "SortedData.txt") + dbdumpinpth = test_data.test_dbdump + dbdumpoutpth = Emailer.make_path(tmpdir, test_data.image_name + "DBDump.txt") if not os.path.exists(test_case.img_gold_parse): os.makedirs(test_case.img_gold_parse) if not os.path.exists(gold_dir): @@ -1664,23 +1737,23 @@ class Test_Runner: os.makedirs(tmpdir) try: copy_file(dbinpth, dboutpth) - if Emailer.file_exists(test_img.sorted_data_file): - copy_file(test_img.sorted_data_file, dataoutpth) + if Emailer.file_exists(test_data.sorted_data_file): + copy_file(test_data.sorted_data_file, dataoutpth) copy_file(dbdumpinpth, dbdumpoutpth) - error_pth = Emailer.make_path(tmpdir, test_img.image_name+"SortedErrors.txt") - copy_file(test_img.sorted_log, error_pth) + error_pth = Emailer.make_path(tmpdir, test_data.image_name+"SortedErrors.txt") + copy_file(test_data.sorted_log, error_pth) except Exception as e: - printerror(test_img, str(e)) + printerror(test_data, str(e)) print(str(e)) print(traceback.format_exc()) # Rebuild the HTML report htmlfolder = "" - for fs in os.listdir(os.path.join(os.getcwd(),test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports")): - if os.path.isdir(os.path.join(os.getcwd(), test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports", fs)): + for fs in os.listdir(os.path.join(os.getcwd(),test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports")): + if os.path.isdir(os.path.join(os.getcwd(), test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports", fs)): htmlfolder = fs - autopsy_html_path = Emailer.make_local_path(test_case.output_dir, test_img.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder) + autopsy_html_path = Emailer.make_local_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports", htmlfolder) - html_path = Emailer.make_path(test_case.output_dir, test_img.image_name, + html_path = Emailer.make_path(test_case.output_dir, test_data.image_name, test_case.Img_Test_Folder, "Reports") try: if not os.path.exists(Emailer.make_path(tmpdir, htmlfolder)): @@ -1699,14 +1772,14 @@ class Test_Runner: os.chdir(zpdir) os.chdir("..") img_gold = "tmp" - img_archive = Emailer.make_path(test_img.image_name+"-archive.zip") + img_archive = Emailer.make_path(test_data.image_name+"-archive.zip") comprssr = zipfile.ZipFile(img_archive, 'w',compression=zipfile.ZIP_DEFLATED) Test_Runner.zipdir(img_gold, comprssr) comprssr.close() os.chdir(oldcwd) del_dir(test_case.img_gold_parse) okay = "Sucessfully rebuilt all gold standards." - print_report(test_img, errors, "REBUILDING", okay) + print_report(test_data, errors, "REBUILDING", okay) def zipdir(path, zip): for root, dirs, files in os.walk(path): @@ -1715,9 +1788,9 @@ class Test_Runner: # Tests Autopsy with RegressionTest.java by by running # the build.xml file through ant - def _run_ant(test_img): + def _run_ant(test_data): # Set up the directories - test_case_path = os.path.join(test_case.output_dir, test_img.image_name) + test_case_path = os.path.join(test_case.output_dir, test_data.image_name) if Emailer.dir_exists(test_case_path): shutil.rmtree(test_case_path) os.makedirs(test_case_path) @@ -1728,19 +1801,19 @@ class Test_Runner: test_case.ant.append(os.path.join("..","..","Testing","build.xml")) test_case.ant.append("regression-test") test_case.ant.append("-l") - test_case.ant.append(test_img.antlog_dir) - test_case.ant.append("-Dimg_path=" + test_img.image_file) + test_case.ant.append(test_data.antlog_dir) + test_case.ant.append("-Dimg_path=" + test_data.image_file) test_case.ant.append("-Dknown_bad_path=" + test_case.known_bad_path) test_case.ant.append("-Dkeyword_path=" + test_case.keyword_path) test_case.ant.append("-Dnsrl_path=" + test_case.nsrl_path) test_case.ant.append("-Dgold_path=" + Emailer.make_path(test_case.gold)) - test_case.ant.append("-Dout_path=" + Emailer.make_local_path(test_case.output_dir, test_img.image_name)) + test_case.ant.append("-Dout_path=" + Emailer.make_local_path(test_case.output_dir, test_data.image_name)) test_case.ant.append("-Dignore_unalloc=" + "%s" % test_case.args.unallocated) test_case.ant.append("-Dtest.timeout=" + str(test_case.timeout)) - printout(test_img, "Ingesting Image:\n" + test_img.image_file + "\n") - printout(test_img, "CMD: " + " ".join(test_case.ant)) - printout(test_img, "Starting test...\n") + printout(test_data, "Ingesting Image:\n" + test_data.image_file + "\n") + printout(test_data, "CMD: " + " ".join(test_case.ant)) + printout(test_data, "Starting test...\n") antoutpth = Emailer.make_local_path(test_case.output_dir, "antRunOutput.txt") antout = open(antoutpth, "a") if SYS is OS.CYGWIN:
Image Path:" + test_img.image_file + "
" + test_data.image_file + "
Image Name:" + test_img.image_name + "
" + test_data.image_name + "
test_case Output Directory:" + test_case.output_dir + "
Autopsy Version:" + test_case.autopsy_version + "
Heap Space:" + test_img.heap_space + "
" + test_data.heap_space + "
Test Start Date:" + test_img.start_date + "
" + test_data.start_date + "
Test End Date:" + test_img.end_date + "
" + test_data.end_date + "
Total Test Time:" + test_img.total_test_time + "
" + test_data.total_test_time + "
Total Ingest Time:" + test_img.total_ingest_time + "
" + test_data.total_ingest_time + "
Exceptions Count:" + str(len(get_exceptions(test_img))) + "
" + str(len(get_exceptions(test_data))) + "
Autopsy OutOfMemoryExceptions:" + str(len(search_logs("OutOfMemoryException", test_img))) + "
" + str(len(search_logs("OutOfMemoryException", test_data))) + "
Autopsy OutOfMemoryErrors:" + str(len(search_logs("OutOfMemoryError", test_img))) + "
" + str(len(search_logs("OutOfMemoryError", test_data))) + "
Tika OutOfMemoryErrors/Exceptions:" + str(Reports._get_num_memory_errors("tika", test_img)) + "
" + str(Reports._get_num_memory_errors("tika", test_data)) + "
Solr OutOfMemoryErrors/Exceptions:" + str(Reports._get_num_memory_errors("solr", test_img)) + "
" + str(Reports._get_num_memory_errors("solr", test_data)) + "
TskCoreExceptions:" + str(len(search_log_set("autopsy", "TskCoreException", test_img))) + "
" + str(len(search_log_set("autopsy", "TskCoreException", test_data))) + "
TskDataExceptions:" + str(len(search_log_set("autopsy", "TskDataException", test_img))) + "
" + str(len(search_log_set("autopsy", "TskDataException", test_data))) + "
Ingest Messages Count:" + str(test_case.ingest_messages) + "
Indexed Files Count:" + str(test_case.indexed_chunks) + "
Out Of Disk Space:\

(will skew other test results)

" + str(len(search_log_set("autopsy", "Stopping ingest due to low disk space on disk", test_img))) + "
" + str(len(search_log_set("autopsy", "Stopping ingest due to low disk space on disk", test_data))) + "
TSK Objects Count:" + str(database.autopsy_objects) + "
Artifacts Count: