From 52fba3c8171abd8d5671d4524923bb8d795a4847 Mon Sep 17 00:00:00 2001 From: Jeff Wallace Date: Wed, 10 Jul 2013 09:43:03 -0400 Subject: [PATCH 1/6] Changed several filename attributes of TestConfiguration to be constant global variables --- test/script/regression.py | 46 +++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 8074516425..08e3997f2b 100644 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -92,6 +92,14 @@ DB_FILENAME = "autopsy.db" # Backup database filename BACKUP_DB_FILENAME = "autopsy_backup.db" +# TODO: Double check this purpose statement +# Folder name for gold standard database testing +IMG_TEST_FOLDER = "AutopsyTestCase" + +# TODO: Double check this purpose statement +# The filename of the log to store error messages +COMMON_LOG = "AutopsyErrors.txt" + Day = 0 #-------------------------------------------------------------# # Parses argv and stores booleans to match command line input # @@ -188,9 +196,6 @@ class TestConfiguration: self.input_dir = Emailer.make_local_path("..","input") # Path : the Path to the input directory self.gold = Emailer.make_path("..", "output", "gold") # Path : the Path to the gold directory self.img_gold = Emailer.make_path(self.gold, 'tmp') - self.common_log = "AutopsyErrors.txt" - self.test_db_file = "autopsy.db" - self.Img_Test_Folder = "AutopsyTestCase" # Logs: self.csv = "" self.global_csv = "" @@ -835,10 +840,10 @@ class TestResultsDiffer: def _compare_to_gold_html(test_data): gold_html_file = Emailer.make_path(test_config.img_gold, test_data.image_name, "Report", "index.html") htmlfolder = "" - for fs in os.listdir(Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports")): - if os.path.isdir(Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports", fs)): + for fs in os.listdir(Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports")): + if os.path.isdir(Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports", fs)): htmlfolder = fs - autopsy_html_path = Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports", htmlfolder, "HTML Report") + autopsy_html_path = Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports", htmlfolder, "HTML Report") try: @@ -853,9 +858,9 @@ class TestResultsDiffer: return #Find all gold .html files belonging to this test_config ListGoldHTML = [] - for fs in os.listdir(Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports", htmlfolder)): + for fs in os.listdir(Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports", htmlfolder)): if(fs.endswith(".html")): - ListGoldHTML.append(Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports", htmlfolder, fs)) + ListGoldHTML.append(Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports", htmlfolder, fs)) #Find all new .html files belonging to this test_config ListNewHTML = [] if(os.path.exists(Emailer.make_path(test_config.img_gold, test_data.image_name))): @@ -973,9 +978,9 @@ class TestData: if(db_type == DBType.GOLD): db_path = Emailer.make_path(self.main_config.img_gold, self.image_name, DB_FILENAME) elif(db_type == DBType.OUTPUT): - db_path = Emailer.make_path(self.main_config.output_dir, self.image_name, self.main_config.Img_Test_Folder, DB_FILENAME) + db_path = Emailer.make_path(self.main_config.output_dir, self.image_name, IMG_TEST_FOLDER, DB_FILENAME) else: - db_path = Emailer.make_path(self.main_config.output_dir, self.image_name, self.main_config.Img_Test_Folder, BACKUP_DB_FILENAME) + db_path = Emailer.make_path(self.main_config.output_dir, self.image_name, IMG_TEST_FOLDER, BACKUP_DB_FILENAME) return db_path class Reports: @@ -1193,7 +1198,7 @@ class Reports: vars.append( str(database.autopsy_objects) ) vars.append( str(database.get_artifacts_count()) ) vars.append( str(database.autopsy_attributes) ) - vars.append( Emailer.make_local_path("gold", test_data.image_name, test_config.test_db_file) ) + vars.append( Emailer.make_local_path("gold", test_data.image_name, DB_FILENAME) ) vars.append( database.get_artifact_comparison() ) vars.append( database.get_attribute_comparison() ) vars.append( Emailer.make_local_path("gold", test_data.image_name, "standard.html") ) @@ -1722,7 +1727,7 @@ class TestRunner: test_data.warning_log = Emailer.make_local_path(output_path, "AutopsyLogs.txt") test_data.antlog_dir = Emailer.make_local_path(output_path, "antlog.txt") test_data.test_dbdump = Emailer.make_path(output_path, test_data.image_name + "Dump.txt") - test_data.common_log_path = Emailer.make_local_path(output_path, test_data.image_name + test_config.common_log) + test_data.common_log_path = Emailer.make_local_path(output_path, test_data.image_name + COMMON_LOG) test_data.sorted_log = Emailer.make_local_path(output_path, test_data.image_name + "SortedErrors.txt") test_data_list.append(test_data) return test_data_list @@ -1784,7 +1789,7 @@ class TestRunner: # Make the CSV log and the html log viewer Reports.generate_reports(test_config.csv, databaseDiff, test_data) # Reset the test_config and return the tests sucessfully finished - clear_dir(Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "ModuleOutput", "keywordsearch")) + clear_dir(Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "ModuleOutput", "keywordsearch")) if(failedbool): attachl.append(test_data.common_log_path) return logres @@ -1795,7 +1800,7 @@ class TestRunner: Clean up SOLR index if in keep mode (-k) """ if not test_config.args.keep: - solr_index = Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "ModuleOutput", "KeywordSearch") + solr_index = Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "ModuleOutput", "KeywordSearch") if clear_dir(solr_index): print_report(test_data, [], "DELETE SOLR INDEX", "Solr index deleted.") else: @@ -1820,8 +1825,8 @@ class TestRunner: gold_dir = test_config.img_gold clear_dir(test_config.img_gold) tmpdir = Emailer.make_path(gold_dir, test_data.image_name) - dbinpth = Emailer.make_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, test_case.test_db_file) - dboutpth = Emailer.make_path(tmpdir, test_config.test_db_file) + dbinpth = Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, test_case.test_db_file) + dboutpth = Emailer.make_path(tmpdir, DB_FILENAME) dataoutpth = Emailer.make_path(tmpdir, test_data.image_name + "SortedData.txt") dbdumpinpth = test_data.test_dbdump dbdumpoutpth = Emailer.make_path(tmpdir, test_data.image_name + "DBDump.txt") @@ -1844,13 +1849,13 @@ class TestRunner: print(traceback.format_exc()) # Rebuild the HTML report htmlfolder = "" - for fs in os.listdir(os.path.join(os.getcwd(),test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports")): - if os.path.isdir(os.path.join(os.getcwd(), test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports", fs)): + for fs in os.listdir(os.path.join(os.getcwd(),test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports")): + if os.path.isdir(os.path.join(os.getcwd(), test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports", fs)): htmlfolder = fs - autopsy_html_path = Emailer.make_local_path(test_config.output_dir, test_data.image_name, test_config.Img_Test_Folder, "Reports", htmlfolder) + autopsy_html_path = Emailer.make_local_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports", htmlfolder) html_path = Emailer.make_path(test_config.output_dir, test_data.image_name, - test_config.Img_Test_Folder, "Reports") + IMG_TEST_FOLDER, "Reports") try: if not os.path.exists(Emailer.make_path(tmpdir, htmlfolder)): os.makedirs(Emailer.make_path(tmpdir, htmlfolder)) @@ -1965,7 +1970,6 @@ class OS: if __name__ == "__main__": global SYS - print(DBType.OUTPUT) if _platform == "linux" or _platform == "linux2": SYS = OS.LINUX elif _platform == "darwin": From d8136f5754625f3eaee688b8f3080170e70ed50a Mon Sep 17 00:00:00 2001 From: Jeff Wallace Date: Wed, 10 Jul 2013 14:08:10 -0400 Subject: [PATCH 2/6] Changed documentation to google python standards. --- test/script/regression.py | 573 ++++++++++++++++++++++++++------------ 1 file changed, 402 insertions(+), 171 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 08e3997f2b..a9a9419d7e 100644 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -68,12 +68,13 @@ import srcupdater # Data Definitions: # -# Path: A path to a file or directory. +# pathto_X: A path to type X. # ConfigFile: An XML file formatted according to the template in myconfig.xml # ParsedConfig: A dom object that represents a ConfigFile # SQLCursor: A cursor recieved from a connection to an SQL database # Nat: A Natural Number - +# Image: An image +# ##### # Enumeration definition (python 3.2 doesn't have enumerations, this is a common solution @@ -104,7 +105,23 @@ Day = 0 #-------------------------------------------------------------# # Parses argv and stores booleans to match command line input # #-------------------------------------------------------------# -class Args: +class Args(object): + """A container for command line options and arguments. + + Attributes: + single: a boolean indicating whether to run in single file mode + single_file: an Image to run the test on + rebuild: a boolean indicating whether to run in rebuild mode + list: a boolean indicating a config file was specified + unallocated: a boolean indicating unallocated space should be ignored + ignore: a boolean indicating the input directory should be ingnored + keep: a boolean indicating whether to keep the SOLR index + verbose: a boolean indicating whether verbose output should be printed + exeception: a boolean indicating whether errors containing exception + exception_string should be printed + exception_sring: a String representing and exception name + fr: a boolean indicating whether gold standard images will be downloaded + """ def __init__(self): self.single = False self.single_file = "" @@ -184,20 +201,49 @@ class Args: return True -class TestConfiguration: - """ +class TestConfiguration(object): + """Container for test configuration data. + The Master Test Configuration. Encapsulates consolidated high level input from config XML file and command-line arguments. + + Attributes: + args: an Args, the command line arguments + output_dir: a pathto_Dir, the output directory + input_dir: a pathto_Dir, the input directory + gold: a pathto_Dir, the gold directory + img_gold: a pathto_Dir, the temp directory where gold images are unzipped to + csv: a pathto_File, the local csv file + global_csv: a pathto_File, the global csv file + html_log: a pathto_File + known_bad_path: + keyword_path: + nsrl_path: + build_path: a pathto_File, the ant build file which runs the tests + autopsy_version: + ingest_messages: a Nat, number of ingest messages + indexed_files: a Nat, the number of indexed files + indexed_chunks: a Nat, the number of indexed chunks + timer: + images: a listof_Image, the images to be tested + timeout: a Nat, the amount of time before killing the test + ant: a listof_String, the ant command to run the tests """ + def __init__(self, args): - self.args = args # Args : the command line arguments + """Inits TestConfiguration and loads a config file if available. + + Args: + args: an Args, the command line arguments. + """ + self.args = args # Paths: - self.output_dir = "" # Path : the path to the output directory - self.input_dir = Emailer.make_local_path("..","input") # Path : the Path to the input directory - self.gold = Emailer.make_path("..", "output", "gold") # Path : the Path to the gold directory - self.img_gold = Emailer.make_path(self.gold, 'tmp') + self.output_dir = "" + self.input_dir = Emailer.make_local_path("..","input") + self.gold = Emailer.make_path("..", "output", "gold") + self.img_gold = Emailer.make_path(self.gold, 'tmp') # Logs: - self.csv = "" + self.csv = "" self.global_csv = "" self.html_log = "" # Ant info: @@ -225,23 +271,6 @@ class TestConfiguration: self._init_imgs() self._init_build_info() - def get_image_name(self, image_file): - path_end = image_file.rfind("/") - path_end2 = image_file.rfind("\\") - ext_start = image_file.rfind(".") - if(ext_start == -1): - name = image_file - if(path_end2 != -1): - name = image_file[path_end2+1:ext_start] - elif(ext_start == -1): - name = image_file[path_end+1:] - elif(path_end == -1): - name = image_file[:ext_start] - elif(path_end!=-1 and ext_start!=-1): - name = image_file[path_end+1:ext_start] - else: - name = image_file[path_end2+1:ext_start] - return name def ant_to_string(self): string = "" @@ -258,9 +287,7 @@ class TestConfiguration: self.ant = [] def _init_imgs(self): - """ - Initialize the list of images to run test on. - """ + """Initialize the list of images to run test on.""" #Identify tests to run and populate test_config with list # If user wants to do a single file and a list (contradictory?) if self.args.single and self.args.list: @@ -293,9 +320,7 @@ class TestConfiguration: self._load_config_file(self.args.config_file) def _init_logs(self): - """ - Setup output folder, logs, and reporting infrastructure - """ + """Setup output folder, logs, and reporting infrastructure.""" if(not Emailer.dir_exists(Emailer.make_path("..", "output", "results"))): os.makedirs(Emailer.make_path("..", "output", "results",)) self.output_dir = Emailer.make_path("..", "output", "results", time.strftime("%Y.%m.%d-%H.%M.%S")) @@ -306,9 +331,7 @@ class TestConfiguration: logging.basicConfig(filename=log_name, level=logging.DEBUG) def _init_build_info(self): - """ - Initializes paths that point to information necessary to run the AutopsyIngest - """ + """Initializes paths that point to information necessary to run the AutopsyIngest.""" global parsed if(self.args.list): build_elements = parsed.getElementsByTagName("build") @@ -326,11 +349,14 @@ class TestConfiguration: self.keyword_path = Emailer.make_path(self.input_dir, "notablekeywords.xml") self.nsrl_path = Emailer.make_path(self.input_dir, "nsrl.txt-md5.idx") - # ConfigFile -> void def _load_config_file(self, config_file): - """ + """Updates this TestConfiguration's attributes from the config file. + Initializes this TestConfiguration by iterating through the XML config file command-line argument. Populates self.images and optional email configuration + + Args: + config_file: ConfigFile - the configuration file to load """ try: global parsed @@ -379,23 +405,42 @@ class TestConfiguration: logging.critical(traceback.format_exc()) print(traceback.format_exc()) - # String -> void def _print_error(self, msg): - """ - Append the given error message to the global error message and print the message to the screen. + """Append the given error message to the global error message and print the message to the screen. + + Args: + msg: String - the error message to print """ global errorem error_msg = "Configuration: " + msg print(error_msg) errorem += error_msg + "\n" -class TskDbDiff: - """ +class TskDbDiff(object): + """Represents the differences between the gold and output databases. + Contains methods to compare two databases and internally store some of the results + + Attributes: + gold_artifacts: + autopsy_artifacts: + gold_attributes: + autopsy_attributes: + gold_objects: + autopsy_objects: + artifact_comparison: + attribute_comparision: + test_data: + autopsy_db_file: + gold_db_file: """ - # TskDbDiff(TestData) def __init__(self, test_data): + """Constructor for TskDbDiff. + + Args: + test_data: TestData - the test data to compare + """ self.gold_artifacts = [] self.autopsy_artifacts = [] self.gold_attributes = 0 @@ -449,10 +494,14 @@ class TskDbDiff: list.append(error) return ";".join(list) - # SQLCursor -> listof_Artifact def _count_artifacts(self, cursor): - """ - Get a list of artifacts from the given SQLCursor + """Get a list of artifacts from the given SQLCursor. + + Args: + cursor: SQLCursor - the cursor to execute on + + Returns: + listof_Artifact - the artifacts found by the query """ cursor.execute("SELECT COUNT(*) FROM blackboard_artifact_types") length = cursor.fetchone()[0] + 1 @@ -462,24 +511,32 @@ class TskDbDiff: artifacts.append(cursor.fetchone()[0]) return artifacts - # SQLCursor -> Nat def _count_attributes(self, cursor): - """ - Count the attributes from the given SQLCursor + """Count the attributes from the given SQLCursor. + + Args: + cursor: SQLCursor - the cursor to execute on + + Returns: + Nat - the number of attributes found by the query """ cursor.execute("SELECT COUNT(*) FROM blackboard_attributes") return cursor.fetchone()[0] - # SQLCursor -> Nat def _count_objects(self, cursor): - """ - Count the objects from the given SQLCursor + """Count the objects from the given SQLCursor. + + Args: + cursor: SQLCursor - the cursor to execute on + + Returns: + Nat - the number of objects found by the query """ cursor.execute("SELECT COUNT(*) FROM tsk_objects") return cursor.fetchone()[0] - # Compares the blackboard artifact counts of two databases def _compare_bb_artifacts(self): + """Compares the blackboard artifact counts of two databases.""" exceptions = [] try: global failedbool @@ -503,9 +560,8 @@ class TskDbDiff: exceptions.append("Error: Unable to compare blackboard_artifacts.\n") return exceptions - # Compares the blackboard atribute counts of two databases - # given the two database cursors def _compare_bb_attributes(self): + """Compares the blackboard attribute counts of two databases.""" exceptions = [] try: if self.gold_attributes != self.autopsy_attributes: @@ -523,9 +579,8 @@ class TskDbDiff: exceptions.append("Error: Unable to compare blackboard_attributes.\n") return exceptions - # Compares the tsk object counts of two databases - # given the two database cursors def _compare_tsk_objects(self): + """Compares the TSK object counts of two databases.""" exceptions = [] try: if self.gold_objects != self.autopsy_objects: @@ -543,10 +598,16 @@ class TskDbDiff: exceptions.append("Error: Unable to compare tsk_objects.\n") return exceptions - # SQLCursor x SQLCursor -> void def _get_basic_counts(self, autopsy_cur, gold_cur): - """ - Get the counts of objects, artifacts, and attributes in the Gold and Ouput databases. + """Count the items necessary to compare the databases. + + Gets the counts of objects, artifacts, and attributes in the Gold + and Ouput databases and updates this TskDbDiff's attributes + accordingly + + Args: + autopsy_cur: SQLCursor - the cursor for the output database + gold_cur: SQLCursor - the cursor for the gold database """ try: # Objects @@ -562,8 +623,9 @@ class TskDbDiff: printerror(self.test_data, "Way out:" + str(e)) def compare_basic_counts(self): - """ - Basic test between output and gold databases. Compares only counts of objects and blackboard items. + """Basic test between output and gold databases. + + Compares only counts of objects and blackboard items. Note: SQLITE needs unix style pathing """ # Check to make sure both db files exist @@ -605,10 +667,17 @@ class TskDbDiff: print_report(self.test_data, exceptions[2], "COMPARE ATTRIBUTES", okay) - # smart method that deals with blackboard comparison to avoid issues with different IDs based on when artifacts were created. - # Dumps sorted text results to output location stored in test_data. - # autopsy_db_file: Output database file def _dump_output_db_bb(autopsy_con, autopsy_db_file, test_data): + """Dumps sorted text results to the output location stored in test_data. + + Smart method that deals with a blackboard comparison to avoid issues + with different IDs based on when artifacts were created. + + Args: + autopsy_con: a SQLConn to the autopsy database. + autopsy_db_file: a pathto_File, the output database. + test_data: the TestData that corresponds with this dump. + """ autopsy_cur2 = autopsy_con.cursor() global errorem global attachl @@ -699,8 +768,14 @@ class TskDbDiff: except Exception as e: printerror(test_data, 'outer exception: ' + str(e)) - # Dumps a database (minus the artifact and attributes) to a text file. def _dump_output_db_nonbb(test_data): + """Dumps a database to a text file. + + Does not dump the artifact and attributes. + + Args: + test_data: the TestData that corresponds with this dump. + """ # Make a copy of the DB autopsy_db_file = test_data.getDBPath(DBType.OUTPUT) backup_db_file = test_data.getDBPath(DBType.BACKUP) @@ -723,8 +798,12 @@ class TskDbDiff: printerror(test_data, "dump_output_db_nonbb: Outer dump Exception:" + str(e)) - # Dumps the given database to text files for later comparison def dump_output_db(test_data): + """Dumps the given database to text files for later comparison. + + Args: + test_data: the TestData that corresponds to this dump. + """ autopsy_db_file = test_data.getDBPath(DBType.OUTPUT) autopsy_con = sqlite3.connect(autopsy_db_file) autopsy_cur = autopsy_con.cursor() @@ -738,12 +817,17 @@ class TskDbDiff: #-------------------------------------------------# # Functions relating to comparing outputs # #-------------------------------------------------# -class TestResultsDiffer: +class TestResultsDiffer(object): + """Compares results for a single test. + """ - # Compares results for a single test. Autopsy has already been run. - # test_data: TestData object - # databaseDiff: TskDbDiff object created based on test_data def run_diff(test_data, databaseDiff): + """Compares results for a single test. + + Args: + test_data: the TestData to use. + databaseDiff: TskDbDiff object created based off test_data + """ try: gold_path = test_config.gold # Tmp location to extract ZIP file into @@ -781,14 +865,17 @@ class TestResultsDiffer: - # @@@ _compare_text could be made more generic with how it forms the paths (i.e. not add ".txt" in the method) and probably merged with + # TODO: _compare_text could be made more generic with how it forms the paths (i.e. not add ".txt" in the method) and probably merged with # compare_errors since they both do basic comparison of text files - # Compares two text files - # output_file: output text file - # gold_file: gold text file - # test_data: Test being performed def _compare_text(output_file, gold_file, test_data): + """Compare two text files. + + Args: + output_file: a pathto_File, the output text file + gold_file: a pathto_File, the input text file + test_data: the TestData of the test being performed + """ gold_dir = Emailer.make_path(test_config.img_gold, test_data.image_name, test_data.image_name + gold_file + ".txt") if(not Emailer.file_exists(output_file)): return @@ -811,8 +898,12 @@ class TestResultsDiffer: global imgfail imgfail = True - # Compare merged error log files def _compare_errors(test_data): + """Compare merged error log files. + + Args: + test_data: the TestData of the test being performed + """ gold_dir = Emailer.make_path(test_config.img_gold, test_data.image_name, test_data.image_name + "SortedErrors.txt") common_log = codecs.open(test_data.sorted_log, "r", "utf_8") gold_log = codecs.open(gold_dir, "r", "utf_8") @@ -835,9 +926,12 @@ class TestResultsDiffer: global imgfail imgfail = True - # Compare the html report file made by - # the regression test against the gold standard html report def _compare_to_gold_html(test_data): + """Compare the output and gold html reports. + + Args: + test_data: the TestData of the test being performed. + """ gold_html_file = Emailer.make_path(test_config.img_gold, test_data.image_name, "Report", "index.html") htmlfolder = "" for fs in os.listdir(Emailer.make_path(test_config.output_dir, test_data.image_name, IMG_TEST_FOLDER, "Reports")): @@ -895,9 +989,18 @@ class TestResultsDiffer: printerror(test_data, str(e) + "\n") logging.critical(traceback.format_exc()) - # Compares file a to file b and any differences are returned - # Only works with report html files, as it searches for the first