diff --git a/Testing/script/regression.py b/Testing/script/regression.py
index f127695f06..a515a42090 100644
--- a/Testing/script/regression.py
+++ b/Testing/script/regression.py
@@ -7,455 +7,1035 @@ import subprocess
import os.path
import shutil
import time
+import datetime
import xml
-#import thread
-#import wmi
-# import win32gui
from xml.dom.minidom import parse, parseString
-
-
-# Last modified 7/25/12 @2:30pm
-# Usage: ./regression.py [-f FILE] OR [-l CONFIG] [OPTIONS]
-# Run the RegressionTest.java file, and compare the result with a gold standard
-# When the -f flag is set, this script only tests the image given by FILE.
-# An indexed NSRL database is expected at ./input/nsrl.txt-md5.idx,
-# and an indexed notable hash database at ./input/notablehashes.txt-md5.idx
-# In addition, any keywords to search for must be in ./input/notablekeywords.xml
-# When the -l flag is set, the script looks for a config.xml file of the given name
-# where images are stored. The above input files can be outsourced to different locations
-# from the config.xml. For usage notes please see the example "config.xml" in
-# the /script folder.
-# Options:
-# -r, --rebuild Rebuild the gold standards from the test results for each image
-# -i, --ignore Ignores the ./input directory when searching for files
-# -u, --unallocated Ignores unallocated space when ingesting. Faster, but less accurate results.
-# -d, --delete Disables the deletion of Solr indexing directory generated by Ingest. Uses more disk space..
-# -v, --verbose Prints logged warnings after each ingest
-# -e, --exception When followed by a string, will only print out the exceptions that occured that contain the string. Case sensitive.
-
-hadErrors = False # If any of the tests failed
-results = {} # Dictionary in which to store map ({imgname}->errors)
-goldDir = "gold" # Directory for gold standards (files should be ./gold/{imgname}/standard.db)
-inDir = "input" # Image files, hash dbs, and keywords.
-# Results will be in ./output/{datetime}/{imgname}/
-outDir = os.path.join("output",time.strftime("%Y.%m.%d-%H.%M"))
-
-CommonLog = ""
-
-
-
-# def AutopsyCrash(image, ignoreUnalloc, list):
- # cwd = wgetcwd()
- # x = 20 #seconds to wait between passes
- # name = imageName(image, ignoreUnalloc, list)
- # TestFolder = os.path.join(cwd, outDir, name, "AutopsyTestCase")
- # y = True #default return of 'Crashed'
- # i = 0 #number of passes to run
- # while(i < 2):
- # print "Sleeping background process for %s seconds" %(str(x))
- # time.sleep(x)
- # if(os.path.exists(TestFolder)):
- # y = False # 'Did not Crash'
- # break
- # else:
- # i+=1
- # if y:
- # print "Autopsy failed to initialize properly, restarting from last image..."
- # c = wmi.WMI()
- # for proc in c.Win32_Process():
- # if proc.name == "NetBeans Platform 7.2":
- # proc.kill()
- # break
- # testAddImageIngest(image, ignoreUnalloc, list)
- # return 1
+#-------------------------------------------------------------#
+# Parses argv and stores booleans to match command line input #
+#-------------------------------------------------------------#
+class Args:
+ def __init__(self):
+ self.single = False
+ self.single_file = ""
+ self.rebuild = False
+ self.list = False
+ self.config_file = ""
+ self.unallocated = False
+ self.ignore = False
+ self.delete = False
+ self.verbose = False
+ self.exception = False
+ self.exception_string = ""
+ def parse(self):
+ sys.argv.pop(0)
+ while sys.argv:
+ arg = sys.argv.pop(0)
+ if(arg == "-f"):
+ try:
+ arg = sys.argv.pop(0)
+ printout("Running on a single file:")
+ printout(path_fix(arg) + "\n")
+ self.single = True
+ self.single_file = path_fix(arg)
+ except:
+ printerror("Error: No single file given.\n")
+ return False
+ elif(arg == "-r" or arg == "--rebuild"):
+ printout("Running in rebuild mode.\n")
+ self.rebuild = True
+ elif(arg == "-l" or arg == "--list"):
+ try:
+ arg = sys.argv.pop(0)
+ printout("Running from configuration file:")
+ printout(arg + "\n")
+ self.list = True
+ self.config_file = arg
+ except:
+ printerror("Error: No configuration file given.\n")
+ return False
+ elif(arg == "-u" or arg == "--unallocated"):
+ printout("Ignoring unallocated space.\n")
+ self.unallocated = True
+ elif(arg == "-i" or arg == "--ignore"):
+ printout("Ignoring the ./input directory.\n")
+ self.ignore = True
+ elif(arg == "-d" or arg == "--delete"):
+ printout("Deleting Solr index after ingest.\n")
+ self.delete = True
+ elif(arg == "-v" or arg == "--verbose"):
+ printout("Running in verbose mode:")
+ printout("Printing all thrown exceptions.\n")
+ self.verbose = True
+ elif(arg == "-e" or arg == "--exception"):
+ try:
+ arg = sys.argv.pop(0)
+ printout("Running in exception mode: ")
+ printout("Printing all exceptions with the string '" + arg + "'\n")
+ self.exception = True
+ self.exception_string = arg
+ except:
+ printerror("Error: No exception string given.")
+ elif arg == "-h" or arg == "--help":
+ printout(usage())
+ return False
+ else:
+ printout(usage())
+ return False
+ # Return the args were sucessfully parsed
+ return True
+
+
+#-----------------------------------------------------#
+# Holds all global variables for each individual test #
+#-----------------------------------------------------#
+class TestAutopsy:
+ def __init__(self):
+ # Paths:
+ self.input_dir = make_local_path("input")
+ self.output_dir = ""
+ self.gold = "gold"
+ # Logs:
+ self.antlog_dir = ""
+ self.common_log = ""
+ self.csv = ""
+ self.html_log = ""
+ # Error tracking
+ self.printerror = []
+ self.printout = []
+ self.report_passed = False
+ # Image info:
+ self.image_file = ""
+ self.image_name = ""
+ # Ant info:
+ self.known_bad_path = ""
+ self.keyword_path = ""
+ self.nsrl_path = ""
+ # Case info
+ self.start_date = ""
+ self.end_date = ""
+ self.total_test_time = ""
+ self.total_ingest_time = ""
+ self.autopsy_version = ""
+ self.heap_space = ""
+ self.service_times = ""
+
+ # Set the timeout to something huge
+ # The entire tester should not timeout before this number in ms
+ # However it only seems to take about half this time
+ # And it's very buggy, so we're being careful
+ self.timeout = 24 * 60 * 60 * 1000 * 1000
+ self.ant = []
+
+ def get_image_name(self, image_file):
+ path_end = image_file.rfind("/")
+ path_end2 = image_file.rfind("\\")
+ ext_start = image_file.rfind(".")
+ if(ext_start == -1):
+ name = image_file
+ if(path_end2 != -1):
+ name = image_file[path_end2+1:ext_start]
+ elif(ext_start == -1):
+ name = image_file[path_end+1:]
+ elif(path_end == -1):
+ name = image_file[:ext_start]
+ elif(path_end!=-1 and ext_start!=-1):
+ name = image_file[path_end+1:ext_start]
+ else:
+ name = image_file[path_end2+1:ext_start]
+ return name
+
+ def ant_to_string(self):
+ string = ""
+ for arg in self.ant:
+ string += (arg + " ")
+ return string
+
+ def reset(self):
+ # Paths:
+ self.input_dir = make_local_path("input")
+ self.gold = "gold"
+ # Logs:
+ self.antlog_dir = ""
+ # Error tracking
+ self.printerror = []
+ self.printout = []
+ self.report_passed = False
+ # Image info:
+ self.image_file = ""
+ self.image_name = ""
+ # Ant info:
+ self.known_bad_path = ""
+ self.keyword_path = ""
+ self.nsrl_path = ""
+ # Case info
+ self.start_date = ""
+ self.end_date = ""
+ self.total_test_time = ""
+ self.total_ingest_time = ""
+ self.heap_space = ""
+ self.service_times = ""
+
+ # Set the timeout to something huge
+ # The entire tester should not timeout before this number in ms
+ # However it only seems to take about half this time
+ # And it's very buggy, so we're being careful
+ self.timeout = 24 * 60 * 60 * 1000 * 1000
+ self.ant = []
+
-# Run ingest on all the images in 'input', using notablekeywords.xml and notablehashes.txt-md5.idx
-def testAddImageIngest(inFile, ignoreUnalloc, list):
- print "================================================"
- print "Ingesting Image: " + inFile
+#---------------------------------------------------------#
+# Holds all database information from querying autopsy.db #
+# and standard.db. Initialized when the autopsy.db file #
+# is compared to the gold standard. #
+#---------------------------------------------------------#
+class Database:
+ def __init__(self):
+ self.gold_artifacts = []
+ self.autopsy_artifacts = []
+ self.gold_attributes = 0
+ self.autopsy_attributes = 0
+ self.gold_objects = 0
+ self.autopsy_objects = 0
+ self.artifact_comparison = []
+ self.attribute_comparison = []
+
+ def clear(self):
+ self.gold_artifacts = []
+ self.autopsy_artifacts = []
+ self.gold_attributes = 0
+ self.autopsy_attributes = 0
+ self.gold_objects = 0
+ self.autopsy_objects = 0
+ self.artifact_comparison = []
+ self.attribute_comparison = []
+
+ def get_artifacts_count(self):
+ total = 0
+ for nums in self.autopsy_artifacts:
+ total += nums
+ return total
+
+ def get_artifact_comparison(self):
+ if not self.artifact_comparison:
+ return "All counts matched"
+ else:
+ return "; ".join(self.artifact_comparison)
+
+ def get_attribute_comparison(self):
+ if not self.attribute_comparison:
+ return "All counts matched"
+ list = []
+ for error in self.attribute_comparison:
+ list.append(error)
+ return ";".join(list)
- # Set up case directory path
- testCaseName = imageName(inFile, ignoreUnalloc, list)
-
- if os.path.exists(os.path.join(outDir,testCaseName)):
- shutil.rmtree(os.path.join(outDir,testCaseName))
- os.makedirs(os.path.join(outDir,testCaseName))
- if not os.path.exists(inDir):
- markError("input dir does not exist", inFile)
- cwd = wgetcwd()
- testInFile = wabspath(inFile)
- global CommonLog
- CommonLog = open(os.path.join(cwd, outDir, imageName(inFile, ignoreUnalloc, list), "CommonLog.txt"), "w") #In this function, because it must be after the makedirs
- # NEEDS windows path (backslashes) for .E00 images to work
- testInFile = testInFile.replace("/", "\\")
- if list:
- knownBadPath = os.path.join(inDir, "notablehashes.txt-md5.idx")
- keywordPath = os.path.join(inDir, "notablekeywords.xml")
- nsrlPath = os.path.join(inDir, "nsrl.txt-md5.idx")
- else:
- knownBadPath = os.path.join(cwd,inDir,"notablehashes.txt-md5.idx")
- keywordPath = os.path.join(cwd,inDir,"notablekeywords.xml")
- nsrlPath = os.path.join(cwd,inDir,"nsrl.txt-md5.idx")
-
- knownBadPath = knownBadPath.replace("/", "\\")
- keywordPath = keywordPath.replace("/", "\\")
- nsrlPath = nsrlPath.replace("/", "\\")
+#----------------------------------#
+# Main testing functions #
+#----------------------------------#
- antlog = os.path.join(cwd,outDir,testCaseName,"antlog.txt")
- antlog = antlog.replace("/", "\\")
-
- timeout = 24 * 60 * 60 * 1000 # default of 24 hours, just to be safe
- size = getImageSize(inFile,ignoreUnalloc, list) # get the size in bytes
- timeout = (size / 1000) / 1000 # convert to MB
- timeout = timeout * 1000 # convert sec to ms
- timeout = timeout * 1.5 # add a little extra umph
- timeout = timeout * 25 # decided we needed A LOT extra to be safe
-
- # set up ant target
- args = ["ant"]
- args.append("-q")
- args.append("-f")
- args.append(os.path.join("..","build.xml"))
- args.append("regression-test")
- args.append("-l")
- args.append(antlog)
- args.append("-Dimg_path=" + testInFile)
- args.append("-Dknown_bad_path=" + knownBadPath)
- args.append("-Dkeyword_path=" + keywordPath)
- args.append("-Dnsrl_path=" + nsrlPath)
- args.append("-Dgold_path=" + os.path.join(cwd,goldDir).replace("/", "\\"))
- args.append("-Dout_path=" + os.path.join(cwd,outDir,testCaseName).replace("/", "\\"))
- args.append("-Dignore_unalloc=" + "%s" % ignoreUnalloc)
- args.append("-Dtest.timeout=" + str(timeout))
-
- # print the ant testing command
- print "CMD: " + " ".join(args)
-
- print "Starting test..."
- # thread.start_new_thread(AutopsyCrash(inFile, ignoreUnalloc, list))
- # thread.start_new_thread(subprocess.call(args))
- subprocess.call(args)
-
-
-def getImageSize(inFile, ignoreUnalloc, list):
- name = imageName(inFile, ignoreUnalloc, list)
- size = 0
- if list:
- size += os.path.getsize(inFile)
- else:
- path = os.path.join(".",inDir)
-
- for files in os.listdir(path):
- filename = os.path.splitext(files)[0]
- if filename == name:
- filepath = os.path.join(path, files)
- if not os.path.samefile(filepath, inFile):
- size += os.path.getsize(filepath)
- size += os.path.getsize(inFile)
- return size
-
-def testCompareToGold(inFile, ignoreUnalloc, list):
-
- global CommonLog
- cwd = wgetcwd()
-
- print "-----------------------------------------------"
- print "Comparing results for " + inFile + " with gold."
-
- name = imageName(inFile, ignoreUnalloc, list)
-
- goldFile = os.path.join("./",goldDir,name,"standard.db")
- testFile = os.path.join("./",outDir,name,"AutopsyTestCase","autopsy.db")
- if os.path.isfile(goldFile) == False:
- markError("No gold standard exists", inFile)
- return
- if os.path.isfile(testFile) == False:
- markError("No database exists", inFile)
- return
-
- # For now, comparing size of blackboard_artifacts,
- # blackboard_attributes,
- # and tsk_objects.
- goldConn = sqlite3.connect(goldFile)
- goldC = goldConn.cursor()
- testConn = sqlite3.connect(testFile)
- testC = testConn.cursor()
-
- CommonLog.write("Comparing Artifacts: \n\r")
- print("Comparing Artifacts: ")
-
- # Keep range in sync with number of items in ARTIFACT_TYPE enum
- for type_id in range(1, 13):
- goldC.execute("select count(*) from blackboard_artifacts where artifact_type_id=%d" % type_id)
- goldArtifacts = goldC.fetchone()[0]
- testC.execute("select count(*) from blackboard_artifacts where artifact_type_id=%d" % type_id)
- testArtifacts = testC.fetchone()[0]
- if(goldArtifacts != testArtifacts):
- errString = str("Artifact counts do not match for type id %d!: " % type_id)
- errString += str("Gold: %d, Test: %d" % (goldArtifacts, testArtifacts))
- CommonLog.write(errString + "\n\r")
- markError(errString, inFile)
- else:
- CommonLog.write("Artifact counts for artifact type id %d match!" % type_id + "\n\r")
- print("Artifact counts for artifact type id %d match!" % type_id)
-
- CommonLog.write("Comparing Attributes: \n\r")
- print("Comparing Attributes: ")
- goldC.execute("select count(*) from blackboard_attributes")
- goldAttributes = goldC.fetchone()[0]
- testC.execute("select count(*) from blackboard_attributes")
- testAttributes = testC.fetchone()[0]
- if(goldAttributes != testAttributes):
- errString = "Attribute counts do not match!: "
- errString += str("Gold: %d, Test: %d" % (goldAttributes, testAttributes))
- CommonLog.write(errString + "\n\r")
- markError(errString, inFile)
- else:
- print("Attribute counts match!")
- print("Comparing TSK Objects: ")
- goldC.execute("select count(*) from tsk_objects")
- goldObjects = goldC.fetchone()[0]
- testC.execute("select count(*) from tsk_objects")
- testObjects = testC.fetchone()[0]
- if(goldObjects != testObjects):
- errString = "TSK Object counts do not match!: "
- errString += str("Gold: %d, Test: %d" % (goldObjects, testObjects))
- CommonLog.write(errString + "\n\r")
- markError(errString, inFile)
- else:
- CommonLog.write("Object counts match!" + "\n\r")
- print("Object counts match!")
-
-def clearGoldDir(inFile, ignoreUnalloc, list):
-
- cwd = wgetcwd()
- inFile = imageName(inFile, ignoreUnalloc, list)
- if os.path.exists(os.path.join(cwd,goldDir,inFile)):
- shutil.rmtree(os.path.join(cwd,goldDir,inFile))
- os.makedirs(os.path.join(cwd,goldDir,inFile))
- print "Clearing gold directory: " + os.path.join(cwd,goldDir,inFile)
-
-def copyTestToGold(inFile, ignoreUnalloc, list):
- print "------------------------------------------------"
- print "Recreating gold standard from results."
- inFile = imageName(inFile, ignoreUnalloc, list)
- cwd = wgetcwd()
- goldFile = os.path.join("./",goldDir,inFile,"standard.db")
- testFile = os.path.join("./",outDir,inFile,"AutopsyTestCase","autopsy.db")
- shutil.copy(testFile, goldFile)
- print "Recreated gold standards"
-
-def copyReportToGold(inFile, ignoreUnalloc, list):
- print "------------------------------------------------"
- print "Recreating gold report from results."
- inFile = imageName(inFile, ignoreUnalloc, list)
- cwd = wgetcwd()
- goldReport = os.path.join("./",goldDir,inFile,"report.html")
- testReportPath = os.path.join("./",outDir,inFile,"AutopsyTestCase","Reports")
- # Because Java adds a timestamp to the report file, one can't call it
- # directly, so one must get a list of files in the dir, which are only
- # reports, then filter for the .html report
- testReport = None
- for files in os.listdir(testReportPath):
- if files.endswith(".html"): # Get the HTML one
- testReport = os.path.join("./",outDir,inFile,"AutopsyTestCase","Reports",files)
- if testReport is None:
- markError("No test report exists", inFile)
- return
- else:
- shutil.copy(testReport, goldReport)
- print "Report copied"
-
-def deleteKeywordFiles(inFile, ignoreUnalloc, list):
- print "------------------------------------------------"
- print "Deleting Keyword Search files"
- inFile = imageName(inFile, ignoreUnalloc, list)
- cwd = wgetcwd()
- shutil.rmtree(os.path.join("./", outDir, inFile, "AutopsyTestCase", "KeywordSearch"))
-
-def testCompareReports(inFile, ignoreUnalloc, list):
- print "------------------------------------------------"
- print "Comparing report to golden report."
- name = imageName(inFile, ignoreUnalloc, list)
- goldReport = os.path.join("./",goldDir,name,"report.html")
- testReportPath = os.path.join("./",outDir,name,"AutopsyTestCase","Reports")
- # Because Java adds a timestamp to the report file, one can't call it
- # directly, so one must get a list of files in the dir, which are only
- # reports, then filter for the .html report
- testReport = None
- for files in os.listdir(testReportPath):
- if files.endswith(".html"): # Get the HTML one
- testReport = os.path.join("./",outDir,name,"AutopsyTestCase","Reports",files)
- if os.path.isfile(goldReport) == False:
- markError("No gold report exists", inFile)
- return
- if testReport is None:
- markError("No test report exists", inFile)
- return
- # Compare the reports
- goldFile = open(goldReport)
- testFile = open(testReport)
- # Search for
because it is first seen in the report
- # immediately after the unnecessary metadata, styles, and timestamp
- gold = goldFile.read()
- test = testFile.read()
- gold = gold[gold.find(""):]
- test = test[test.find(""):]
- # Splitting allows for printouts of what the difference is
- goldList = split(gold, 50)
- testList = split(test, 50)
- failed = 0
- for goldSplit, testSplit in zip(goldList, testList):
- if goldSplit != testSplit:
- failed = 1
- #print "Got: " + testSplit
- #print "Expected: " + goldSplit
- break
- if(failed):
- errString = "Reports do not match."
- markError(errString, inFile)
- else:
- print "Reports match.\n\n"
-
-def reportErrors(image, verbose, search):
- global CommonLog
- cwd = wgetcwd()
- files = []
- exceptions = []
- antlog = open(os.path.join(cwd, outDir, image, "antlog.txt"), "r")
- autopsyLog0 = open(os.path.join(cwd, outDir, image, "logs", "autopsy.log.0"), "r")
- messages = open(os.path.join(cwd, outDir, image, "logs", "messages.log"), "r")
- solrLogError0 = open(os.path.join(cwd, outDir, image, "logs", "solr.log.error.0"), "r")
- files = [antlog, autopsyLog0, messages, solrLogError0]
- numWarnings = 0
- numExceptions = 0
- for file in files:
- for line in file:
- if search is not None:
- if(("exception" in line) or ("EXCEPTION" in line) or ("Exception" in line)) and search in line:
- exceptions.append("From " + file.name[file.name.rfind("/")+1:] +":\n\n" + line + "\n\n")
- numExceptions +=1
+# Iterates through an XML configuration file to find all given elements
+def run_config_test(config_file):
+ try:
+ parsed = parse(config_file)
+ input_dir = parsed.getElementsByTagName("indir")[0].getAttribute("value").encode()
+ case.input_dir = input_dir
+
+ # Generate the top navbar of the HTML for easy access to all images
+ values = []
+ for element in parsed.getElementsByTagName("image"):
+ value = element.getAttribute("value").encode()
+ if file_exists(value):
+ values.append(value)
+ html_add_images(values)
+
+ # Run the test for each file in the configuration
+ for element in parsed.getElementsByTagName("image"):
+ value = element.getAttribute("value").encode()
+ if file_exists(value):
+ run_test(value)
else:
- if verbose:
- if("warning" in line) or ("WARNING" in line) or ("Warning" in line):
- exceptions.append("From " + file.name[file.name.rfind("/")+1:] +":\n\n" + line+ "\n\n")
- numWarnings +=1
- if("exception" in line) or ("EXCEPTION" in line) or ("Exception" in line):
- exceptions.append("From " + file.name[file.name.rfind("/")+1:] +":\n\n" + line+ "\n\n")
- numExceptions +=1
- file.close()
- if verbose:
- CommonLog.write("Included warnings\n\r")
- if search is not None:
- CommonLog.write("Looked for errors with \"" + search + "\"\n\r")
- CommonLog.write("Total entries: " + str(len(exceptions)) + "\n\r")
- for error in exceptions:
- CommonLog.write(error + "\n\r")
- return (numWarnings, numExceptions)
+ printerror("Warning: Image file listed in the configuration does not exist:")
+ printerror(value + "\n")
+ except Exception as e:
+ printerror("Error: There was an error running with the configuration file.")
+ printerror(str(e) + "\n")
+
+# Runs the test on the single given file.
+# The path must be guarenteed to be a correct path.
+def run_test(image_file):
+ if not image_type(image_file) != IMGTYPE.UNKNOWN:
+ printerror("Error: Image type is unrecognized:")
+ printerror(image_file + "\n")
+ return
+
+ # Set the case to work for this test
+ case.image_file = image_file
+ case.image_name = case.get_image_name(image_file)
+ case.antlog_dir = make_local_path(case.output_dir, case.image_name, "antlog.txt")
+ case.known_bad_path = make_path(case.input_dir, "notablehashes.txt-md5.idx")
+ case.keyword_path = make_path(case.input_dir, "notablekeywords.xml")
+ case.nsrl_path = make_path(case.input_dir, "nsrl.txt-md5.idx")
-
-def split(input, size):
- return [input[start:start+size] for start in range(0, len(input), size)]
+ run_ant()
+
+ # After the java has ran:
+ copy_logs()
+ generate_common_log()
+ try:
+ fill_case_data()
+ except StringNotFoundException as e:
+ e.print_error()
+ except Exception as e:
+ printerror("Error: Unknown fatal error when filling case data.")
+ printerror(str(e) + "\n")
+
+ # If running in rebuild mode (-r)
+ if args.rebuild:
+ rebuild()
+ # If deleting Solr index (-d)
+ if args.delete:
+ solr_index = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "KeywordSearch")
+ clear_dir(solr_index)
+ print_report([], "DELETE SOLR INDEX", "Solr index deleted.")
+ # If running in verbose mode (-v)
+ if args.verbose:
+ errors = report_all_errors()
+ okay = "No warnings or errors in any log files."
+ print_report(errors, "VERBOSE", okay)
+ # If running in exception mode (-e)
+ if args.exception:
+ exceptions = report_with(args.exception_string)
+ okay = "No warnings or exceptions found containing text '" + args.exception_string + "'."
+ print_report(exceptions, "EXCEPTION", okay)
+
+ # Now test in comparison to the gold standards
+ compare_to_gold_db()
+ compare_to_gold_html()
+
+ # Make the CSV log and the html log viewer
+ generate_csv()
+ generate_html()
+
+ # Reset the case and return the tests sucessfully finished
+ case.reset()
+ return True
-class ImgType:
+# Tests Autopsy with RegressionTest.java by by running
+# the build.xml file through ant
+def run_ant():
+ # Set up the directories
+ test_case_path = os.path.join(case.output_dir, case.image_name)
+ if dir_exists(test_case_path):
+ shutil.rmtree(test_case_path)
+ os.makedirs(test_case_path)
+ if not dir_exists(make_local_path("gold")):
+ os.makedirs(make_local_path("gold"))
+
+ case.ant = ["ant"]
+ case.ant.append("-q")
+ case.ant.append("-f")
+ case.ant.append(os.path.join("..","build.xml"))
+ case.ant.append("regression-test")
+ case.ant.append("-l")
+ case.ant.append(case.antlog_dir)
+ case.ant.append("-Dimg_path=" + case.image_file)
+ case.ant.append("-Dknown_bad_path=" + case.known_bad_path)
+ case.ant.append("-Dkeyword_path=" + case.keyword_path)
+ case.ant.append("-Dnsrl_path=" + case.nsrl_path)
+ case.ant.append("-Dgold_path=" + make_local_path(case.gold))
+ case.ant.append("-Dout_path=" + make_local_path(case.output_dir, case.image_name))
+ case.ant.append("-Dignore_unalloc=" + "%s" % args.unallocated)
+ case.ant.append("-Dtest.timeout=" + str(case.timeout))
+
+ printout("Ingesting Image:\n" + case.image_file + "\n")
+ printout("CMD: " + " ".join(case.ant))
+ printout("Starting test...\n")
+ subprocess.call(case.ant)
+
+# Returns the type of image file, based off extension
+class IMGTYPE:
RAW, ENCASE, SPLIT, UNKNOWN = range(4)
-
-def imageType(inFile):
- extStart = inFile.rfind(".")
- if (extStart == -1):
- return ImgType.UNKNOWN
- ext = inFile[extStart:].lower()
+def image_type(image_file):
+ ext_start = image_file.rfind(".")
+ if (ext_start == -1):
+ return IMGTYPE.UNKNOWN
+ ext = image_file[ext_start:].lower()
if (ext == ".img" or ext == ".dd"):
- return ImgType.RAW
+ return IMGTYPE.RAW
elif (ext == ".e01"):
- return ImgType.ENCASE
+ return IMGTYPE.ENCASE
elif (ext == ".aa" or ext == ".001"):
- return ImgType.SPLIT
+ return IMGTYPE.SPLIT
else:
- return ImgType.UNKNOWN
+ return IMGTYPE.UNKNOWN
-def imageName(inFile, ignoreUnalloc, list):
- pathEnd = inFile.rfind("/")
- pathEnd2 = inFile.rfind("\\")
- extStart = inFile.rfind(".")
- if(extStart == -1 and extStart == -1):
- name = inFile
- if(pathEnd2 != -1):
- name = inFile[pathEnd2+1:extStart]
- elif(extStart == -1):
- name = inFile[pathEnd+1:]
- elif(pathEnd == -1):
- name = inFile[:extStart]
- elif(pathEnd!=-1 and extStart!=-1):
- name = inFile[pathEnd+1:extStart]
+
+
+#-----------------------------------------------------------#
+# Functions relating to rebuilding and comparison #
+# of gold standards #
+#-----------------------------------------------------------#
+
+# Rebuilds the gold standards by copying the test-generated database
+# and html report files into the gold directory
+def rebuild():
+ # Errors to print
+ errors = []
+ # Delete the current gold standards
+ gold_dir = make_local_path(case.gold, case.image_name)
+ clear_dir(gold_dir)
+
+ # Rebuild the database
+ gold_from = make_local_path(case.output_dir, case.image_name,
+ "AutopsyTestCase", "autopsy.db")
+ gold_to = make_local_path(case.gold, case.image_name, "standard.db")
+ try:
+ copy_file(gold_from, gold_to)
+ except FileNotFoundException as e:
+ errors.append(e.error)
+ except Exception as e:
+ printerror("Error: Unknown fatal error when rebuilding the gold database.")
+ errors.append(str(e))
+
+ # Rebuild the HTML report
+ html_path = make_local_path(case.output_dir, case.image_name,
+ "AutopsyTestCase", "Reports")
+ try:
+ html_from = get_file_in_dir(html_path, ".html")
+ html_to = make_local_path(case.gold, case.image_name, "standard.html")
+ copy_file(html_from, html_to)
+ except FileNotFoundException as e:
+ errors.append(e.error)
+ except Exception as e:
+ printerror("Error: Unknown fatal error when rebuilding the gold html report.")
+ errors.append(str(e))
+
+ okay = "Sucessfully rebuilt all gold standards."
+ print_report(errors, "REBUILDING", okay)
+
+# Using the global case's variables, compare the database file made by the
+# regression test to the gold standard database file
+# Initializes the global database, which stores the information retrieved
+# from queries while comparing
+def compare_to_gold_db():
+ # SQLITE needs unix style pathing
+ gold_db_file = os.path.join("./", case.gold, case.image_name, "standard.db")
+ autopsy_db_file = os.path.join("./", case.output_dir, case.image_name,
+ "AutopsyTestCase", "autopsy.db")
+ if not file_exists(autopsy_db_file):
+ printerror("Error: Database file does not exist at:")
+ printerror(autopsy_db_file + "\n")
+ return
+ if not file_exists(gold_db_file):
+ printerror("Error: Gold database file does not exist at:")
+ printerror(gold_db_file + "\n")
+ return
+
+ # compare size of bb artifacts, attributes, and tsk objects
+ gold_con = sqlite3.connect(gold_db_file)
+ gold_cur = gold_con.cursor()
+ autopsy_con = sqlite3.connect(autopsy_db_file)
+ autopsy_cur = autopsy_con.cursor()
+
+ exceptions = []
+ database.clear()
+ # Testing tsk_objects
+ exceptions.append(compare_tsk_objects(gold_cur, autopsy_cur))
+ # Testing blackboard_artifacts
+ exceptions.append(compare_bb_artifacts(gold_cur, autopsy_cur))
+ # Testing blackboard_attributes
+ exceptions.append(compare_bb_attributes(gold_cur, autopsy_cur))
+
+ database.artifact_comparison = exceptions[1]
+ database.attribute_comparison = exceptions[2]
+
+ okay = "All counts match."
+ print_report(exceptions[0], "COMPARE TSK OBJECTS", okay)
+ print_report(exceptions[1], "COMPARE ARTIFACTS", okay)
+ print_report(exceptions[2], "COMPARE ATTRIBUTES", okay)
+
+# Using the global case's variables, compare the html report file made by
+# the regression test against the gold standard html report
+def compare_to_gold_html():
+ gold_html_file = make_local_path(case.gold, case.image_name, "standard.html")
+ autopsy_html_path = make_local_path(case.output_dir, case.image_name,
+ "AutopsyTestCase", "Reports")
+ try:
+ autopsy_html_file = get_file_in_dir(autopsy_html_path, ".html")
+
+ if not file_exists(gold_html_file):
+ printerror("Error: No gold html report exists at:")
+ printerror(gold_html_file + "\n")
+ return
+ if not file_exists(autopsy_html_file):
+ printerror("Error: No case html report exists at:")
+ printerror(autopsy_html_file + "\n")
+ return
+
+ errors = []
+ errors = compare_report_files(autopsy_html_file, gold_html_file)
+ okay = "The test report matches the gold report."
+ print_report(errors, "REPORT COMPARISON", okay)
+ if not errors:
+ case.report_passed = True
+ except FileNotFoundException as e:
+ e.print_error()
+ except DirNotFoundException as e:
+ e.print_error()
+ except Exception as e:
+ printerror("Error: Unknown fatal error comparing databases.")
+ printerror(str(e) + "\n")
+
+# Compares the blackboard artifact counts of two databases
+# given the two database cursors
+def compare_bb_artifacts(gold_cur, autopsy_cur):
+ exceptions = []
+ for type_id in range(1, 13):
+ gold_cur.execute("SELECT COUNT(*) FROM blackboard_artifacts WHERE artifact_type_id=%d" % type_id)
+ gold_artifacts = gold_cur.fetchone()[0]
+ autopsy_cur.execute("SELECT COUNT(*) FROM blackboard_artifacts WHERE artifact_type_id=%d" % type_id)
+ autopsy_artifacts = autopsy_cur.fetchone()[0]
+ # Append the results to the global database
+ database.gold_artifacts.append(gold_artifacts)
+ database.autopsy_artifacts.append(autopsy_artifacts)
+ # Report every discrepancy
+ if gold_artifacts != autopsy_artifacts:
+ error = str("Artifact counts do not match for type id %d. " % type_id)
+ error += str("Gold: %d, Test: %d" % (gold_artifacts, autopsy_artifacts))
+ exceptions.append(error)
+ return exceptions
+
+# Compares the blackboard atribute counts of two databases
+# given the two database cursors
+def compare_bb_attributes(gold_cur, autopsy_cur):
+ exceptions = []
+ gold_cur.execute("SELECT COUNT(*) FROM blackboard_attributes")
+ gold_attributes = gold_cur.fetchone()[0]
+ autopsy_cur.execute("SELECT COUNT(*) FROM blackboard_attributes")
+ autopsy_attributes = autopsy_cur.fetchone()[0]
+ # Give the database the attributes
+ database.gold_attributes = gold_attributes
+ database.autopsy_attributes = autopsy_attributes
+ if gold_attributes != autopsy_attributes:
+ error = "Attribute counts do not match. "
+ error += str("Gold: %d, Test: %d" % (gold_attributes, autopsy_attributes))
+ exceptions.append(error)
+ return exceptions
+
+# Compares the tsk object counts of two databases
+# given the two database cursors
+def compare_tsk_objects(gold_cur, autopsy_cur):
+ exceptions = []
+ gold_cur.execute("SELECT COUNT(*) FROM tsk_objects")
+ gold_objects = gold_cur.fetchone()[0]
+ autopsy_cur.execute("SELECT COUNT(*) FROM tsk_objects")
+ autopsy_objects = autopsy_cur.fetchone()[0]
+ # Give the database the attributes
+ database.gold_objects = gold_objects
+ database.autopsy_objects = autopsy_objects
+ if gold_objects != autopsy_objects:
+ error = "TSK Object counts do not match. "
+ error += str("Gold: %d, Test: %d" % (gold_objects, autopsy_objects))
+ exceptions.append(error)
+ return exceptions
+
+
+
+#-------------------------------------------------#
+# Functions relating to error reporting #
+#-------------------------------------------------#
+
+# Generate the "common log": a log of all exceptions and warnings
+# from each log file generated by Autopsy
+def generate_common_log():
+ logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+ common_log = open(case.common_log, "a")
+ common_log.write("--------------------------------------------------\n")
+ common_log.write(case.image_name + "\n")
+ common_log.write("--------------------------------------------------\n")
+ for file in os.listdir(logs_path):
+ log = open(make_path(logs_path, file), "r")
+ lines = log.readlines()
+ for line in lines:
+ if ("exception" in line) or ("EXCEPTION" in line) or ("Exception" in line):
+ common_log.write("From " + log.name[log.name.rfind("/")+1:] +":\n" + line + "\n")
+ if ("warning" in line) or ("WARNING" in line) or ("Warning" in line):
+ common_log.write("From " + log.name[log.name.rfind("/")+1:] +":\n" + line + "\n")
+ log.close()
+ common_log.write("\n\n")
+ common_log.close()
+
+# Fill in the global case's variables that require the log files
+def fill_case_data():
+ # Open autopsy.log.0
+ log_path = make_local_path(case.output_dir, case.image_name, "logs", "autopsy.log.0")
+ log = open(log_path)
+
+ # Set the case starting time based off the first line of autopsy.log.0
+ # *** If logging time format ever changes this will break ***
+ case.start_date = log.readline().split(" java.")[0]
+
+ # Set the case ending time based off the "create" time (when the file was copied)
+ case.end_date = time.ctime(os.path.getmtime(log_path))
+
+ # Set the case total test time
+ # Start date must look like: "Jul 16, 2012 12:57:53 PM"
+ # End date must look like: "Mon Jul 16 13:02:42 2012"
+ # *** If logging time format ever changes this will break ***
+ start = datetime.datetime.strptime(case.start_date, "%b %d, %Y %I:%M:%S %p")
+ end = datetime.datetime.strptime(case.end_date, "%a %b %d %H:%M:%S %Y")
+ case.total_test_time = str(end - start)
+
+ # Set Autopsy version, heap space, ingest time, and service times
+ version_line = search_logs("INFO: Application name: Autopsy, version:")[0]
+ case.autopsy_version = get_word_at(version_line, 5).rstrip(",")
+ case.heap_space = search_logs("Heap memory usage:")[0].rstrip().split(": ")[1]
+ ingest_line = search_logs("INFO: Ingest (including enqueue)")[0]
+ case.total_ingest_time = get_word_at(ingest_line, 5).rstrip()
+ try:
+ service_lines = search_log("autopsy.log.0", "to process()")
+ service_list = []
+ for line in service_lines:
+ words = line.split(" ")
+ # Kind of forcing our way into getting this data
+ # If this format changes, the tester will break
+ i = words.index("secs.")
+ times = words[i-4] + " "
+ times += words[i-3] + " "
+ times += words[i-2] + " "
+ times += words[i-1] + " "
+ times += words[i]
+ service_list.append(times)
+ case.service_times = "; ".join(service_list)
+ except FileNotFoundException as e:
+ e.print_error()
+ except Exception as e:
+ printerror("Error: Unknown fatal error when finding service times.")
+ printerror(srt(e) + "\n")
+
+# Generate the CSV log file
+def generate_csv():
+ # If the CSV file hasn't already been generated, this is the
+ # first run, and we need to add the column names
+ if not file_exists(case.csv):
+ csv_header()
+
+ # Now add on the fields to a new row
+ try:
+ csv = open(case.csv, "a")
+ # Variables that need to be written
+ image_path = case.image_file
+ name = case.image_name
+ path = case.output_dir
+ autopsy_version = case.autopsy_version
+ heap_space = case.heap_space
+ start_date = case.start_date
+ end_date = case.end_date
+ total_test_time = case.total_test_time
+ total_ingest_time = case.total_ingest_time
+ service_times = case.service_times
+ exceptions_count = str(len(get_exceptions()))
+ tsk_objects_count = str(database.autopsy_objects)
+ artifacts_count = str(database.get_artifacts_count())
+ attributes_count = str(database.autopsy_attributes)
+ gold_db_name = "standard.db"
+ artifact_comparison = database.get_artifact_comparison()
+ attribute_comparison = database.get_attribute_comparison()
+ gold_report_name = "standard.html"
+ report_comparison = str(case.report_passed)
+ ant = case.ant_to_string()
+
+ # Make a list with all the strings in it
+ seq = (image_path, name, path, autopsy_version, heap_space, start_date, end_date, total_test_time,
+ total_ingest_time, service_times, exceptions_count, tsk_objects_count, artifacts_count,
+ attributes_count, gold_db_name, artifact_comparison, attribute_comparison,
+ gold_report_name, report_comparison, ant)
+ # Join it together with a ", "
+ output = "|".join(seq)
+ output += "\n"
+ # Write to the log!
+ csv.write(output)
+ csv.close()
+ except StringNotFoundException as e:
+ e.print_error()
+ printerror("Error: CSV file was not created.\n")
+ except Exception as e:
+ printerror("Error: Unknown fatal error when creating CSV file.")
+ printerror(str(e) + "\n")
+
+# Generates the CSV header (column names)
+def csv_header():
+ csv = open(case.csv, "w")
+ seq = ("Image Path", "Image Name", "Output Case Directory", "Autopsy Version",
+ "Heap Space Setting", "Test Start Date", "Test End Date", "Total Test Time",
+ "Total Ingest Time", "Service Times", "Exceptions Count", "TSK Objects Count", "Artifacts Count",
+ "Attributes Count", "Gold Database Name", "Artifacts Comparison",
+ "Attributes Comparison", "Gold Report Name", "Report Comparison", "Ant Command Line")
+ output = "|".join(seq)
+ output += "\n"
+ csv.write(output)
+ csv.close()
+
+# Returns a list of all the exceptions listed in the common log
+def get_exceptions():
+ try:
+ exceptions = []
+ common_log = open(case.common_log, "r")
+ lines = common_log.readlines()
+ for line in lines:
+ if ("exception" in line) or ("EXCEPTION" in line) or ("Exception" in line):
+ exceptions.append(line)
+ common_log.close()
+ return exceptions
+ except:
+ raise FileNotFoundExeption(case.common_log)
+
+# Returns a list of all the warnings listed in the common log
+def get_warnings():
+ try:
+ warnings = []
+ common_log = open(case.common_log, "r")
+ lines = common_log.readlines()
+ for line in lines:
+ if ("warning" in line) or ("WARNING" in line) or ("Warning" in line):
+ warnings.append(line)
+ common_log.close()
+ return warnings
+ except:
+ raise FileNotFoundExeption(case.common_log)
+
+# Returns all the errors found in the common log in a list
+def report_all_errors():
+ try:
+ return get_warnings() + get_exceptions()
+ except FileNotFoundException as e:
+ e.print_error()
+ except Exception as e:
+ printerror("Error: Unknown fatal error when reporting all errors.")
+ printerror(str(e) + "\n")
+
+# Returns any lines from the common log with the given string in them (case sensitive)
+def report_with(exception):
+ try:
+ exceptions = []
+ common_log = open(case.common_log, "r")
+ lines = common_log.readlines()
+ for line in lines:
+ if exception in line:
+ exceptions.append(line)
+ common_log.close()
+ return exceptions
+ except:
+ printerror("Error: Cannot find the common log at:")
+ printerror(case.common_log + "\n")
+
+# Search through all the known log files for a specific string.
+# Returns a list of all lines with that string
+def search_logs(string):
+ logs_path = make_local_path(case.output_dir, case.image_name, "logs")
+ results = []
+ for file in os.listdir(logs_path):
+ log = open(make_path(logs_path, file), "r")
+ lines = log.readlines()
+ for line in lines:
+ if string in line:
+ results.append(line)
+ log.close()
+ if results:
+ return results
+ raise StringNotFoundException(string)
+# Search through all the known log files for a specific string.
+# Returns a list of all lines with that string
+
+# Searches the given log for the given string
+# Returns a list of all lines with that string
+def search_log(log, string):
+ logs_path = make_local_path(case.output_dir, case.image_name, "logs", log)
+ try:
+ results = []
+ log = open(logs_path, "r")
+ lines = log.readlines()
+ for line in lines:
+ if string in line:
+ results.append(line)
+ log.close()
+ if results:
+ return results
+ raise StringNotFoundException(string)
+ except:
+ raise FileNotFoundException(logs_path)
+
+# Print a report for the given errors with the report name as name
+# and if no errors are found, print the okay message
+def print_report(errors, name, okay):
+ if errors:
+ printerror("--------< " + name + " >----------")
+ for error in errors:
+ printerror(error)
+ printerror("--------< / " + name + " >--------\n")
else:
- name = inFile[pathEnd2+1:extStart]
- if(ignoreUnalloc):
- name+="-u"
- return name
+ printout("-----------------------------------------------------------------")
+ printout("< " + name + " - " + okay + " />")
+ printout("-----------------------------------------------------------------\n")
-def markError(errString, inFile):
- global hadErrors
- hadErrors = True
- errors = results.get(inFile, [])
- errors.append(errString)
- results[inFile] = errors
- print errString
+# Used instead of the print command when printing out an error
+def printerror(string):
+ print string
+ case.printerror.append(string)
+# Used instead of the print command when printing out anything besides errors
+def printout(string):
+ print string
+ case.printout.append(string)
+
+# Generates the HTML log file
+def generate_html():
+ # If the file doesn't exist yet, this is the first case to run for
+ # this test, so we need to make the start of the html log
+ if not file_exists(case.html_log):
+ write_html_head()
+ html = open(case.html_log, "a")
+ # The image title
+ title = "\
+ "
+ # The script errors found
+ errors = "\
+
\
+
"
+ # For each error we have logged in the case
+ for error in case.printerror:
+ # Replace < and > to avoid any html display errors
+ errors += "
" + error.replace("<", "<").replace(">", ">") + "
"
+ # If there is a \n, we probably want a
in the html
+ if "\n" in error:
+ errors += "
"
+ errors += "
"
+ # All the testing information
+ info = "\
+
\
+
\
+
"
+ # The individual elements
+ info += "Image Path: | "
+ info += "" + case.image_file + " |
"
+ info += "Image Name: | "
+ info += "" + case.image_name + " |
"
+ info += "Case Output Directory: | "
+ info += "" + case.output_dir + " |
"
+ info += "Autopsy Version: | "
+ info += "" + case.autopsy_version + " |
"
+ info += "Heap Space: | "
+ info += "" + case.heap_space + " |
"
+ info += "Test Start Date: | "
+ info += "" + case.start_date + " |
"
+ info += "Test End Date: | "
+ info += "" + case.end_date + " |
"
+ info += "Total Test Time: | "
+ info += "" + case.total_test_time + " |
"
+ info += "Total Ingest Time: | "
+ info += "" + case.total_ingest_time + " |
"
+ info += "Exceptions Count: | "
+ info += "" + str(len(get_exceptions())) + " |
"
+ info += "TSK Objects Count: | "
+ info += "" + str(database.autopsy_objects) + " |
"
+ info += "Artifacts Count: | "
+ info += "" + str(database.get_artifacts_count()) + " |
"
+ info += "Attributes Count: | "
+ info += "" + str(database.autopsy_attributes) + " |
"
+ info += "
\
+
"
+ # For all the general print statements in the case
+ output = "\
+
\
+
"
+ # For each printout in the case's list
+ for out in case.printout:
+ output += "
" + out + "
"
+ # If there was a \n it probably means we want a
in the html
+ if "\n" in out:
+ output += "
"
+ output += "
"
+
+ foot = "