From 35bc7b1cb60ca42882dc8c65df4b50760c7a5890 Mon Sep 17 00:00:00 2001 From: 0xNF Date: Wed, 18 Jul 2012 09:32:28 -0400 Subject: [PATCH] regression.py : changed the single command to -s, and the ignore command to -i to reduce confusion regressionTesting.dox : updated the documentation to reflect yesterdays update Other: Uploaded an example XML config file --- Testing/script/config.xml | 16 +++++++++ Testing/script/regression.py | 52 +++++++++++++++--------------- docs/doxygen/regressionTesting.dox | 28 +++++++++++----- 3 files changed, 62 insertions(+), 34 deletions(-) create mode 100644 Testing/script/config.xml diff --git a/Testing/script/config.xml b/Testing/script/config.xml new file mode 100644 index 0000000000..844d5d62fd --- /dev/null +++ b/Testing/script/config.xml @@ -0,0 +1,16 @@ + + + + + + + + + + \ No newline at end of file diff --git a/Testing/script/regression.py b/Testing/script/regression.py index bbc50d230d..795b5a26b6 100644 --- a/Testing/script/regression.py +++ b/Testing/script/regression.py @@ -12,7 +12,7 @@ from xml.dom.minidom import parse, parseString # Last modified 7/17/12 @5pm -# Usage: ./regression.py [-i FILE] OR [-l CONFIG] [OPTIONS] +# Usage: ./regression.py [-s FILE] OR [-l CONFIG] [OPTIONS] # Run the RegressionTest.java file, and compare the result with a gold standard # When the -i flag is set, this script only tests the image given by FILE. # An indexed NSRL database is expected at ./input/nsrl.txt-md5.idx, @@ -23,7 +23,7 @@ from xml.dom.minidom import parse, parseString # the /script folder. # Options: # -r, --rebuild Rebuild the gold standards from the test results for each image -# -u, --ignore Ignores unallocated space when ingesting. Faster, but less accurate results. +# -i, --ignore Ignores unallocated space when ingesting. Faster, but less accurate results. hadErrors = False # If any of the tests failed @@ -41,8 +41,10 @@ def testAddImageIngest(inFile, ignoreUnalloc, list): # Set up case directory path testCaseName = imageName(inFile) + + #check for flags to append to folder name if ignoreUnalloc: - testCaseName+="-u" + testCaseName+="-i" if list: testCaseName+="-l" if os.path.exists(os.path.join(outDir,testCaseName)): @@ -128,7 +130,7 @@ def testCompareToGold(inFile, ignore): name = imageName(inFile) if ignore: - name += ("-u") + name += ("-i") cwd = wgetcwd() goldFile = os.path.join("./",goldDir,name,"standard.db") testFile = os.path.join("./",outDir,name,"AutopsyTestCase","autopsy.db") @@ -189,7 +191,7 @@ def clearGoldDir(inFile, ignore, list): cwd = wgetcwd() inFile = imageName(inFile) if ignore: - inFile += "-u" + inFile += "-i" if list: inFile += "-l" if os.path.exists(os.path.join(cwd,goldDir,inFile)): @@ -201,7 +203,7 @@ def copyTestToGold(inFile, ignore, list): print "Recreating gold standard from results." inFile = imageName(inFile) if ignore: - inFile += "-u" + inFile += "-i" if list: inFile += "-l" cwd = wgetcwd() @@ -214,7 +216,7 @@ def copyReportToGold(inFile, ignore, list): print "Recreating gold report from results." inFile = imageName(inFile) if ignore: - inFile += "-u" + inFile += "-i" if list: inFile += "-l" cwd = wgetcwd() @@ -238,7 +240,7 @@ def testCompareReports(inFile, ignore, list): print "Comparing report to golden report." name = imageName(inFile) if ignore: - name += "-u" + name += "-i" if list: name += "-l" goldReport = os.path.join("./",goldDir,name,"report.html") @@ -343,7 +345,7 @@ def wabspath(inFile): def copyLogs(inFile, ignore, list): name = imageName(inFile) if ignore: - name +="-u" + name +="-i" if list: name+="-l" logDir = os.path.join("..","build","test","qa-functional","work","userdir0","var","log") @@ -363,13 +365,16 @@ def testFile(image, rebuild, ignore, list): def usage(): usage = "\ - Usage: ./regression.py [-i FILE] [OPTIONS] \n\n\ + Usage: ./regression.py [-s FILE] [OPTIONS] \n\n\ Run the RegressionTest.java file, and compare the result with a gold standard \n\n\ When the -i flag is set, this script only tests the image given by FILE.\n\ By default, it tests every image in ./input/\n\n\ An indexed NSRL database is expected at ./input/nsrl.txt-md5.idx,\n\ and an indexed notable hash database at ./input/notablehashes.txt-md5.idx\n\ In addition, any keywords to search for must be in ./input/notablekeywords.xml\n\n\ + When the -l flag is set, the script looks for a config.xml file of the given name\n\ + where images are stored. For usage notes please see the example config.xml in\n\ + the /script folder.\ Options:\n\n\ -r, --rebuild\t\tRebuild the gold standards from the test results for each image\n\n\ -u, --ignore\t\tIgnore unallocated space while ingesting" @@ -382,32 +387,33 @@ def main(): list = False test = True argi = 1 - Config = None #file pointed to by -l - imgListB = [] #list of legal images from imgListA generated by main() + Config = None #file pointed to by --list + imgListB = [] #list of legal images from config cwd = wgetcwd() while argi < len(sys.argv): arg = sys.argv[argi] - if arg == "-i" and argi+1 < len(sys.argv): + if arg == "-s" and argi+1 < len(sys.argv): #check for single single = True argi+=1 image = sys.argv[argi] print "Running on single image: " + image - if arg == "-l" or arg == "--list": + if arg == "-l" or arg == "--list": #check for config file list = True argi+=1 #check for file in ./ if(os.path.isfile(os.path.join("./", sys.argv[argi]))): Config = parse(os.path.join("./", sys.argv[argi])) + #else check if it is a specified path elif (os.path.exists(wabspath(sys.argv[argi]))): Config = parse(sys.argv[argi]) else: print sys.argv[argi] print wabspath(sys.argv[argi]) markError("Ran with " + arg +" but no such file exists", arg) - elif (arg == "--rebuild") or (arg == "-r"): + elif (arg == "--rebuild") or (arg == "-r"): #check for rebuild flag rebuild = True print "Running in REBUILD mode" - elif (arg == "--ignore") or (arg == "-u"): + elif (arg == "--ignore") or (arg == "-i"): #check for ignore flag ignore = True print "Ignoring unallocated space" else: @@ -419,18 +425,12 @@ def main(): if list: listImages = [] errors = 0 - global inDir - out = Config.getElementsByTagName("indir")[0].getAttribute("value").encode() - # proc = subprocess.Popen(("cygpath", "-u", Config.getElementsByTagName("indir")[0].getAttribute("value")), stdout=subprocess.PIPE) - # out,err = proc.communicate() - # out.rstrip() - # print "number of slashs is:" + str(out.count("/")) - # out.replace("/", "\\\\") - # print "out is" + out + global inDir + out = Config.getElementsByTagName("indir")[0].getAttribute("value").encode() #there should only be one indir element in the config inDir = out for element in Config.getElementsByTagName("image"): elem = element.getAttribute("value").encode() - proc2 = subprocess.Popen(("cygpath", "-u", elem), stdout=subprocess.PIPE) + proc2 = subprocess.Popen(("cygpath", "-i", elem), stdout=subprocess.PIPE) out2,err = proc2.communicate() out2 = out2.rstrip() if(os.path.exists(out2) and os.path.isfile(out2)): @@ -438,7 +438,7 @@ def main(): else: print out2 + " is not a valid path or is not an image" errors+=1 - print "error total: " + str(errors) + print "Illegal files specified: " + str(errors) for image in listImages: testFile(image, rebuild, ignore, list) elif test: diff --git a/docs/doxygen/regressionTesting.dox b/docs/doxygen/regressionTesting.dox index d76f2086d8..19005cb05c 100644 --- a/docs/doxygen/regressionTesting.dox +++ b/docs/doxygen/regressionTesting.dox @@ -60,27 +60,39 @@ https://github.com/sleuthkit/autopsy/blob/master/BUILDING.txt 3) Run regression tests - 3a) From the Cygwin terminal, navigate to the /script folder and run "./regression.py". The script will automatically begin Autopsy and run ingestion and analysis on all the images from the /input directory, and will close when finished. The Cygwin terminal will print out whether or not errors were encountered at the end of each image's test. + 3a) From the Cygwin terminal, navigate to the /script folder and run "./regression.py". The script will automatically begin Autopsy and run ingestion and analysis on all the images from the ./input directory, and will close when finished. The Cygwin terminal will print out whether or not errors were encountered at the end of each image's test. + + 3b) It is possible to run testing on images that do not reside within ./input. To accomplish this, a configuration file is specified as a flag when run. Please see 4 below. \section regression_test_update Options - Currently, the script supports three argument parameters: + Currently, the script supports four argument parameters: - -i {imgname} : runs the test with a single image as specified by {imgname}. Must include the path, it does not read from input. - -r or --rebuild : runs in REBUILD mode, see 4 below - -u or --ignore : runs ignoring unallocated space. Useful for determining framework is operational. Appends "-u" to the output and gold folders for distinguishing them from others. Will automatically compare the right reports. + -s {imgname} : runs the test with a single image as specified by {imgname}. Must include the path, it does not read from input. + -r or --rebuild : runs in REBUILD mode, see 5 below + -l {config} or --list {config} : runs with the specified {config} file. If you do not specify a path to the file, it searches for it under ./. It will attempt to search for the file by the specified path name if surrounded in quotes. + -u or --ignore : runs ignoring unallocated space. Useful for determining framework is operational. Appends "-u" to the output and gold folders for distinguishing them from others. Will automatically compare the right reports. These can be run in any combination. -4) OPTIONAL: Update the standards databases - 4a) From the Cygwin terminal, navigate to +\section regression_test_config Running regression tests with a config file + +4) Running with a configuration file + + The config file is any user defined XML file that has the following properties: 1 element with the tag "indir" to specify where the hash databases are, and any number of elements with the tag "image", where the program will look for external images. These images do not have to be held locally. They may be on network shares, or other attached drives. We have included a sample "config.xml" in the script folder as a guide. Note that the config file may be named anything. + + + +5) OPTIONAL: Update the standards databases + + 5a) From the Cygwin terminal, navigate to autopsy/Testing/script - 4b) run "./regression.py -r", The script will automatically delete pre-existing standards.db files and generate the updated ones in the proper locations (/script/gold/{name of image}). + 5b) run "./regression.py -r", The script will automatically delete pre-existing standards.db files and generate the updated ones in the proper locations (/script/gold/{name of image}). Running in -r will also generate a golden report file built from the image. Normal runs of regression.py compare their generated report against the golden one, and report any differences in the file, ignoring the timestamp.