regression.py : changed the single command to -s, and the ignore command to -i to reduce confusion

regressionTesting.dox : updated the documentation to reflect yesterdays update

Other:
Uploaded an example XML config file

Testing/script/config.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="ASCII"?>
+<!-- This file is an example config file for regression.py. 
+ the following tags are mandatory:
+ 1 "indir" tag that specifies where the program can find the hash databases
+ any number of "image" tags that specify where the images are stored. They do not have to be local.
+ invalid paths and images are ignored by the tester. 
+
+ in this file, the first three tags are accepted by the tester, while the last two are disregarded as errors-->
+
+<Properties>
+    <indir name="indir" value="C:\Users\0xNF\autopsyA\Testing\script\important_files" />
+    <image name="image1" value="P:\shared\Testing\script\input\64mb2.img"/>
+    <image name="image2" value="C:\Users\0xNF\Documents\backups	\Testing\script\input\MC1001-0001.E01"/>
+    <image name="image3" value="should be error"/>
+    <image name="image4" value="C:\Invalid\Path\Does\Not\Exist.img"/>
+</Properties>

Testing/script/regression.py

@@ -12,7 +12,7 @@ from xml.dom.minidom import parse, parseString
 
 
 #  Last modified 7/17/12 @5pm
-#  Usage: ./regression.py [-i FILE] OR [-l CONFIG]  [OPTIONS]
+#  Usage: ./regression.py [-s FILE] OR [-l CONFIG]  [OPTIONS]
 #  Run the RegressionTest.java file, and compare the result with a gold standard
 #  When the -i flag is set, this script only tests the image given by FILE.
 #  An indexed NSRL database is expected at ./input/nsrl.txt-md5.idx,
@@ -23,7 +23,7 @@ from xml.dom.minidom import parse, parseString
 #  the /script folder.
 #    Options: 
 #    -r, --rebuild      Rebuild the gold standards from the test results for each image
-#    -u, --ignore       Ignores unallocated space when ingesting. Faster, but less accurate results.
+#    -i, --ignore       Ignores unallocated space when ingesting. Faster, but less accurate results.
   
 
 hadErrors = False # If any of the tests failed
@@ -41,8 +41,10 @@ def testAddImageIngest(inFile, ignoreUnalloc, list):
 
   # Set up case directory path
   testCaseName = imageName(inFile)
+  
+  #check for flags to append to folder name
   if ignoreUnalloc:
-    testCaseName+="-u"
+    testCaseName+="-i"
   if list:
     testCaseName+="-l"
   if os.path.exists(os.path.join(outDir,testCaseName)):
@@ -128,7 +130,7 @@ def testCompareToGold(inFile, ignore):
 
   name = imageName(inFile)
   if ignore:
-   name += ("-u")
+   name += ("-i")
   cwd = wgetcwd()
   goldFile = os.path.join("./",goldDir,name,"standard.db")  
   testFile = os.path.join("./",outDir,name,"AutopsyTestCase","autopsy.db")
@@ -189,7 +191,7 @@ def clearGoldDir(inFile, ignore, list):
   cwd = wgetcwd()
   inFile = imageName(inFile)
   if ignore:
-    inFile += "-u"
+    inFile += "-i"
   if list:
     inFile += "-l"
   if os.path.exists(os.path.join(cwd,goldDir,inFile)):
@@ -201,7 +203,7 @@ def copyTestToGold(inFile, ignore, list):
   print "Recreating gold standard from results."
   inFile = imageName(inFile)
   if ignore:
-    inFile += "-u"
+    inFile += "-i"
   if list:
     inFile += "-l"
   cwd = wgetcwd()
@@ -214,7 +216,7 @@ def copyReportToGold(inFile, ignore, list):
   print "Recreating gold report from results."
   inFile = imageName(inFile)
   if ignore:
-    inFile += "-u"
+    inFile += "-i"
   if list:
     inFile += "-l"
   cwd = wgetcwd()
@@ -238,7 +240,7 @@ def testCompareReports(inFile, ignore, list):
   print "Comparing report to golden report."
   name = imageName(inFile)
   if ignore:
-    name += "-u"
+    name += "-i"
   if list:
     name += "-l"
   goldReport = os.path.join("./",goldDir,name,"report.html")  
@@ -343,7 +345,7 @@ def wabspath(inFile):
 def copyLogs(inFile, ignore, list):
   name = imageName(inFile)
   if ignore:
-   name +="-u"
+   name +="-i"
   if list:
     name+="-l"
   logDir = os.path.join("..","build","test","qa-functional","work","userdir0","var","log")
@@ -363,13 +365,16 @@ def testFile(image, rebuild, ignore, list):
       
 def usage():
   usage = "\
-  Usage: ./regression.py [-i FILE] [OPTIONS] \n\n\
+  Usage: ./regression.py [-s FILE] [OPTIONS] \n\n\
   Run the RegressionTest.java file, and compare the result with a gold standard \n\n\
   When the -i flag is set, this script only tests the image given by FILE.\n\
   By default, it tests every image in ./input/\n\n\
   An indexed NSRL database is expected at ./input/nsrl.txt-md5.idx,\n\
   and an indexed notable hash database at ./input/notablehashes.txt-md5.idx\n\
   In addition, any keywords to search for must be in ./input/notablekeywords.xml\n\n\
+  When the -l flag is set, the script looks for a config.xml file of the given name\n\
+  where images are stored. For usage notes please see the example config.xml in\n\
+  the /script folder.\
     Options:\n\n\
     -r, --rebuild\t\tRebuild the gold standards from the test results for each image\n\n\
     -u, --ignore\t\tIgnore unallocated space while ingesting"
@@ -382,32 +387,33 @@ def main():
   list = False
   test = True
   argi = 1
-  Config = None   #file pointed to by -l
+  Config = None   #file pointed to by --list
-  imgListB = []   #list of legal images from imgListA generated by main()
+  imgListB = []   #list of legal images from config
   cwd = wgetcwd()
   while argi < len(sys.argv):
       arg = sys.argv[argi]
-      if arg == "-i" and argi+1 < len(sys.argv):
+      if arg == "-s" and argi+1 < len(sys.argv): #check for single
           single = True
           argi+=1
           image = sys.argv[argi]
           print "Running on single image: " + image
-      if arg == "-l" or arg == "--list":
+      if arg == "-l" or arg == "--list":    #check for config file
             list = True
             argi+=1
             #check for file in ./
             if(os.path.isfile(os.path.join("./", sys.argv[argi]))):
                  Config = parse(os.path.join("./", sys.argv[argi]))
+            #else check if it is a specified path
             elif (os.path.exists(wabspath(sys.argv[argi]))):
                 Config = parse(sys.argv[argi])
             else:
                 print sys.argv[argi]
                 print wabspath(sys.argv[argi])
                 markError("Ran with " + arg +" but no such file exists", arg)
-      elif (arg  == "--rebuild") or (arg == "-r"):
+      elif (arg  == "--rebuild") or (arg == "-r"):  #check for rebuild flag
           rebuild = True
           print "Running in REBUILD mode"
-      elif (arg == "--ignore") or (arg == "-u"):
+      elif (arg == "--ignore") or (arg == "-i"):    #check for ignore flag
           ignore = True
           print "Ignoring unallocated space"
       else:
@@ -419,18 +425,12 @@ def main():
   if list:
     listImages = []
     errors = 0
-    global inDir
+    global inDir    
-    out = Config.getElementsByTagName("indir")[0].getAttribute("value").encode()
+    out = Config.getElementsByTagName("indir")[0].getAttribute("value").encode() #there should only be one indir element in the config
-    # proc = subprocess.Popen(("cygpath", "-u", Config.getElementsByTagName("indir")[0].getAttribute("value")), stdout=subprocess.PIPE)
-    # out,err = proc.communicate()
-    # out.rstrip()
-    # print "number of slashs is:" + str(out.count("/"))
-    # out.replace("/", "\\\\")
-    # print "out is" + out
     inDir = out
     for element in Config.getElementsByTagName("image"):
         elem = element.getAttribute("value").encode()
-        proc2 = subprocess.Popen(("cygpath", "-u", elem), stdout=subprocess.PIPE)
+        proc2 = subprocess.Popen(("cygpath", "-i", elem), stdout=subprocess.PIPE)
         out2,err = proc2.communicate()
         out2 = out2.rstrip()
         if(os.path.exists(out2) and os.path.isfile(out2)):
@@ -438,7 +438,7 @@ def main():
         else:
             print out2 + " is not a valid path or is not an image"
             errors+=1
-    print "error total: " + str(errors)
+    print "Illegal files specified: " + str(errors)
     for image in listImages:
         testFile(image, rebuild, ignore, list)
   elif test:

docs/doxygen/regressionTesting.dox

@@ -60,27 +60,39 @@ https://github.com/sleuthkit/autopsy/blob/master/BUILDING.txt
 
 3) Run regression tests
 
-	3a) From the Cygwin terminal, navigate to the /script folder and run "./regression.py". The script will automatically begin Autopsy and run ingestion and analysis on all the images from the /input directory, and will close when finished. The Cygwin terminal will print out whether or not errors were encountered at the end of each image's test.
+	3a) From the Cygwin terminal, navigate to the /script folder and run "./regression.py". The script will automatically begin Autopsy and run ingestion and analysis on all the images from the ./input directory, and will close when finished. The Cygwin terminal will print out whether or not errors were encountered at the end of each image's test.
+
+	3b) It is possible to run testing on images that do not reside within ./input. To accomplish this, a configuration file is specified as a flag when run. Please see 4 below.
 
 
 
 \section regression_test_update Options
 		
-		Currently, the script supports three argument parameters:
+		Currently, the script supports four argument parameters:
 
-			-i {imgname}	: runs the test with a single image as specified by {imgname}. Must include the path, it does not read from input.
+			-s {imgname}			: runs the test with a single image as specified by {imgname}. Must include the path, it does not read from input.
-			-r or --rebuild	: runs in REBUILD mode, see 4 below
+			-r or --rebuild			: runs in REBUILD mode, see 5 below
-			-u or --ignore	: runs ignoring unallocated space. Useful for determining framework is operational. Appends "-u" to the output and gold folders for distinguishing them from others. Will automatically compare the right reports.
+			-l {config} or --list {config}	: runs with the specified {config} file. If you do not specify a path to the file, it searches for it under ./. It will attempt to search for the file by the specified path name if surrounded in quotes. 
+			-u or --ignore			: runs ignoring unallocated space. Useful for determining framework is operational. Appends "-u" to the output and gold folders for distinguishing them from others. Will automatically compare the right reports.
 
 		These can be run in any combination.
 
-4) OPTIONAL: Update the standards databases
 
-	4a) From the Cygwin terminal, navigate to 
+\section regression_test_config Running regression tests with a config file
+
+4) Running with a configuration file
+	
+	The config file is any user defined XML file that has the following properties: 1 element with the tag "indir" to specify where the hash databases are, and any number of elements with the tag "image", where the program will look for external images. These images do not have to be held locally. They may be on network shares, or other attached drives. We have included a sample "config.xml" in the script folder as a guide. Note that the config file may be named anything.
+
+	
+
+5) OPTIONAL: Update the standards databases
+
+	5a) From the Cygwin terminal, navigate to 
 
 		autopsy/Testing/script
 
-	4b)  run "./regression.py -r", The script will automatically delete pre-existing standards.db files and generate the updated ones in the proper locations (/script/gold/{name of image}).
+	5b)  run "./regression.py -r", The script will automatically delete pre-existing standards.db files and generate the updated ones in the proper locations (/script/gold/{name of image}).
 
 Running in -r will also generate a golden report file built from the image. Normal runs of regression.py compare their generated report against the golden one, and report any differences in the file, ignoring the timestamp.