Fixing issue with git merger.

Testing/script/regression.py

@@ -7,6 +7,7 @@ import os.path
 import shutil
 import time
 
+# Last modified 7/13/12 3@ pm
 #  Usage: ./regression.py [-i FILE] [OPTIONS]
 #  Run the RegressionTest.java file, and compare the result with a gold standard
 #  When the -i flag is set, this script only tests the image given by FILE.
@@ -16,22 +17,25 @@ import time
 #  In addition, any keywords to search for must be in ./input/notablekeywords.xml
 #    Options:
 #    -r, --rebuild      Rebuild the gold standards from the test results for each image
+#    -u, --ignore
 
 hadErrors = False # If any of the tests failed
 results = {}      # Dictionary in which to store map ({imgname}->errors)
 goldDir = "gold"  # Directory for gold standards (files should be ./gold/{imgname}/standard.db)
 inDir = "input"   # Image files, hash dbs, and keywords.
 # Results will be in ./output/{datetime}/{imgname}/
-outDir = os.path.join("output",time.strftime("%Y.%m.%d-%H.%M")) 
+outDir = os.path.join("output",time.strftime("%Y.%m.%d-%H.%M"))
 
 
 # Run ingest on all the images in 'input', using notablekeywords.xml and notablehashes.txt-md5.idx
-def testAddImageIngest(inFile):
+def testAddImageIngest(inFile, ignoreUnalloc):
   print "================================================"
   print "Ingesting Image: " + inFile
 
   # Set up case directory path
   testCaseName = imageName(inFile)
+  if ignoreUnalloc:
+    testCaseName+="-u"
   if os.path.exists(os.path.join(outDir,testCaseName)):
     shutil.rmtree(os.path.join(outDir,testCaseName))
   os.makedirs(os.path.join(outDir,testCaseName))
@@ -40,9 +44,24 @@ def testAddImageIngest(inFile):
 
   cwd = wgetcwd()
   testInFile = wabspath(inFile)
+  # NEEDS windows path (backslashes) for .E00 images to work
+  testInFile = testInFile.replace("/", "\\")
   knownBadPath = os.path.join(cwd,inDir,"notablehashes.txt-md5.idx")
+  knownBadPath = knownBadPath.replace("/", "\\")
   keywordPath = os.path.join(cwd,inDir,"notablekeywords.xml")
+  keywordPath = keywordPath.replace("/", "\\")
   nsrlPath = os.path.join(cwd,inDir,"nsrl.txt-md5.idx")
+  nsrlPath = nsrlPath.replace("/", "\\")
+
+  antlog = os.path.join(cwd,outDir,testCaseName,"antlog.txt")
+  antlog = antlog.replace("/", "\\")
+
+  timeout = 24 * 60 * 60 * 1000    # default of 24 hours, just to be safe
+  size = getImageSize(inFile)      # get the size in bytes
+  timeout = (size / 1000) / 1000   # convert to MB
+  timeout = timeout * 1000         # convert sec to ms
+  timeout = timeout * 1.5          # add a little extra umph
+  timeout = timeout * 25       # decided we needed A LOT extra to be safe
 
   # set up ant target
   args = ["ant"]
@@ -51,13 +70,16 @@ def testAddImageIngest(inFile):
   args.append(os.path.join("..","build.xml"))
   args.append("regression-test")
   args.append("-l")
-  args.append(os.path.join(cwd,outDir,testCaseName,"antlog.txt"))
+  args.append(antlog)
-  args.append("-Dimg_path=" + testInFile.replace("/","\\"))
+  args.append("-Dimg_path=" + testInFile)
-  args.append("-Dknown_bad_path=" + knownBadPath.replace("/","\\"))
+  args.append("-Dknown_bad_path=" + knownBadPath)
-  args.append("-Dkeyword_path=" + keywordPath.replace("/","\\"))
+  args.append("-Dkeyword_path=" + keywordPath)
-  args.append("-Dnsrl_path=" + nsrlPath.replace("/","\\"))
+  args.append("-Dnsrl_path=" + nsrlPath)
-  args.append("-Dgold_path=" + os.path.join(cwd,goldDir).replace("/","\\"))
+  args.append("-Dgold_path=" + os.path.join(cwd,goldDir).replace("/", "\\"))
-  args.append("-Dout_path=" + os.path.join(cwd,outDir,testCaseName).replace("/","\\"))
+  args.append("-Dout_path=" + os.path.join(cwd,outDir,testCaseName).replace("/", "\\"))
+  args.append("-Dignore_unalloc=" + "%s" % ignoreUnalloc)
+  args.append("-Dtest.timeout=" + str(timeout))
+
 
   # print the ant testing command
   print "CMD: " + " ".join(args)
@@ -68,13 +90,27 @@ def testAddImageIngest(inFile):
   #fnull.close();
   subprocess.call(args)
 
-def testCompareToGold(inFile):
+def getImageSize(inFile):
+  name = imageName(inFile)
+  path = os.path.join(".",inDir)
+  size = 0
+  for files in os.listdir(path):
+    filename = os.path.splitext(files)[0]
+    if filename == name:
+      filepath = os.path.join(path, files)
+      if not os.path.samefile(filepath, inFile):
+        size += os.path.getsize(filepath)
+  size += os.path.getsize(inFile)
+  return size
+
+def testCompareToGold(inFile, ignore):
   print "-----------------------------------------------"
   print "Comparing results for " + inFile + " with gold."
 
   name = imageName(inFile)
+  if ignore:
+   name += ("-u")
   cwd = wgetcwd()
-  
   goldFile = os.path.join("./",goldDir,name,"standard.db")  
   testFile = os.path.join("./",outDir,name,"AutopsyTestCase","autopsy.db")
   if os.path.isfile(goldFile) == False:
@@ -130,41 +166,54 @@ def testCompareToGold(inFile):
   else:
       print("Object counts match!")
 
-def clearGoldDir(inFile):
+def clearGoldDir(inFile, ignore):
   cwd = wgetcwd()
   inFile = imageName(inFile)
+  if ignore:
+    inFile += "-u"
   if os.path.exists(os.path.join(cwd,goldDir,inFile)):
     shutil.rmtree(os.path.join(cwd,goldDir,inFile))
   os.makedirs(os.path.join(cwd,goldDir,inFile))
 
-def copyTestToGold(inFile): 
+def copyTestToGold(inFile, ignore): 
   print "------------------------------------------------"
   print "Recreating gold standard from results."
   inFile = imageName(inFile)
+  if ignore:
+    inFile += "-u"
   cwd = wgetcwd()
   goldFile = os.path.join("./",goldDir,inFile,"standard.db")
   testFile = os.path.join("./",outDir,inFile,"AutopsyTestCase","autopsy.db")
   shutil.copy(testFile, goldFile)
 
-def copyReportToGold(inFile): 
+def copyReportToGold(inFile, ignore): 
   print "------------------------------------------------"
   print "Recreating gold report from results."
   inFile = imageName(inFile)
+  if ignore:
+    inFile += "-u"
   cwd = wgetcwd()
   goldReport = os.path.join("./",goldDir,inFile,"report.html")
   testReportPath = os.path.join("./",outDir,inFile,"AutopsyTestCase","Reports")
   # Because Java adds a timestamp to the report file, one can't call it
   # directly, so one must get a list of files in the dir, which are only
   # reports, then filter for the .html report
+  testReport = None
   for files in os.listdir(testReportPath):
     if files.endswith(".html"): # Get the HTML one
       testReport = os.path.join("./",outDir,inFile,"AutopsyTestCase","Reports",files)
-  shutil.copy(testReport, goldReport)
+  if testReport is None:
+    markError("No test report exists", inFile)
+    return
+  else:
+    shutil.copy(testReport, goldReport)
 
-def testCompareReports(inFile):
+def testCompareReports(inFile, ignore):
   print "------------------------------------------------"
   print "Comparing report to golden report."
   name = imageName(inFile)
+  if ignore:
+    name += "-u"
   goldReport = os.path.join("./",goldDir,name,"report.html")  
   testReportPath = os.path.join("./",outDir,name,"AutopsyTestCase","Reports")
   # Because Java adds a timestamp to the report file, one can't call it
@@ -255,22 +304,27 @@ def wabspath(inFile):
     out,err = proc.communicate()
     return out.rstrip()
 
-def copyLogs(inFile):
+def copyLogs(inFile, ignore):
+  name = imageName(inFile)
+  if ignore:
+   name +="-u"
   logDir = os.path.join("..","build","test","qa-functional","work","userdir0","var","log")
-  shutil.copytree(logDir,os.path.join(outDir,imageName(inFile),"logs"))
+  shutil.copytree(logDir,os.path.join(outDir,name,"logs"))
 
-def testFile(image, rebuild):
+def testFile(image, rebuild, ignore):
   if imageType(image) != ImgType.UNKNOWN:
-    testAddImageIngest(image)
+    if ignore:
-    #print imageName(image)
+      testAddImageIngest(image, True)
-    copyLogs(image)
-    if rebuild:
-      clearGoldDir(image)
-      copyTestToGold(image)
-      copyReportToGold(image)
     else:
-      testCompareToGold(image)
+      testAddImageIngest(image, False)
-      testCompareReports(image)
+    copyLogs(image, ignore)
+    if rebuild:
+      clearGoldDir(image, ignore)
+      copyTestToGold(image, ignore)
+      copyReportToGold(image, ignore)
+    else:
+      testCompareToGold(image, ignore)
+      testCompareReports(image, ignore)
 
 def usage() :
   usage = "\
@@ -282,12 +336,14 @@ def usage() :
   and an indexed notable hash database at ./input/notablehashes.txt-md5.idx\n\
   In addition, any keywords to search for must be in ./input/notablekeywords.xml\n\n\
     Options:\n\n\
-    -r, --rebuild\t\tRebuild the gold standards from the test results for each image"
+    -r, --rebuild\t\tRebuild the gold standards from the test results for each image\n\n\
+    -u, --nounalloc\t\tIgnore unallocated space while ingesting"
   return usage
 
 def main():
   rebuild = False
   single = False
+  ignore = False
   test = True
   argi = 1
   while argi < len(sys.argv):
@@ -300,15 +356,18 @@ def main():
       elif (arg  == "--rebuild") or (arg == "-r"):
           rebuild = True
           print "Running in REBUILD mode"
+      elif (arg == "--nounalloc") or (arg == "-u"):
+          ignore = True
+          print "Ignoring unallocated space"
       else:
           test = False
           print usage()
       argi+=1
   if single:
-    testFile(image, rebuild)
+    testFile(image, rebuild, ignore)
   elif test:
     for inFile in os.listdir(inDir):
-      testFile(os.path.join(inDir,inFile), rebuild)
+      testFile(os.path.join(inDir,inFile), rebuild, ignore)
 
   if hadErrors == True:
     print "**********************************************"
@@ -323,4 +382,4 @@ def main():
       print("\t%s" % errString)
 
 if __name__ == "__main__":
-  main()
+  main()