Infinite Testing changes

Supports the new HTML report format for comparing gold reports.
Added a modified java test package that has some changes specific to the infinite testing.

Testing/script/regression.py

@@ -14,6 +14,7 @@ from sys import platform as _platform
 import time
 import traceback
 import xml
+from time import localtime, strftime
 from xml.dom.minidom import parse, parseString
 
 #
@@ -38,7 +39,7 @@ from xml.dom.minidom import parse, parseString
 # but do not overwrite any existing variables as they are used frequently.
 #
 
-
+Day = 0
 
 #-------------------------------------------------------------#
 # Parses argv and stores booleans to match command line input #
@@ -56,6 +57,7 @@ class Args:
         self.verbose = False
         self.exception = False
         self.exception_string = ""
+        self.mugen = False
     
     def parse(self):
         sys.argv.pop(0)
@@ -106,6 +108,9 @@ class Args:
                     self.exception_string = arg
                 except:
                     printerror("Error: No exception string given.")
+            elif(arg == "-m" or arg == "--mugen"):
+                printout("Running in mugen mode. Will loop through config file until interrupted.")
+                self.mugen = True
             elif arg == "-h" or arg == "--help":
                 printout(usage())
                 return False
@@ -154,6 +159,8 @@ class TestAutopsy:
         self.ingest_messages = 0
         self.indexed_files = 0
         self.indexed_chunks = 0
+        # Infinite Testing info
+        timer = 0
         
         # Set the timeout to something huge
         # The entire tester should not timeout before this number in ms
@@ -184,7 +191,9 @@ class TestAutopsy:
         string = ""
         for arg in self.ant:
             string += (arg + " ")
-        return string
+        return string    
+
+     
         
     def reset(self):
         # Logs:
@@ -214,6 +223,7 @@ class TestAutopsy:
         # And it's very buggy, so we're being careful
         self.timeout = 24 * 60 * 60 * 1000 * 1000
         self.ant = []
+       
 
 
 
@@ -328,10 +338,13 @@ class Database:
 #      Main testing functions      #
 #----------------------------------#
 
+
+
 # Iterates through an XML configuration file to find all given elements        
 def run_config_test(config_file):
     try:
         parsed = parse(config_file)
+        counts = {}
         if parsed.getElementsByTagName("indir"):
             case.input_dir = parsed.getElementsByTagName("indir")[0].getAttribute("value").encode().decode("utf-8")
         if parsed.getElementsByTagName("global_csv"):
@@ -346,13 +359,36 @@ def run_config_test(config_file):
         html_add_images(values)
         
         # Run the test for each file in the configuration
-        for element in parsed.getElementsByTagName("image"):
+        global args
-            value = element.getAttribute("value").encode().decode("utf-8")
+        if(args.mugen):
-            if file_exists(value):
+            #set all times an image has been processed to 0
-                run_test(value)
+            for element in parsed.getElementsByTagName("image"):
-            else:
+                value = element.getAttribute("value").encode().decode("utf-8")
-                printerror("Warning: Image file listed in the configuration does not exist:")
+                counts[value] = 0
-                printerror(value + "\n")
+            #Begin infiniloop
+            while(True):
+                for elem in counts.keys():
+                    if(newDay()):
+                        setDay()
+                        gitPull("sleuthkit")
+                        vsBuild()
+                        gitPull("autopsy")
+                        antBuild("datamodel", False)
+                        antBuild("autopsy", True)
+                    if file_exists(elem):
+                        counts[elem]+=1
+                        run_test(elem, counts[elem])
+                    else:
+                        printerror("Warning: Image file listed in the configuration does not exist:")
+                        printerror(elem + "\n")
+        else:
+            for img in values:  
+                if file_exists(img):
+                    run_test(img, 0)
+                else:
+                    printerror("Warning: Image file listed in configuration does not exist:")
+                    printrttot(value + "\n")
+           
     except Exception as e:
         printerror("Error: There was an error running with the configuration file.")
         printerror(str(e) + "\n")
@@ -360,7 +396,7 @@ def run_config_test(config_file):
 
 # Runs the test on the single given file.
 # The path must be guarenteed to be a correct path.
-def run_test(image_file):
+def run_test(image_file, count):
     if not image_type(image_file) != IMGTYPE.UNKNOWN:
         printerror("Error: Image type is unrecognized:")
         printerror(image_file + "\n")
@@ -368,7 +404,7 @@ def run_test(image_file):
         
     # Set the case to work for this test
     case.image_file = image_file
-    case.image_name = case.get_image_name(image_file)
+    case.image_name = case.get_image_name(image_file) + "(" + str(count) + ")"
     case.antlog_dir = make_local_path(case.output_dir, case.image_name, "antlog.txt")
     case.known_bad_path = make_path(case.input_dir, "notablehashes.txt-md5.idx")
     case.keyword_path = make_path(case.input_dir, "notablekeywords.xml")
@@ -377,7 +413,6 @@ def run_test(image_file):
     logging.debug("--------------------")
     logging.debug(case.image_name)
     logging.debug("--------------------")
-    
     run_ant()
     time.sleep(2) # Give everything a second to process
     
@@ -436,7 +471,6 @@ def run_ant():
     os.makedirs(test_case_path)
     if not dir_exists(make_local_path("gold")):
         os.makedirs(make_local_path("gold"))
-    
     case.ant = ["ant"]
     case.ant.append("-q")
     case.ant.append("-f")
@@ -451,6 +485,7 @@ def run_ant():
     case.ant.append("-Dgold_path=" + make_local_path(case.gold))
     case.ant.append("-Dout_path=" + make_local_path(case.output_dir, case.image_name))
     case.ant.append("-Dignore_unalloc=" + "%s" % args.unallocated)
+    case.ant.append("-Dmugen_mode" + str(args.mugen))
     case.ant.append("-Dtest.timeout=" + str(case.timeout))
     
     printout("Ingesting Image:\n" + case.image_file + "\n")
@@ -508,12 +543,20 @@ def rebuild():
         errors.append(str(e) + "\n")
     
     # Rebuild the HTML report
-    html_path = make_local_path(case.output_dir, case.image_name,
+    htmlfolder = ""
-                                  "AutopsyTestCase", "Reports")
+    for fs in os.listdir(os.path.join(os.getcwd(),case.output_dir, case.image_name, "AutopsyTestCase", "Reports")):
-    try:     
+        if os.path.isdir(os.path.join(os.getcwd(), case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)):
-        html_from = get_file_in_dir(html_path, ".html")
+            htmlfolder = fs
-        html_to = make_local_path(case.gold, case.image_name, "standard.html")
+    autopsy_html_path = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)
-        copy_file(html_from, html_to)
+    
+    
+    #html_path = make_local_path(case.output_dir, case.image_name,
+    #                             "AutopsyTestCase", "Reports")
+    try:
+        os.makedirs(os.path.join(os.getcwd(), case.gold, case.image_name, htmlfolder))
+        for file in os.listdir(autopsy_html_path):
+            html_to = make_local_path(case.gold, case.image_name, file)
+            copy_file(get_file_in_dir(autopsy_html_path, file), html_to)
     except FileNotFoundException as e:
         errors.append(e.error)
     except Exception as e:
@@ -584,11 +627,17 @@ def compare_to_gold_db():
 # Using the global case's variables, compare the html report file made by
 # the regression test against the gold standard html report
 def compare_to_gold_html():
-    gold_html_file = make_local_path(case.gold, case.image_name, "standard.html")
+    gold_html_file = make_local_path(case.gold, case.image_name, "index.html")
-    autopsy_html_path = make_local_path(case.output_dir, case.image_name,
+    htmlfolder = ""
-                                        "AutopsyTestCase", "Reports")
+    for fs in os.listdir(os.path.join(os.getcwd(),case.output_dir, case.image_name, "AutopsyTestCase", "Reports")):
+        if os.path.isdir(os.path.join(os.getcwd(), case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)):
+            htmlfolder = fs
+    autopsy_html_path = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder) #, "AutopsyTestCase", "Reports", htmlfolder)
+    print autopsy_html_path                                
+    
+    
     try:
-        autopsy_html_file = get_file_in_dir(autopsy_html_path, ".html")
+        autopsy_html_file = get_file_in_dir(autopsy_html_path, "index.html")
                 
         if not file_exists(gold_html_file):
             printerror("Error: No gold html report exists at:")
@@ -598,13 +647,35 @@ def compare_to_gold_html():
             printerror("Error: No case html report exists at:")
             printerror(autopsy_html_file + "\n")
             return
-        
+        #Find all gold .html files belonging to this case
-        errors = []
+        ListGoldHTML = []
-        errors = compare_report_files(autopsy_html_file, gold_html_file)
+        for fs in os.listdir(os.path.join(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)):
+            if(fs.endswith(".html")):
+                ListGoldHTML.append(os.path.join(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, fs))
+        #Find all new .html files belonging to this case
+        ListNewHTML = []
+        for fs in os.listdir(os.path.join(case.gold, case.image_name)):
+            if (fs.endswith(".html")):
+                ListNewHTML.append(os.path.join(case.gold, case.image_name, fs))
+        #ensure both reports have the same number of files and are in the same order
+        if(len(ListGoldHTML) != len(ListNewHTML)):
+            printerror("The reports did not have the same number of files. One of the reports may have been corrupted")
+        else:
+            ListGoldHTML.sort()
+            ListNewHTML.sort()
+          
+        total = {"Gold": 0, "New": 0}
+        for x in range(0, len(ListGoldHTML)):
+            count = compare_report_files(ListGoldHTML[x], ListNewHTML[x])
+            total["Gold"]+=count[0]
+            total["New"]+=count[1]
         okay = "The test report matches the gold report."
+        errors=["Gold report had " + str(total["Gold"]) +" errors", "New report had " + str(total["New"]) + " errors."]
         print_report(errors, "REPORT COMPARISON", okay)
-        if not errors:
+        if total["Gold"] == total["New"]:
             case.report_passed = True
+        else:
+            printerror("The reports did not match each other.\n " + errors[0] +" and the " + errors[1])
     except FileNotFoundException as e:
         e.print_error()
     except DirNotFoundException as e:
@@ -1157,6 +1228,63 @@ def html_add_images(full_image_names):
 #         Helper functions         #
 #----------------------------------#
 
+def setDay():
+    global Day
+    Day = int(strftime("%d", localtime()))
+        
+def getLastDay():
+    return Day
+        
+def getDay():
+    return int(strftime("%d", localtime()))
+        
+def newDay():
+    return getLastDay() != getDay()
+
+#Pulls from git
+def gitPull(TskOrAutopsy):
+    global SYS
+    ccwd = ""
+    toPull = "http://www.github.com/sleuthkit/" + TskOrAutopsy
+    call = ["git", "pull", toPull]
+    if TskOrAutopsy == "sleuthkit":
+        ccwd = os.path.join("..", "..", "..", "TSK")
+    else:
+        ccwd = os.path.join("..", "..")
+    subprocess.call(call, stdout=subprocess.PIPE, cwd=ccwd)
+
+#Builds TSK as a win32 applicatiion
+def vsBuild():
+    #Please ensure that the current working directory is $autopsy/testing/script
+    vs = []
+    vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe")
+    vs.append(os.path.join("..", "..", "..","TSK", "win32", "Tsk-win.sln"))
+    vs.append("/p:configuration=release")
+    vs.append("/p:platform=win32")
+    vs.append("/t:rebuild")
+    print vs
+    subprocess.call(vs)
+    
+ 
+#Builds Autopsy or the Datamodel
+def antBuild(which, Build):
+    directory = os.path.join("..", "..")
+    ant = []
+    if which == "datamodel":
+        directory = os.path.join("..", "TSK", "bindings", "java")
+    ant.append("ant")
+    ant.append("-f")
+    ant.append(directory)
+    ant.append("clean")
+    if(Build):
+        ant.append("build")
+    else:
+        ant.append("dist")
+    subprocess.call(ant)
+        
+    
+#Watches clock and waits for current ingest to be done
+
 # Verifies a file's existance
 def file_exists(file):
     try:
@@ -1241,6 +1369,16 @@ def get_file_in_dir(dir, ext):
         raise FileNotFoundException(dir)
     except:
         raise DirNotFoundException(dir)
+        
+def find_file_in_dir(dir, name, ext):
+    try: 
+        for file in os.listdir(dir):
+            if file.startswith(name):
+                if file.endswith(ext):
+                    return make_path(dir, file)
+        raise FileNotFoundException(dir)
+    except:
+        raise DirNotFoundException(dir)
 
 # Compares file a to file b and any differences are returned
 # Only works with report html files, as it searches for the first <ul>
@@ -1254,15 +1392,12 @@ def compare_report_files(a_path, b_path):
     
     a_list = split(a, 50)
     b_list = split(b, 50)
-    exceptions = []
     if not len(a_list) == len(b_list):
-        exceptions.append("The reports do not match.")
+        ex = (len(a_list), len(b_list))
-        test = "The test HTML report has " + str(len(a_list)) + " segments."
+        return ex
-        gold = "The gold HTML report has " + str(len(b_list)) + " segments."
+    else: 
-        exceptions.append(test)
+        return (0, 0)
-        exceptions.append(gold)
+  
-    return exceptions
-
 # Split a string into an array of string of the given size
 def split(input, size):
     return [input[start:start+size] for start in range(0, len(input), size)]
@@ -1399,7 +1534,7 @@ def main():
                 printerror("Error: Image file does not exist at:")
                 printerror(args.single_file)
                 return
-            run_test(args.single_file)
+            run_test(args.single_file, 0)
         # If user has not selected a single file, and does not want to ignore
         #  the input directory, continue on to parsing ./input
         if (not args.single) and (not args.ignore):
@@ -1407,7 +1542,7 @@ def main():
                 # Make sure it's not a required hash/keyword file or dir
                 if (not required_input_file(file) and
                     not os.path.isdir(make_path(case.input_dir, file))):
-                    run_test(make_path(case.input_dir, file))
+                    run_test(make_path(case.input_dir, file), 0)
                
         write_html_foot()
 

Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java

@@ -25,12 +25,17 @@ import java.awt.Toolkit;
 import java.awt.image.BufferedImage;
 import java.io.File;
 import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.Random;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.imageio.ImageIO;
 import javax.swing.JDialog;
+import javax.swing.JTabbedPane;
 import javax.swing.JTextField;
 import junit.framework.Test;
 import junit.framework.TestCase;
@@ -42,6 +47,7 @@ import org.netbeans.jemmy.operators.JButtonOperator;
 import org.netbeans.jemmy.operators.JCheckBoxOperator;
 import org.netbeans.jemmy.operators.JDialogOperator;
 import org.netbeans.jemmy.operators.JFileChooserOperator;
+import org.netbeans.jemmy.operators.JTabbedPaneOperator;
 import org.netbeans.jemmy.operators.JTableOperator;
 import org.netbeans.jemmy.operators.JTextFieldOperator;
 import org.netbeans.junit.NbModuleSuite;
@@ -54,6 +60,7 @@ import org.sleuthkit.autopsy.ingest.IngestManager;
  * known_bad_path: Path to a database of known bad hashes
  * keyword_path: Path to a keyword list xml file
  * ignore_unalloc: Boolean whether to ignore unallocated space or not
+ * mugen_mode: whether or not this test will run certain keyword settings.
  * 
  * Without these properties set, the test will fail to run correctly.
  * To run this test correctly, you should use the script 'regression.py'
@@ -186,7 +193,7 @@ public class RegressionTest extends TestCase{
         WizardOperator wo = new WizardOperator("Add Image");
         JTableOperator jto = new JTableOperator(wo, 0);
         int row = jto.findCellRow("Keyword Search", 1, 0);
-        jto.clickOnCell(row, 1);
+        jto.clickOnCell(row, 1);        
         JButtonOperator jbo1 = new JButtonOperator(wo, "Advanced");
         jbo1.pushNoBlock();
     }
@@ -202,6 +209,14 @@ public class RegressionTest extends TestCase{
         jfco0.chooseFile(words);
         JCheckBoxOperator jcbo = new JCheckBoxOperator(jdo, "Enable for ingest", 0);
         jcbo.doClick();
+        if(Boolean.parseBoolean(System.getProperty("mugen_mode"))){
+            JTabbedPaneOperator jtpo = new JTabbedPaneOperator(jdo);
+            jtpo.selectPage("String Extraction");
+            JCheckBoxOperator jcbo0 = new JCheckBoxOperator(jtpo, "Arabic (Arabic)");
+            jcbo0.doClick();
+            JCheckBoxOperator jcbo1 = new JCheckBoxOperator(jtpo, "Han (Chinese, Japanese, Korean)");
+            jcbo1.doClick();
+        }
         JButtonOperator jbo2 = new JButtonOperator(jdo, "OK", 0);
         jbo2.pushNoBlock();
         WizardOperator wo = new WizardOperator("Add Image");
@@ -254,22 +269,19 @@ public class RegressionTest extends TestCase{
         logger.info("Generate Report Button");
         JDialog reportDialog = JDialogOperator.waitJDialog("Generate Report", false, false);
         JDialogOperator reportDialogOperator = new JDialogOperator(reportDialog);
-        JCheckBoxOperator jcbo0 = new JCheckBoxOperator(reportDialogOperator, "Excel");
-        jcbo0.doClick();
-        JCheckBoxOperator jcbo1 = new JCheckBoxOperator(reportDialogOperator, "Default XML");
-        jcbo1.doClick();
-        JCheckBoxOperator jcbo2 = new JCheckBoxOperator(reportDialogOperator, "Body File");
-        jcbo2.doClick();
         JButtonOperator jbo0 = new JButtonOperator(reportDialogOperator, "Generate Report");
-        jbo0.pushNoBlock();
+        DateFormat dateFormat = new SimpleDateFormat("MM-dd-yyyy-HH-mm-ss");
+        Date date = new Date();
+        String datenotime = dateFormat.format(date);
+        jbo0.pushNoBlock();       
         new Timeout("pausing", 3000).sleep(); // Give it a few seconds to generate
         screenshot("Finished Report");
-        
         JDialog previewDialog = JDialogOperator.waitJDialog("Report Preview", false, false);
         JDialogOperator previewDialogOperator = new JDialogOperator(previewDialog);
         JButtonOperator jbo1 = new JButtonOperator(previewDialogOperator, "Close");
         jbo1.pushNoBlock();
         new Timeout("pausing", 3000).sleep(); // Give the program a second to idle to be safe
+        System.setProperty("ReportStr", datenotime);
         screenshot("Done Testing");
     }
     
@@ -282,9 +294,10 @@ public class RegressionTest extends TestCase{
             ImageIO.write(capture, "png", new File(outPath + "\\" + name + ".png"));
             new Timeout("pausing", 1000).sleep(); // give it a second to save
         } catch (IOException ex) {
-            logger.info("IOException taking screenshot.");
+            logger.log(Level.WARNING, "IOException taking screenshot.", ex);
         } catch (AWTException ex) {
-            logger.info("AWTException taking screenshot.");
+            logger.log(Level.WARNING, "AWTException taking screenshot.", ex);
+            
         }
     }