Merge branch 'master' into scalpel-carver

.gitignore

@@ -46,8 +46,10 @@ genfiles.properties
 /branding/nbproject/*
 !/branding/nbproject/project.xml
 !/branding/nbproject/project.properties
-/test/input/
 /test/input/*
+!/test/input/notablehashes.txt-md5.idx
+!/test/input/notablekeywords.xml
+!/test/input/NSRL.txt-md5.idx
 /test/output/*
 !/test/output/gold
 /test/output/gold/tmp

Core/src/org/sleuthkit/autopsy/casemodule/Case.java

@@ -816,7 +816,7 @@ public class Case {
      * Invoke the creation of startup dialog window.
      */
     static public void invokeStartupDialog() {
-        StartupWindow.getInstance().open();
+        StartupWindowProvider.getInstance().open();
     }
 
     /**

Core/src/org/sleuthkit/autopsy/casemodule/CaseCloseAction.java

@@ -86,7 +86,7 @@ public final class CaseCloseAction extends CallableSystemAction implements Prese
             EventQueue.invokeLater(new Runnable() {
                 @Override
                 public void run() {
-                    StartupWindow.getInstance().open();
+                    StartupWindowProvider.getInstance().open();
                 }
             });
         } catch (Exception ex) {

Core/src/org/sleuthkit/autopsy/casemodule/CaseOpenAction.java

@@ -85,7 +85,7 @@ public final class CaseOpenAction implements ActionListener {
             } else {
                 // try to close Startup window if there's one
                 try {
-                    StartupWindow.getInstance().close();
+                    StartupWindowProvider.getInstance().close();
                 } catch (Exception ex) {
                     // no need to show the error message to the user.
                     logger.log(Level.WARNING, "Error closing startup window.", ex);
@@ -97,7 +97,7 @@ public final class CaseOpenAction implements ActionListener {
                             + ": " + ex.getMessage(), "Error", JOptionPane.ERROR_MESSAGE);
                     logger.log(Level.WARNING, "Error opening case in folder " + path, ex);
 
-                    StartupWindow.getInstance().open();
+                    StartupWindowProvider.getInstance().open();
                 }
             }
         }

Core/src/org/sleuthkit/autopsy/casemodule/NewCaseWizardPanel1.java

@@ -291,7 +291,7 @@ class NewCaseWizardPanel1 implements WizardDescriptor.ValidatingPanel<WizardDesc
             createdDirectory = caseDirPath;
             // try to close Startup window if there's one
             try {
-                StartupWindow.getInstance().close();
+                StartupWindowProvider.getInstance().close();
             } catch (Exception ex) {
                 logger.log(Level.WARNING, "Startup window didn't close as expected.", ex);
 

Core/src/org/sleuthkit/autopsy/casemodule/OpenRecentCasePanel.java

@@ -187,7 +187,7 @@ class OpenRecentCasePanel extends javax.swing.JPanel {
         if (!casePath.equals("")) {
             // Close the startup menu
             try {
-                StartupWindow.getInstance().close();
+                StartupWindowProvider.getInstance().close();
                 CueBannerPanel.closeOpenRecentCasesWindow();
             } catch (Exception ex) {
                 logger.log(Level.WARNING, "Error: couldn't open case: " + caseName, ex);
@@ -200,7 +200,7 @@ class OpenRecentCasePanel extends javax.swing.JPanel {
 
                      //if case is not opened, open the start window
                     if (Case.isCaseOpen() == false) {
-                        StartupWindow.getInstance().open();
+                        StartupWindowProvider.getInstance().open();
                     }
  
                 } else {

Core/src/org/sleuthkit/autopsy/casemodule/RecentItems.java

@@ -65,7 +65,7 @@ class RecentItems implements ActionListener {
 
                     @Override
                     public void run() {
-                        StartupWindow.getInstance().open();
+                        StartupWindowProvider.getInstance().open();
                     }
                     
                 });

Core/src/org/sleuthkit/autopsy/casemodule/StartupWindow.java

@@ -26,40 +26,28 @@ import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
 import javax.swing.JDialog;
 import javax.swing.JFrame;
+import org.openide.util.lookup.ServiceProvider;
 
 /**
- * Displays 
+ * The default implementation of the Autopsy startup window
  */
-public final class StartupWindow extends JDialog {
+@ServiceProvider(service=StartupWindowInterface.class)
+public final class StartupWindow extends JDialog implements StartupWindowInterface {
 
     private static StartupWindow instance;
     private static final String TITLE = "Welcome";
     private static Dimension DIMENSIONS = new Dimension(750, 400);
 
-    private StartupWindow(JFrame frame, String title, boolean isModal) {
-        super(frame, title, isModal);
+    public StartupWindow() {
+        super(new JFrame(TITLE), TITLE, true);
         init();
     }
 
-    /**
-     * Get the startup window
-     * @return the startup window singleton
-     */
-    public static synchronized StartupWindow getInstance() {
-        if (StartupWindow.instance == null) {
-            JFrame frame = new JFrame(TITLE);
-            boolean isModal = true;
-            StartupWindow.instance = new StartupWindow(frame, TITLE, isModal);
-        }
-
-
-        return instance;
-    }
 
     /**
      * Shows the startup window.
      */
-    public void init() {
+    private void init() {
 
         Dimension screenDimension = Toolkit.getDefaultToolkit().getScreenSize();
 
@@ -88,6 +76,7 @@ public final class StartupWindow extends JDialog {
 
     }
     
+    @Override
     public void open() {
         setVisible(true);
     }
@@ -95,6 +84,7 @@ public final class StartupWindow extends JDialog {
     /**
      * Closes the startup window.
      */
+    @Override
     public void close() {
         this.setVisible(false);
     }

Core/src/org/sleuthkit/autopsy/casemodule/StartupWindowInterface.java

@@ -0,0 +1,35 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2013 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.casemodule;
+
+/**
+ * Interface for startup window implementations
+ */
+public interface StartupWindowInterface {
+
+    /**
+     * Shows and makes active the startup window
+     */
+    public void open();
+
+    /**
+     * Closes the startup window
+     */
+    public void close();
+}

Core/src/org/sleuthkit/autopsy/casemodule/StartupWindowProvider.java

@@ -0,0 +1,103 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2013 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.casemodule;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.logging.Level;
+import org.openide.util.Lookup;
+import org.sleuthkit.autopsy.coreutils.Logger;
+
+/**
+ * Provides the start up window to rest of the application. It may return the
+ * main / default startup window, or a custom one if it has been discovered.
+ * 
+ * All that is required to create a custom startup window in a module and active it, 
+ * is to implement StartupWindowInterface and register it with lookup as a ServiceProvider.
+ * The custom startup window is automatically chosen over the default one, given it is the only external module custom startup window.
+ */
+public class StartupWindowProvider implements StartupWindowInterface {
+
+    private static volatile StartupWindowProvider instance;
+    private static final Logger logger = Logger.getLogger(StartupWindowProvider.class.getName());
+    private volatile StartupWindowInterface startupWindowToUse;
+
+    public static StartupWindowProvider getInstance() {
+        if (instance == null) {
+            synchronized (StartupWindowProvider.class) {
+                if (instance == null) {
+                    instance = new StartupWindowProvider();
+                    instance.init();
+                }
+            }
+        }
+
+        return instance;
+    }
+
+    private void init() {
+        if (startupWindowToUse == null) {
+            //discover the registered windows
+            Collection<? extends StartupWindowInterface> startupWindows =
+                    Lookup.getDefault().lookupAll(StartupWindowInterface.class);
+
+            int windowsCount = startupWindows.size();
+            if (windowsCount > 2) {
+                logger.log(Level.WARNING, "More than 2 (" + windowsCount + ") start up windows discovered, will use the first custom one");
+            } else if (windowsCount == 1) {
+                startupWindowToUse = startupWindows.iterator().next();
+                logger.log(Level.INFO, "Will use the default startup window: " + startupWindowToUse.toString());
+            } else {
+                //pick the non default one
+                Iterator<? extends StartupWindowInterface> it = startupWindows.iterator();
+                while (it.hasNext()) {
+                    StartupWindowInterface window = it.next();
+                    if (!org.sleuthkit.autopsy.casemodule.StartupWindow.class.isInstance(window)) {
+                        startupWindowToUse = window;
+                        logger.log(Level.INFO, "Will use the custom startup window: " + startupWindowToUse.toString());
+                        break;
+
+                    }
+                }
+                
+                if (startupWindowToUse == null) {
+                    logger.log(Level.SEVERE, "Unexpected error, no custom startup window found, using the default");
+                    startupWindowToUse = new org.sleuthkit.autopsy.casemodule.StartupWindow();
+                }
+                
+            }
+
+
+        }
+    }
+
+    @Override
+    public void open() {
+        if (startupWindowToUse != null) {
+            startupWindowToUse.open();
+        }
+    }
+
+    @Override
+    public void close() {
+       if (startupWindowToUse != null) {
+           startupWindowToUse.close();
+       }
+    }
+}

Core/src/org/sleuthkit/autopsy/core/Installer.java

@@ -124,7 +124,7 @@ public class Installer extends ModuleInstall {
             javaFxInit = false;
             final String msg = "Error initializing JavaFX.  ";
             final String details = " Some features will not be available. "
-                    + " Check that you have the right JRE installed (Sun JRE > 1.7.10). ";
+                    + " Check that you have the right JRE installed (Oracle JRE > 1.7.10). ";
             logger.log(Level.SEVERE, msg
 		       + details, e);
 

Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java

@@ -578,22 +578,21 @@ class IngestScheduler {
          * of skipped
          * @return true if should be enqueued, false otherwise
          */
-        private static boolean shouldEnqueueTask(ProcessTask processTask) {
+        private static boolean shouldEnqueueTask(final ProcessTask processTask) {
             final AbstractFile aFile = processTask.file;
 
             //if it's unalloc file, skip if so scheduled
-            if (processTask.context.isProcessUnalloc() == false) {
-                if (aFile.isVirtual() == true) {
+            if (processTask.context.isProcessUnalloc() == false
+                    && aFile.getType().equals(TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS //unalloc files
+                        ) ) {
                     return false;
             }
-            }
 
             String fileName = aFile.getName();
             if (fileName.equals(".") || fileName.equals("..")) {
                 return false;
             }
-            if (aFile.isVirtual() == false && aFile.isFile() == true
-                    && aFile.getType() == TSK_DB_FILES_TYPE_ENUM.FS) {
+            else if (aFile instanceof org.sleuthkit.datamodel.File ) {
                 final org.sleuthkit.datamodel.File f = (File) aFile;
 
                 //skip files in root dir, starting with $, containing : (not default attributes)

NEWS.txt

@@ -9,6 +9,7 @@ Improvements:
 
 Bugfixes:
 - Keyword Search: fix when Solr does not cleanly shutdown
+- fix for "Process Unallocated Space" option doesn't do anything
 
 
 

Test/script/regression.py

@@ -27,6 +27,7 @@ import re
 import zipfile
 import zlib
 import Emailer
+import srcupdater
 
 #
 # Please read me...
@@ -71,6 +72,7 @@ class Args:
 		self.contin = False
 		self.gold_creation = False
 		self.daily = False
+		self.fr = False
 	
 	def parse(self):
 		global nxtproc 
@@ -139,6 +141,9 @@ class Args:
 			elif arg == "-d" or arg == "--daily":
 				printout("Running daily")
 				self.daily = True
+			elif arg == "-fr" or arg == "--forcerun":
+				printout("Not downloading new images")
+				self.fr = True
 			else:
 				printout(usage())
 				return False
@@ -295,7 +300,8 @@ class Database:
 			global failedbool
 			global errorem
 			failedbool = True
-			errorem += "There was a difference in the number of artifacts for " + case.image + ".\n"
+			global imgfail
+			imgfail = True
 			return "; ".join(self.artifact_comparison)
 		
 	def get_attribute_comparison(self):
@@ -304,7 +310,8 @@ class Database:
 		global failedbool
 		global errorem
 		failedbool = True
-		errorem += "There was a difference in the number of attributes for " + case.image + ".\n"
+		global imgfail
+		imgfail = True
 		list = []
 		for error in self.attribute_comparison:
 			list.append(error)
@@ -351,6 +358,8 @@ class Database:
 	def generate_gold_artifacts(self):
 		if not self.gold_artifacts:
 			gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
+			if(not file_exists(gold_db_file)):
+				gold_db_file = Emailer.make_path(case.gold_parse, case.image_name, "autopsy.db")
 			gold_con = sqlite3.connect(gold_db_file)
 			gold_cur = gold_con.cursor()
 			gold_cur.execute("SELECT COUNT(*) FROM blackboard_artifact_types")
@@ -367,6 +376,8 @@ class Database:
 	def generate_gold_attributes(self):
 		if self.gold_attributes == 0:
 			gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
+			if(not file_exists(gold_db_file)):
+				gold_db_file = Emailer.make_path(case.gold_parse, case.image_name, "autopsy.db")
 			gold_con = sqlite3.connect(gold_db_file)
 			gold_cur = gold_con.cursor()
 			gold_cur.execute("SELECT COUNT(*) FROM blackboard_attributes")
@@ -375,6 +386,8 @@ class Database:
 	def generate_gold_objects(self):
 		if self.gold_objects == 0:
 			gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
+			if(not file_exists(gold_db_file)):
+				gold_db_file = Emailer.make_path(case.gold_parse, case.image_name, "autopsy.db")
 			gold_con = sqlite3.connect(gold_db_file)
 			gold_cur = gold_con.cursor()
 			gold_cur.execute("SELECT COUNT(*) FROM tsk_objects")
@@ -392,21 +405,30 @@ class Database:
 def run_config_test(config_file):
 	try:
 		global parsed
+		global errorem
+		global attachl
 		count = 0
 		parsed = parse(config_file)
+		case
 		counts = {}
 		if parsed.getElementsByTagName("indir"):
 			case.input_dir = parsed.getElementsByTagName("indir")[0].getAttribute("value").encode().decode("utf_8")
 		if parsed.getElementsByTagName("global_csv"):
 			case.global_csv = parsed.getElementsByTagName("global_csv")[0].getAttribute("value").encode().decode("utf_8")
 			case.global_csv = Emailer.make_local_path(case.global_csv)
-		
+		if parsed.getElementsByTagName("golddir"):
+			case.gold_parse = parsed.getElementsByTagName("golddir")[0].getAttribute("value").encode().decode("utf_8")
+			case.gold_parse = Emailer.make_path(case.gold_parse, "tmp")
+		else:
+			case.gold_parse = case.gold
 		# Generate the top navbar of the HTML for easy access to all images
 		values = []
 		for element in parsed.getElementsByTagName("image"):
 			value = element.getAttribute("value").encode().decode("utf_8")
 			if file_exists(value):
 				values.append(value)
+			else:
+				print("File: ", value, " doesn't exist")
 		count = len(values)
 		archives = Emailer.make_path(case.gold, "..")
 		arcount = 0
@@ -430,6 +452,8 @@ def run_config_test(config_file):
 			#Begin infiniloop
 			if(newDay()):
 				global daycount
+				setDay()
+				srcupdater.compile(errorem, attachl, parsed)
 				if(daycount > 0):
 					print("starting process")
 					outputer = open("ScriptLog.txt", "a")
@@ -457,7 +481,8 @@ def run_config_test(config_file):
 # The path must be guarenteed to be a correct path.
 def run_test(image_file, count):
 	global parsed
-	print(args.config_file)
+	global imgfail
+	imgfail = False
 	if image_type(image_file) == IMGTYPE.UNKNOWN:
 		printerror("Error: Image type is unrecognized:")
 		printerror(image_file + "\n")
@@ -525,6 +550,10 @@ def run_test(image_file, count):
 			gold_path = case.gold
 			img_gold = Emailer.make_path(case.gold, case.image_name)
 			img_archive = Emailer.make_local_path("..", "output", "gold", case.image_name+"-archive.zip")
+			if(not file_exists(img_archive)):
+				img_archive = Emailer.make_path(case.gold_parse, "..", case.image_name+"-archive.zip")
+				gold_path = case.gold_parse
+				img_gold = Emailer.make_path(gold_path, case.image_name)
 			extrctr = zipfile.ZipFile(img_archive, 'r', compression=zipfile.ZIP_DEFLATED)
 			extrctr.extractall(gold_path)
 			extrctr.close
@@ -616,17 +645,19 @@ def image_type(image_file):
 def rebuild():
 	# Errors to print
 	errors = []
+	if(case.gold_parse == None):
+		case.gold_parse = case.gold
 	# Delete the current gold standards
-	gold_dir = Emailer.make_path(case.gold, case.image_name)
+	gold_dir = Emailer.make_path(case.gold_parse)
 	clear_dir(gold_dir)
 	dbinpth = Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "autopsy.db")
-	dboutpth = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
-	if not os.path.exists(case.gold):
-		os.makedirs(case.gold)
+	dboutpth = Emailer.make_path(gold_dir, "autopsy.db")
+	if not os.path.exists(case.gold_parse):
+		os.makedirs(case.gold_parse)
 	if not os.path.exists(gold_dir):
 		os.makedirs(gold_dir)
 	copy_file(dbinpth, dboutpth)
-	error_pth = Emailer.make_path(case.gold, case.image_name, case.image_name+"SortedErrors.txt")
+	error_pth = Emailer.make_path(gold_dir, case.image_name+"SortedErrors.txt")
 	copy_file(case.sorted_log, error_pth)
 	# Rebuild the HTML report
 	htmlfolder = ""
@@ -638,9 +669,9 @@ def rebuild():
 	html_path = Emailer.make_path(case.output_dir, case.image_name,
 								 "AutopsyTestCase", "Reports")
 	try:
-		os.makedirs(os.path.join(case.gold, case.image_name, htmlfolder))
+		os.makedirs(os.path.join(gold_dir, htmlfolder))
 		for file in os.listdir(autopsy_html_path):
-			html_to = Emailer.make_path(case.gold, case.image_name, file.replace("HTML Report", "Report"))
+			html_to = Emailer.make_path(gold_dir, file.replace("HTML Report", "Report"))
 			copy_dir(get_file_in_dir(autopsy_html_path, file), html_to)
 	except FileNotFoundException as e:
 		errors.append(e.error)
@@ -648,10 +679,9 @@ def rebuild():
 		errors.append("Error: Unknown fatal error when rebuilding the gold html report.")
 		errors.append(str(e) + "\n")
 	oldcwd = os.getcwd()
-	zpdir = case.gold
+	zpdir = case.gold_parse
 	os.chdir(zpdir)
 	img_gold = case.image_name
-	print(img_gold)
 	img_archive = Emailer.make_path("..", case.image_name+"-archive.zip")
 	comprssr = zipfile.ZipFile(img_archive, 'w',compression=zipfile.ZIP_DEFLATED)
 	zipdir(img_gold, comprssr)
@@ -673,6 +703,8 @@ def zipdir(path, zip):
 def compare_to_gold_db():
 	# SQLITE needs unix style pathing
 	gold_db_file = Emailer.make_path(case.gold, case.image_name, "autopsy.db")
+	if(not file_exists(gold_db_file)):
+		gold_db_file = Emailer.make_path(case.gold_parse, case.image_name, "autopsy.db")
 	autopsy_db_file = Emailer.make_path(case.output_dir, case.image_name,
 									  "AutopsyTestCase", "autopsy.db")
 	# Try to query the databases. Ignore any exceptions, the function will
@@ -728,6 +760,8 @@ def compare_to_gold_db():
 # the regression test against the gold standard html report
 def compare_to_gold_html():
 	gold_html_file = Emailer.make_path(case.gold, case.image_name, "Report", "index.html")
+	if(not file_exists(gold_html_file)):
+		gold_html_file = Emailer.make_path(case.gold_parse, case.image_name, "Report", "index.html")
 	htmlfolder = ""
 	for fs in os.listdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports")):
 		if os.path.isdir(Emailer.make_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", fs)):
@@ -752,9 +786,15 @@ def compare_to_gold_html():
 				ListGoldHTML.append(os.path.join(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder, fs))
 		#Find all new .html files belonging to this case
 		ListNewHTML = []
+		if(os.path.exists(Emailer.make_path(case.gold, case.image_name))):
 			for fs in os.listdir(Emailer.make_path(case.gold, case.image_name)):
 				if (fs.endswith(".html")):
 					ListNewHTML.append(Emailer.make_path(case.gold, case.image_name, fs))
+		if(not case.gold_parse == None or case.gold == case.gold_parse):
+			if(file_exists(Emailer.make_path(case.gold_parse, case.image_name))):
+				for fs in os.listdir(Emailer.make_path(case.gold_parse, case.image_name)):
+					if (fs.endswith(".html")):
+						ListNewHTML.append(Emailer.make_path(case.gold_parse, case.image_name, fs))
 		#ensure both reports have the same number of files and are in the same order
 		if(len(ListGoldHTML) != len(ListNewHTML)):
 			printerror("The reports did not have the same number of files. One of the reports may have been corrupted")
@@ -788,17 +828,20 @@ def compare_to_gold_html():
 def compare_bb_artifacts():
 	exceptions = []
 	try:
-		for type_id in range(1, 13):
+		global failedbool
+		global errorem
 		if database.gold_artifacts != database.autopsy_artifacts:
+			failedbool = True
+			global imgfail
+			imgfail = True
+			errorem += "There was a difference in the number of artifacts for " + case.image + ".\n"
+		for type_id in range(1, 13):
+			if database.gold_artifacts[type_id] != database.autopsy_artifacts[type_id]:
 				error = str("Artifact counts do not match for type id %d. " % type_id)
 				error += str("Gold: %d, Test: %d" %
 							(database.gold_artifacts[type_id],
 							 database.autopsy_artifacts[type_id]))
 				exceptions.append(error)
-				global failedbool
-				global errorem
-				failedbool = True
-				errorem += "There was a difference in the number of artifacts for " + case.image + ".\n"
 		return exceptions
 	except Exception as e:
 		exceptions.append("Error: Unable to compare blackboard_artifacts.\n")
@@ -816,6 +859,8 @@ def compare_bb_attributes():
 			global failedbool
 			global errorem
 			failedbool = True
+			global imgfail
+			imgfail = True
 			errorem += "There was a difference in the number of attributes for " + case.image + ".\n"
 			return exceptions
 	except Exception as e:
@@ -834,6 +879,8 @@ def compare_tsk_objects():
 			global failedbool
 			global errorem
 			failedbool = True
+			global imgfail
+			imgfail = True
 			errorem += "There was a difference between the tsk object counts for " + case.image + " .\n"
 			return exceptions
 	except Exception as e:
@@ -863,17 +910,15 @@ def generate_common_log():
 			for line in log:
 				line = line.replace(rep_path, "CASE")
 				if line.startswith("Exception"):
-					common_log.write("From " + file +":\n" +  line + "\n")
-				elif line.startswith("WARNING"):
-					common_log.write("From " + file +":\n" +  line + "\n")
+					common_log.write(file +": " +  line)
 				elif line.startswith("Error"):
-					common_log.write("From " + file +":\n" +  line + "\n")
+					common_log.write(file +": " +  line)
 				elif line.startswith("SEVERE"):
-					common_log.write("From " + file +":\n" +  line + "\n")
+					common_log.write(file +":" +  line)
 				else:
-					warning_log.write("From " + file +":\n" +  line + "\n")
+					warning_log.write(file +": " +  line)
 			log.close()
-		common_log.write("\n\n")
+		common_log.write("\n")
 		common_log.close()
 		case.sorted_log = Emailer.make_local_path(case.output_dir, case.image_name, case.image_name + "SortedErrors.txt")
 		srtcmdlst = ["sort", case.common_log_path, "-o", case.sorted_log]
@@ -885,23 +930,28 @@ def generate_common_log():
 		
 def compare_errors():
 	gold_dir = Emailer.make_path(case.gold, case.image_name, case.image_name + "SortedErrors.txt")
+	if(not file_exists(gold_dir)):
+			gold_dir = Emailer.make_path(case.gold_parse, case.image_name, case.image_name + "SortedErrors.txt")
 	common_log = codecs.open(case.sorted_log, "r", "utf_8")
 	gold_log = codecs.open(gold_dir, "r", "utf_8")
 	gold_dat = gold_log.read()
 	common_dat = common_log.read()
 	patrn = re.compile("\d")
 	if (not((re.sub(patrn, 'd', gold_dat)) == (re.sub(patrn, 'd', common_dat)))):
-		diff_dir = Emailer.make_local_path(case.output_dir, case.image_name, "ErrorDiff.txt")
+		diff_dir = Emailer.make_local_path(case.output_dir, case.image_name, case.image_name+"_AutopsyErrors-Diff.txt")
 		diff_file = open(diff_dir, "w") 
 		dffcmdlst = ["diff", case.sorted_log, gold_dir]
 		subprocess.call(dffcmdlst, stdout = diff_file)
 		global attachl
 		global errorem
 		global failedbool
+		attachl.append(case.sorted_log)
 		attachl.append(diff_dir)
-		errorem += "There was a difference in the exceptions Log.\n"
+		errorem += "There was a difference in the exceptions Log for " + case.image_name + ".\n"
 		print("Exceptions didn't match.\n")
 		failedbool = True
+		global imgfail
+		imgfail = True
 
 # Fill in the global case's variables that require the log files
 def fill_case_data():
@@ -1203,6 +1253,7 @@ def printout(string):
 def generate_html():
 	# If the file doesn't exist yet, this is the first case to run for
 	# this test, so we need to make the start of the html log
+	global imgfail
 	if not file_exists(case.html_log):
 		write_html_head()
 	try:
@@ -1217,9 +1268,12 @@ def generate_html():
 				 <a href='#" + case.image_name + "-general'>General Output</a> |\
 				 <a href='#" + case.image_name + "-logs'>Logs</a>\
 				 </h2>"
-				 
 		# The script errors found
-		errors = "<div id='errors'>\
+		if imgfail:
+			ids = 'errors1'
+		else:
+			ids = 'errors'
+		errors = "<div id='" + ids + "'>\
 				  <h2><a name='" + case.image_name + "-errors'>Errors and Warnings</a></h2>\
 				  <hr color='#FF0000'>"
 		# For each error we have logged in the case
@@ -1234,7 +1288,7 @@ def generate_html():
 		# Links to the logs
 		logs = "<div id='logs'>\
 				<h2><a name='" + case.image_name + "-logs'>Logs</a></h2>\
-				<hr color='#00a00f'>"
+				<hr color='#282828'>"
 		logs_path = Emailer.make_local_path(case.output_dir, case.image_name, "logs")
 		for file in os.listdir(logs_path):
 			logs += "<p><a href='file:\\" + Emailer.make_path(logs_path, file) + "' target='_blank'>" + file + "</a></p>"
@@ -1243,7 +1297,7 @@ def generate_html():
 		# All the testing information
 		info = "<div id='info'>\
 				<h2><a name='" + case.image_name + "-info'>Information</a></h2>\
-				<hr color='#0005FF'>\
+				<hr color='#282828'>\
 				<table cellspacing='5px'>"
 		# The individual elements
 		info += "<tr><td>Image Path:</td>"
@@ -1332,13 +1386,14 @@ def write_html_head():
 			h1 span { font-size: 12px; font-weight: 100; }\
 			h2 { font-family: Tahoma; padding: 0px; margin: 0px; }\
 			hr { width: 100%; height: 1px; border: none; margin-top: 10px; margin-bottom: 10px; }\
-			#errors { background: #FFCFCF; border: 1px solid #FF0000; color: #FF0000; padding: 10px; margin: 20px; }\
-			#info { background: #D2D3FF; border: 1px solid #0005FF; color: #0005FF; padding: 10px; margin: 20px; }\
+			#errors { background: #CCCCCC; border: 1px solid #282828; color: #282828; padding: 10px; margin: 20px; }\
+			#errors1 { background: #CC0000; border: 1px solid #282828; color: #282828; padding: 10px; margin: 20px; }\
+			#info { background: #CCCCCC; border: 1px solid #282828; color: #282828; padding: 10px; margin: 20px; }\
 			#general { background: #CCCCCC; border: 1px solid #282828; color: #282828; padding: 10px; margin: 20px; }\
-			#logs { background: #8cff97; border: 1px solid #00820c; color: #00820c; padding: 10px; margin: 20px; }\
+			#logs { background: #CCCCCC; border: 1px solid #282828; color: #282828; padding: 10px; margin: 20px; }\
 			#errors p, #info p, #general p, #logs p { pading: 0px; margin: 0px; margin-left: 5px; }\
-			#info table td { color: #0005FF; font-size: 12px; min-width: 225px; }\
-			#logs a { color: #00820c; }\
+			#info table td { color: ##282828; font-size: 12px; min-width: 225px; }\
+			#logs a { color: ##282828; }\
 			</style>\
 			<body>"
 	html.write(head)
@@ -1361,7 +1416,7 @@ def html_add_images(full_image_names):
 	links = []
 	for full_name in full_image_names:
 		name = case.get_image_name(full_name)
-		links.append("<a href='#" + name + "'>" + name + "</a>")
+		links.append("<a href='#" + name + "(0)'>" + name + "</a>")
 	html.write("<p align='center'>" + (" | ".join(links)) + "</p>")
 
 
@@ -1538,6 +1593,7 @@ Options:
   -e ex		 Prints out all errors containing ex.
   -l cfg		Runs from configuration file cfg.
   -c			Runs in a loop over the configuration file until canceled. Must be used in conjunction with -l
+  -fr			Will not try download gold standard images
 	"""
 
 
@@ -1587,6 +1643,8 @@ def execute_test():
 	global failedbool
 	global html
 	global attachl
+	if(not dir_exists(Emailer.make_path("..", "output", "results"))):
+		os.makedirs(Emailer.make_path("..", "output", "results",))
 	case.output_dir = Emailer.make_path("..", "output", "results", time.strftime("%Y.%m.%d-%H.%M.%S"))
 	os.makedirs(case.output_dir)
 	case.common_log = "AutopsyErrors.txt"
@@ -1626,6 +1684,8 @@ def execute_test():
 	logres = search_common_log("TskCoreException")
 	if (len(logres)>0):
 		failedbool = True
+		global imgfail
+		imgfail = True
 		global errorem
 		errorem += "Autopsy Nightly test failed.\n"
 		passFail = False
@@ -1633,6 +1693,7 @@ def execute_test():
 			errorem += lm
 	html.close()
 	if failedbool:
+		passFail = False
 		attachl.append(case.common_log_path)
 		attachl.insert(0, html.name)
 	else:
@@ -1641,7 +1702,7 @@ def execute_test():
 		passFail = True
 		attachl = []
 	if not args.gold_creation:
-		Emailer.send_email(parsed, errorem, attachl, html)
+		Emailer.send_email(parsed, errorem, attachl, passFail)
 		
 def secs_till_tommorow():
 	seconds = (23*3600)-(int(strftime("%H", localtime()))*3600)
@@ -1664,17 +1725,6 @@ def main():
 	global daycount
 	global redo
 	global passed
-	inpvar = raw_input("Your input images may be out of date, do you want to update?(y/n): ")
-	if(inpvar.lower() == 'y' or inpvar.lower() == 'yes'):
-		antin = ["ant"]
-		antin.append("-f")
-		antin.append(os.path.join("..","..","build.xml"))
-		antin.append("test-download-imgs")
-		if SYS is OS.CYGWIN:
-			subprocess.call(antin)
-		elif SYS is OS.WIN:
-			theproc = subprocess.Popen(antin, shell = True, stdout=subprocess.PIPE)
-			theproc.communicate()
 	daycount = 0
 	failedbool = False
 	redo = False
@@ -1688,9 +1738,18 @@ def main():
 	# The arguments were given wrong:
 	if not args.parse():
 		case.reset()
-		pass
+		return
+	if(not args.fr):
+		antin = ["ant"]
+		antin.append("-f")
+		antin.append(os.path.join("..","..","build.xml"))
+		antin.append("test-download-imgs")
+		if SYS is OS.CYGWIN:
+			subprocess.call(antin)
+		elif SYS is OS.WIN:
+			theproc = subprocess.Popen(antin, shell = True, stdout=subprocess.PIPE)
+			theproc.communicate()
 	# Otherwise test away!
-	else:
 	execute_test()
 	if(args.daily and args.contin):
 		time.sleep(secs_till_tommorow())
@@ -1703,7 +1762,6 @@ def main():
 		execute_test()
 		case = TestAutopsy()
 
-
 class OS:
   LINUX, MAC, WIN, CYGWIN = range(4)	  
 if __name__ == "__main__":

test/input/NSRL.txt-md5.idx

@@ -0,0 +1,6 @@
+00000000000000000000000000000000000000000|md5sum
+00000000000000000000000000000000000000001|NSRLcreator.txt
+0D4A1C9ED5A49CAF22FD5F52C666DE2C|0000000000000045
+35BA15EC1C3CF03531282147DB10A089|0000000000000100
+91C66396EEC4BCEEAF6EDE7A48F60C63|0000000000000152
+A99F69068F958CF412E7F8B8A0142B41|0000000000000000

test/input/notablehashes.txt-md5.idx

@@ -0,0 +1,6 @@
+00000000000000000000000000000000000000000|md5sum
+00000000000000000000000000000000000000001|notablehashescreator.txt
+48199F51973F317459E80E18DC744B12|0000000000000000
+5CCB10AEA1EC335139715D4AA44D0EE0|0000000000000062
+94610B03A4295300AA29C3364DB18683|0000000000000108
+A06B65C36E0E5A8229749375C3AAC4B1|0000000000000160

test/input/notablekeywords.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<keyword_lists>
+ <keyword_list created="2012-03-23 11:06:17" modified="2012-03-23 11:08:48" name="notable_keywords" use_for_ingest="true">
+  <keyword literal="true">Jean</keyword>
+  <keyword literal="true">Personalized</keyword>
+  <keyword literal="true">DIRT</keyword>  
+  <keyword literal="true">Osama</keyword>
+  <keyword literal="true">bomb</keyword>
+  <keyword literal="true">hacking</keyword>
+  <keyword literal="true">molotov</keyword>
+  <keyword literal="true">nuclear</keyword>
+  <keyword literal="true">صحافة و إعلام</keyword>
+  <keyword literal="true">مطلوبا</keyword>
+ </keyword_list>
+</keyword_lists>

test/script/Emailer.py

@@ -25,12 +25,12 @@ def send_email(parsed, errorem, attachl, passFail):
 	# Create the container (outer) email message.
 	msg = MIMEMultipart()
 	if(passFail):
-		msg['Subject'] = 'Autopsy Nightly test passed.'
+		msg['Subject'] = '[Test]Autopsy test passed.'
 	else:
-		msg['Subject'] = 'Autopsy Nightly test failed.'
+		msg['Subject'] = '[Test]Autopsy test failed.'
 	# me == the sender's email address
 	# family = the list of all recipients' email addresses
-	msg['From'] = 'AutopsyContinuousTest'
+	msg['From'] = 'AutopsyTest'
 	msg['To'] = toval
 	msg.preamble = 'This is a test'
 	container = MIMEText(errorem, 'plain')

test/script/srcupdater.py

@@ -15,14 +15,17 @@ import xml
 from xml.dom.minidom import parse, parseString
 import Emailer
 
-def compile():
+def compile(errore, attachli, parsedin):
 	global redo
 	global tryredo
 	global failedbool
 	global errorem
+	errorem = errore
 	global attachl
+	attachl = attachli
 	global passed
 	global parsed
+	parsed = parsedin
 	passed = True
 	tryredo = False
 	redo = True
@@ -79,9 +82,11 @@ def vsBuild():
 	global passed
 	global parsed
 	#Please ensure that the current working directory is $autopsy/testing/script
+	oldpath = os.getcwd()
+	os.chdir(os.path.join("..", "..", "..","sleuthkit", "win32"))
 	vs = []
 	vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe")
-	vs.append(os.path.join("..", "..", "..","sleuthkit", "win32", "Tsk-win.sln"))
+	vs.append(os.path.join("Tsk-win.sln"))
 	vs.append("/p:configuration=release")
 	vs.append("/p:platform=win32")
 	vs.append("/t:clean")
@@ -91,6 +96,7 @@ def vsBuild():
 	VSout = open(VSpth, 'a')
 	subprocess.call(vs, stdout=VSout)
 	VSout.close()
+	os.chdir(oldpath)
 	chk = os.path.join("..", "..", "..","sleuthkit", "win32", "Release", "libtsk_jni.dll")
 	try:
 		open(chk)
@@ -152,18 +158,14 @@ def antBuild(which, Build):
 
 
 def main():
-	global parsed
-	global errorem
-	global attachl
-	errorem = ""
-	attachl = []
+	errore = ""
+	attachli = []
 	config_file = ""
 	arg = sys.argv.pop(0)
 	arg = sys.argv.pop(0)
 	config_file = arg
-	print(config_file)
-	parsed = parse(config_file)
-	compile()
+	parsedin = parse(config_file)
+	compile(errore, attachli, parsedin)
 	
 class OS:
   LINUX, MAC, WIN, CYGWIN = range(4)