Merge branch 'develop' of github.com:sleuthkit/autopsy into develop

Core/src/org/sleuthkit/autopsy/datamodel/ResultsNode.java

@@ -36,8 +36,8 @@ public class ResultsNode extends DisplayableItemNode {
                 new KeywordHits(sleuthkitCase),
                 new HashsetHits(sleuthkitCase),
                 new EmailExtracted(sleuthkitCase),
-                new InterestingHits(sleuthkitCase),
+                new InterestingHits(sleuthkitCase)
-                new Tags())), Lookups.singleton(NAME));
+                )), Lookups.singleton(NAME));
         setName(NAME);
         setDisplayName(NAME);
         this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/results.png"); //NON-NLS

Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java

@@ -66,6 +66,7 @@ import org.sleuthkit.autopsy.datamodel.Reports;
 import org.sleuthkit.autopsy.datamodel.Results;
 import org.sleuthkit.autopsy.datamodel.ResultsNode;
 import org.sleuthkit.autopsy.datamodel.RootContentChildren;
+import org.sleuthkit.autopsy.datamodel.Tags;
 import org.sleuthkit.autopsy.datamodel.Views;
 import org.sleuthkit.autopsy.datamodel.ViewsNode;
 import org.sleuthkit.autopsy.ingest.IngestManager;
@@ -359,6 +360,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
                     items.add(new DataSources());
                     items.add(new Views(tskCase));
                     items.add(new Results(tskCase));
+                    items.add(new Tags());
                     items.add(new Reports());
                     contentChildren = new RootContentChildren(items);
                     Node root = new AbstractNode(contentChildren) {

Core/src/org/sleuthkit/autopsy/examples/SampleFileIngestModule.java

@@ -104,7 +104,8 @@ class SampleFileIngestModule implements FileIngestModule {
 
         // Skip anything other than actual file system files.
         if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
-                || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) {
+                || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
+                || (file.isFile() == false)) {
             return IngestModule.ProcessResult.OK;
         }
 

Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java

@@ -80,6 +80,10 @@ public final class ExifParserFileIngestModule implements FileIngestModule {
         if (content.getType().equals(TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
             return ProcessResult.OK;
         }
+        
+        if (content.isFile() == false) {
+            return ProcessResult.OK;
+        }
 
         // skip known
         if (content.getKnown().equals(TskData.FileKnown.KNOWN)) {

Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java

@@ -92,7 +92,8 @@ public class FileExtMismatchIngestModule implements FileIngestModule {
     public ProcessResult process(AbstractFile abstractFile) {
         // skip non-files
         if ((abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
-                || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) {
+                || (abstractFile.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
+                || (abstractFile.isFile() == false)) {
             return ProcessResult.OK;
         }
 

Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java

@@ -99,8 +99,8 @@ public class FileTypeIdIngestModule implements FileIngestModule {
          * Skip unallocated space and unused blocks files.
          */
         if ((file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
-                || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) {
+                || (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)
-
+                || (file.isFile() == false)) {
             return ProcessResult.OK;
         }
 

Core/src/org/sleuthkit/autopsy/modules/interestingitems/InterestingItemDefsPanel.java

@@ -256,8 +256,9 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
             option = JOptionPane.showConfirmDialog(null, panel, NbBundle.getMessage(FilesSetPanel.class, "FilesSetPanel.title"), JOptionPane.OK_CANCEL_OPTION, JOptionPane.PLAIN_MESSAGE);
         } while (option == JOptionPane.OK_OPTION && !panel.isValidDefinition());
 
-        // If rule set with same name already exists, do not add to the filesSets hashMap.
+        // While adding new ruleset(selectedSet == null), if rule set with same name already exists, do not add to the filesSets hashMap.
-        if(this.filesSets.containsKey(panel.getFilesSetName())) {
+        // In case of editing an existing ruleset(selectedSet != null), following check is not performed.
+        if(this.filesSets.containsKey(panel.getFilesSetName()) && selectedSet == null) {
             MessageNotifyUtil.Message.error(NbBundle.getMessage(this.getClass(),
                                                                 "InterestingItemDefsPanel.doFileSetsDialog.duplicateRuleSet.text",
                                                                 panel.getFilesSetName()));
@@ -315,7 +316,7 @@ final class InterestingItemDefsPanel extends IngestModuleGlobalSettingsPanel imp
                 rules.remove(selectedRule.getUuid());
             }
             FilesSet.Rule newRule = new FilesSet.Rule(panel.getRuleName(), panel.getFileNameFilter(), panel.getMetaTypeFilter(), panel.getPathFilter());
-            rules.put(Integer.toString(newRule.hashCode()), newRule);
+            rules.put(newRule.getUuid(), newRule);
 
             // Add the new/edited files set definition, replacing any previous 
             // definition with the same name and refreshing the display.

Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java

@@ -29,6 +29,7 @@ import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.nio.file.Files;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -50,8 +51,8 @@ import org.netbeans.jellytools.WizardOperator;
 import org.netbeans.jemmy.Timeout;
 import org.netbeans.jemmy.Timeouts;
 import org.netbeans.jemmy.operators.JButtonOperator;
-import org.netbeans.jemmy.operators.JListOperator;
 import org.netbeans.jemmy.operators.JCheckBoxOperator;
+import org.netbeans.jemmy.operators.JComboBoxOperator;
 import org.netbeans.jemmy.operators.JDialogOperator;
 import org.netbeans.jemmy.operators.JFileChooserOperator;
 import org.netbeans.jemmy.operators.JLabelOperator;
@@ -88,25 +89,57 @@ public class RegressionTest extends TestCase {
         super(name);
     }
 
+    /**
+     * This method is used to escape file/directory path. Example: \\NetworkLocation\foo\bar
+     * get escaped to \\\\NetworkLocation\foo\bar so that it can be used as intended.
+     * @param path
+     * @return escaped path the the file/directory location.
+     */
+    private static String getEscapedPath(String path) {
+        if (path.startsWith("\\")) {
+            return "\\" + path;
+        } else {
+            return path;
+        }
+    }
+
     /**
      * Creates suite from particular test cases.
      */
     public static Test suite() {
         // run tests with specific configuration
+        File img_path = new File(getEscapedPath(System.getProperty("img_path")));
         NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(RegressionTest.class).
                 clusters(".*").
                 enableModules(".*");
-        conf = conf.addTest("testNewCaseWizardOpen",
+        if (img_path.isFile()) {
-                "testNewCaseWizard",
+            conf = conf.addTest("testNewCaseWizardOpen",
-                "testStartAddDataSource",
+                    "testNewCaseWizard",
-                "testConfigureIngest1",
+                    "testStartAddImageFileDataSource",
-                "testConfigureHash",
+                    "testConfigureIngest1",
-                "testConfigureIngest2",
+                    "testConfigureHash",
-                "testConfigureSearch",
+                    "testConfigureIngest2",
-                "testAddSourceWizard1",
+                    "testConfigureSearch",
-                "testIngest",
+                    "testAddSourceWizard1",
-                "testGenerateReportToolbar",
+                    "testIngest",
-                "testGenerateReportButton");
+                    "testGenerateReportToolbar",
+                    "testGenerateReportButton");
+        }
+
+        if (img_path.isDirectory()) {
+            conf = conf.addTest("testNewCaseWizardOpen",
+                    "testNewCaseWizard",
+                    "testStartAddLogicalFilesDataSource",
+                    "testConfigureIngest1",
+                    "testConfigureHash",
+                    "testConfigureIngest2",
+                    "testConfigureSearch",
+                    "testAddSourceWizard1",
+                    "testIngest",
+                    "testGenerateReportToolbar",
+                    "testGenerateReportButton");
+        }
+
         return NbModuleSuite.create(conf);
 
 
@@ -118,7 +151,7 @@ public class RegressionTest extends TestCase {
     @Override
     public void setUp() {
 
-        logger.info("########  " + System.getProperty("img_path") + "  #######");
+        logger.info("########  " + getEscapedPath(System.getProperty("img_path")) + "  #######");
         Timeouts.setDefault("ComponentOperator.WaitComponentTimeout", 1000000);
     }
 
@@ -142,7 +175,7 @@ public class RegressionTest extends TestCase {
         JTextFieldOperator jtfo1 = new JTextFieldOperator(wo, 1);
         jtfo1.typeText("AutopsyTestCase"); // Name the case "AutopsyTestCase"
         JTextFieldOperator jtfo0 = new JTextFieldOperator(wo, 0);
-        jtfo0.typeText(System.getProperty("out_path"));
+        jtfo0.typeText(getEscapedPath(System.getProperty("out_path")));
         wo.btNext().clickMouse();
         JTextFieldOperator jtfo2 = new JTextFieldOperator(wo, 0);
         jtfo2.typeText("000"); // Set the case number
@@ -152,19 +185,32 @@ public class RegressionTest extends TestCase {
         wo.btFinish().clickMouse();
     }
 
-    public void testStartAddDataSource() {
+    public void testStartAddImageFileDataSource() {
         logger.info("Starting Add Image process");
         WizardOperator wo = new WizardOperator("Add Data");
         JTextFieldOperator jtfo0 = new JTextFieldOperator(wo, 0);
-        String img_path = System.getProperty("img_path");
+        String img_path = getEscapedPath(System.getProperty("img_path"));
-        if (img_path.startsWith("\\")) {
-            img_path = "\\" + img_path;
-        }
         String imageDir = img_path;
         ((JTextField) jtfo0.getSource()).setText(imageDir);
         wo.btNext().clickMouse();
     }
 
+    public void testStartAddLogicalFilesDataSource() {
+        logger.info("Starting Add Logical Files process");
+        WizardOperator wo = new WizardOperator("Add Data");
+        JComboBoxOperator comboBoxOperator = new JComboBoxOperator(wo);
+        // select the item indexed 2 (Logical Files) from the drop-down list.
+        comboBoxOperator.selectItem(2);
+        JButtonOperator addButtonOperator = new JButtonOperator(wo, "Add");
+        addButtonOperator.pushNoBlock();
+        JFileChooserOperator fileChooserOperator = new JFileChooserOperator();
+        fileChooserOperator.setCurrentDirectory(new File(getEscapedPath(System.getProperty("img_path"))));
+        // set the current directory one level above the directory containing logicalFileSet folder.
+        fileChooserOperator.goUpLevel();
+        fileChooserOperator.chooseFile(new File(getEscapedPath(System.getProperty("img_path"))).getName());
+        wo.btNext().clickMouse();
+    }
+
     public void testAddSourceWizard1() {
         WizardOperator wo = new WizardOperator("Add Data");
         while (!wo.btFinish().isEnabled()) {
@@ -189,8 +235,8 @@ public class RegressionTest extends TestCase {
         JDialog hashMainDialog = JDialogOperator.waitJDialog("Hash Set Configuration", false, false);
         JDialogOperator hashMainDialogOperator = new JDialogOperator(hashMainDialog);
         List<String> databases = new ArrayList<String>();
-        databases.add(System.getProperty("nsrl_path"));
+        databases.add(getEscapedPath(System.getProperty("nsrl_path")));
-        databases.add(System.getProperty("known_bad_path"));
+        databases.add(getEscapedPath(System.getProperty("known_bad_path")));
         for (String database : databases) {
             JButtonOperator importButtonOperator = new JButtonOperator(hashMainDialogOperator, "Import");
             importButtonOperator.pushNoBlock();
@@ -228,7 +274,7 @@ public class RegressionTest extends TestCase {
         logger.info("Search Configure");
         JDialog jd = JDialogOperator.waitJDialog("Advanced Keyword Search Configuration", false, false);
         JDialogOperator jdo = new JDialogOperator(jd);
-        String words = System.getProperty("keyword_path");
+        String words = getEscapedPath(System.getProperty("keyword_path"));
         JButtonOperator jbo0 = new JButtonOperator(jdo, "Import List", 0);
         jbo0.pushNoBlock();
         JFileChooserOperator jfco0 = new JFileChooserOperator();
@@ -318,7 +364,7 @@ public class RegressionTest extends TestCase {
         try {
             Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
             BufferedImage capture = new Robot().createScreenCapture(screenRect);
-            String outPath = System.getProperty("out_path");
+            String outPath = getEscapedPath(System.getProperty("out_path"));
             ImageIO.write(capture, "png", new File(outPath + "\\" + name + ".png"));
             new Timeout("pausing", 1000).sleep(); // give it a second to save
         } catch (IOException ex) {

build.xml

@@ -71,10 +71,15 @@
         <copy file="${basedir}/KNOWN_ISSUES.txt" tofile="${zip-tmp}/${app.name}/KNOWN_ISSUES.txt"/>
         <unzip src="${thirdparty.dir}/gstreamer/${os.family}/i386/0.10.7/gstreamer.zip" dest="${zip-tmp}/${app.name}/gstreamer"/>
         <copy file="${basedir}/icons/icon.ico" tofile="${zip-tmp}/${app.name}/icon.ico" overwrite="true"/>
-        <!-- Copy the Autopsy documentation to the docs folder -->
+        <!-- Copy the Autopsy documentation to the docs folder if user-docs exist -->
-        <copy flatten="true" todir="${zip-tmp}/${app.name}/docs">
+        <if>
-            <fileset dir="${basedir}/docs/doxygen-user/user-docs"/>
+            <available file="${basedir}/docs/doxygen-user/user-docs" type="dir"/>
-        </copy>
+            <then>
+                <copy flatten="true" todir="${zip-tmp}/${app.name}/docs">
+                    <fileset dir="${basedir}/docs/doxygen-user/user-docs"/>
+                </copy>
+            </then>
+        </if>
         
         <antcall target="copyLibsToZip"/>
         

docs/doxygen/modDevPython.dox

@@ -61,8 +61,8 @@ To distribute and share your Python module, ZIP up the folder and send it around
 Jython allows you to access all of the Java classes.  So, you should read the following sections of this document.  All you should ignore is the Java environment setup sections.
 
 There are only two types of modules that you can make with Python.  Those (along with a sample file) are listed below:
-- Ingest Modules (both file-level and data source-level): https://github.com/sleuthkit/autopsy/blob/develop/Core/src/org/sleuthkit/autopsy/examples/ingestmodule.py
+- Ingest Modules (both file-level and data source-level): https://github.com/sleuthkit/autopsy/blob/develop/pythonExamples/ingestmodule.py
-- Report Modules: https://github.com/sleuthkit/autopsy/blob/develop/Core/src/org/sleuthkit/autopsy/examples/reportmodule.py
+- Report Modules: https://github.com/sleuthkit/autopsy/blob/develop/pythonExamples/reportmodule.py
 
 */
 

test/script/regression.py

@@ -247,8 +247,8 @@ class TestRunner(object):
         if test_data.main_config.timing:
             print("Run time test passed: ", test_data.run_time_passed)
             test_data.overall_passed = (test_data.html_report_passed and
-            test_data.errors_diff_passed and test_data.db_diff_passed and
+            test_data.errors_diff_passed and test_data.db_diff_passed)
-            test_data.run_time_passed)
+        #    test_data.run_time_passed not considered for test_data.overall_passed
         # otherwise, do the usual
         else:
             test_data.overall_passed = (test_data.html_report_passed and
@@ -909,6 +909,11 @@ class TestResultsDiffer(object):
         oldtime = int(line[:line.find("ms")].replace(',', ''))
         file.close()
 
+        # If we don't have a previous run time bail out here to
+        # avoid dividing by zero below.
+        if oldtime == 0:
+            return True
+            
         newtime = test_data.total_ingest_time
         
         # write newtime to the file inside the report dir.
@@ -1263,16 +1268,13 @@ class Logs(object):
             rep_path = rep_path.replace("\\\\", "\\")
             for file in os.listdir(logs_path):
                 log = codecs.open(make_path(logs_path, file), "r", "utf_8")
-                for line in log:
+                try:
-                    line = line.replace(rep_path, "test_data")
+                    for line in log:
-                    if line.startswith("Exception"):
+                        line = line.replace(rep_path, "test_data")
-                        common_log.write(file +": " +  line)
+                        if line.startswith("SEVERE"):
-                    elif line.startswith("Error"):
+                            common_log.write(file +": " +  line)
-                        common_log.write(file +": " +  line)
+                except UnicodeDecodeError as e:
-                    elif line.startswith("SEVERE"):
+                    pass
-                        common_log.write(file +":" +  line)
-                    else:
-                        warning_log.write(file +": " +  line)
                 log.close()
             common_log.write("\n")
             common_log.close()

test/script/regression_utils.py

@@ -38,14 +38,14 @@ def wgetcwd():
 def file_exists(file):
 	try:
 		if os.path.exists(file):
-			return os.path.isfile(file)
+			return os.path.exists(file) and os.path.isfile(file)
 	except:
 		return False
 
 # Verifies a directory's existance
 def dir_exists(dir):
 	try:
-		return os.path.exists(dir)
+		return os.path.exists(dir) and os.path.isdir(dir)
 	except:
 		return False
 
@@ -71,6 +71,8 @@ def required_input_file(name):
 		return False
 
 def image_type(image_file):
+    if (dir_exists(image_file)):
+        return IMGTYPE.LOGICAL
     ext_start = image_file.rfind(".")
     if (ext_start == -1):
         return IMGTYPE.UNKNOWN
@@ -86,12 +88,15 @@ def image_type(image_file):
 
 # Returns the type of image file, based off extension
 class IMGTYPE:
-    RAW, ENCASE, SPLIT, UNKNOWN = range(4)
+    RAW, ENCASE, SPLIT, LOGICAL, UNKNOWN = range(5)
 
 def get_image_name(image_file):
     path_end = image_file.rfind("/")
     path_end2 = image_file.rfind("\\")
     ext_start = image_file.rfind(".")
+    if (image_type(image_file) == IMGTYPE.LOGICAL):
+        name = image_file[path_end2+1:]
+        return name
     if(ext_start == -1):
         name = image_file
     if(path_end2 != -1):

test/script/tskdbdiff.py

@@ -91,7 +91,7 @@ class TskDbDiff(object):
             self._dump_diff = TskDbDiff._get_tmp_file("DBDump-Diff", ".txt")
         else:
             self._bb_dump = os.path.join(self.output_dir, "BlackboardDump.txt")
-            self._bb_dump_diff = os.path.join(self.output_dir, "BlackboardDumpDiff.txt")
+            self._bb_dump_diff = os.path.join(self.output_dir, "BlackboardDump-Diff.txt")
             self._dump = os.path.join(self.output_dir, "DBDump.txt")
             self._dump_diff = os.path.join(self.output_dir, "DBDump-Diff.txt")
 

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -78,7 +78,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
             return ProcessResult.OK;
         }
 
-        if (abstractFile.isVirtual()) {
+        if ((abstractFile.isFile() == false)) {
             return ProcessResult.OK;
         }
 
@@ -144,10 +144,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         } else if (result == PstParser.ParseResult.ENCRYPT) {
             // encrypted pst: Add encrypted file artifact
             try {
-                BlackboardArtifact generalInfo = abstractFile.getGenInfoArtifact();
+                BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED);
-                generalInfo.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ENCRYPTION_DETECTED.getTypeID(),
+                artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
-                        EmailParserModuleFactory.getModuleName(),
+                services.fireModuleDataEvent(new ModuleDataEvent(EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED));
-                        NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel")));
             } catch (TskCoreException ex) {
                 logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS
             }