Merge branch 'develop' of github.com:sleuthkit/autopsy into solr8_libraries

Core/nbproject/project.xml

@@ -345,6 +345,7 @@
                 <package>org.sleuthkit.autopsy.textextractors.configs</package>
                 <package>org.sleuthkit.autopsy.texttranslation</package>
                 <package>org.sleuthkit.datamodel</package>
+                <package>org.sleuthkit.datamodel.blackboardutils</package>
             </public-packages>
             <class-path-extension>
                 <runtime-relative-path>ext/commons-lang3-3.8.1.jar</runtime-relative-path>

Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java

@@ -227,6 +227,7 @@ public class ExtractedContent implements AutopsyVisitableItem {
         // maps the artifact type to its child node 
         private final HashMap<BlackboardArtifact.Type, TypeNode> typeNodeList = new HashMap<>();
 
+        @SuppressWarnings("deprecation")
         TypeFactory() {
             super();
 

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java

@@ -22,25 +22,19 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 import java.util.logging.Level;
 import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.datamodel.BlackboardAttribute;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
- * Template parse method for reports that make blackboard attributes from a
- * single key value pair.
- *
- * This parse implementation will create 1 artifact per XRY entity.
+ * Template parse method for reports that make artifacts from a single XRY
+ * Entity.
  */
-abstract class AbstractSingleKeyValueParser implements XRYFileParser {
+abstract class AbstractSingleEntityParser implements XRYFileParser {
     
-    private static final Logger logger = Logger.getLogger(AbstractSingleKeyValueParser.class.getName());
+    private static final Logger logger = Logger.getLogger(AbstractSingleEntityParser.class.getName());
 
-    private static final char KEY_VALUE_DELIMITER = ':';
-    
     protected static final String PARSER_NAME = "XRY DSP";
 
     @Override
@@ -52,14 +46,13 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser {
             String xryEntity = reader.nextEntity();
             String[] xryLines = xryEntity.split("\n");
 
-            List<BlackboardAttribute> attributes = new ArrayList<>();
+            List<XRYKeyValuePair> keyValuePairs = new ArrayList<>();
 
             //First line of the entity is the title, the entity will always be non-empty.
             logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
 
             String namespace = "";
             //Process each line, searching for a key value pair or a namespace.
-            //If neither are found, an error message is logged.
             for (int i = 1; i < xryLines.length; i++) {
                 String xryLine = xryLines[i];
 
@@ -71,60 +64,46 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser {
                     continue;
                 }
 
-                //Find the XRY key on this line. Assume key is the value between
-                //the start of the line and the first delimiter.
-                int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER);
-                if (keyDelimiter == -1) {
+                //Check if this line resembles a Key Value pair.
+                if(!XRYKeyValuePair.isPair(xryLine)) {
                     logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value "
-                            + "pair on this line (in brackets) [ %s ], but one was not detected."
-                            + " Here is the previous line [ %s ]. What does this mean?", xryLine, xryLines[i - 1]));
+                            + "pair on this line (in brackets) [ %s ], but one was not detected.", 
+                            xryLine));
                     continue;
                 }
-                String key = xryLine.substring(0, keyDelimiter).trim();
-                String value = xryLine.substring(keyDelimiter + 1).trim();
+                
+                XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine, namespace);
 
-                if (!isKey(key)) {
+                //Verify the implementation recognizes the key.
+                if (!canProcess(pair)) {
                     logger.log(Level.WARNING, String.format("[XRY DSP] The following key, "
-                            + "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..."
-                            + " Here is the previous line [ %s ] for context. What does this key mean?", key, value, xryLines[i - 1]));
+                            + "value pair (in brackets) [ %s ] was not recognized. Discarding...",
+                            pair));
                     continue;
                 }
 
-                if (value.isEmpty()) {
-                    logger.log(Level.WARNING, String.format("[XRY DSP] The following key "
-                            + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding..."
-                            + " Here is the previous line for context [ %s ]. What does this mean?", key, xryLines[i - 1]));
+                //Empty values are meaningless for blackboard attributes.
+                if (pair.getValue().isEmpty()) {
+                    logger.log(Level.WARNING, String.format("[XRY DSP] The following key value pair"
+                            + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding...", 
+                            pair));
                     continue;
                 }
-
-                //Create the attribute, if any.
-                Optional<BlackboardAttribute> attribute = makeAttribute(namespace, key, value);
-                if(attribute.isPresent()) {
-                    attributes.add(attribute.get());
-                }
+                
+                keyValuePairs.add(pair);
             }
-
-            //Only create artifacts with non-empty attributes.
-            if (!attributes.isEmpty()) {
-                makeArtifact(attributes, parent);
+            
+            if(!keyValuePairs.isEmpty()) {
+                makeArtifact(keyValuePairs, parent);
             }
         }
     }
 
     /**
-     * Determines if the key candidate is a known key. A key candidate is a
-     * string literal that begins a line and is terminated by a semi-colon.
-     *
-     * Ex:
-     *
-     * Call Type : Missed
-     *
-     * "Call Type" would be the key candidate that was extracted.
-     *
-     * @param key Key to test. These keys are trimmed of whitespace only.
-     * @return Indication if this key can be processed.
+     * Determines if the XRY key value pair can be processed by the 
+     * implementation.
      */
-    abstract boolean isKey(String key);
+    abstract boolean canProcess(XRYKeyValuePair pair);
 
     /**
      * Determines if the namespace candidate is a known namespace. A namespace
@@ -144,19 +123,8 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser {
     abstract boolean isNamespace(String nameSpace);
 
     /**
-     * Creates an attribute from the extracted key value pair.
-     * 
-     * @param nameSpace The namespace of this key value pair.
-     * It will have been verified with isNamespace, otherwise it will be empty.
-     * @param key The key that was verified with isKey.
-     * @param value The value associated with that key.
-     * @return The corresponding blackboard attribute, if any.
+     * Makes an artifact from the parsed key value pairs.
      */
-    abstract Optional<BlackboardAttribute> makeAttribute(String nameSpace, String key, String value);
-
-    /**
-     * Makes an artifact from the parsed attributes.
-     */
-    abstract void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException;
+    abstract void makeArtifact(List<XRYKeyValuePair> keyValuePairs, Content parent) throws TskCoreException;
 
 }

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java

@@ -18,9 +18,16 @@
  */
 package org.sleuthkit.autopsy.datasourceprocessors.xry;
 
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalQueries;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.logging.Level;
@@ -33,71 +40,79 @@ import org.sleuthkit.datamodel.TskCoreException;
 /**
  * Parses XRY Calls files and creates artifacts.
  */
-final class XRYCallsFileParser extends AbstractSingleKeyValueParser {
+final class XRYCallsFileParser extends AbstractSingleEntityParser {
 
     private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName());
 
     //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
     //function for more details.
     private static final DateTimeFormatter DATE_TIME_PARSER
-            = DateTimeFormatter.ofPattern("O a h:m:s M/d/y");
+            = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
+
+    private static final String DEVICE_LOCALE = "(device)";
+    private static final String NETWORK_LOCALE = "(network)";
+
     /**
-     * All of the known XRY keys for call reports.
+     * All of the known XRY keys for call reports and their corresponding
+     * blackboard attribute types, if any.
      */
     private enum XryKey {
-        TEL("tel"),
-        NAME_MATCHED("name (matched)"),
-        TIME("time"),
-        DIRECTION("direction"),
-        CALL_TYPE("call type"),
-        DURATION("duration"),
-        STORAGE("storage"),
-        INDEX("index"),
-        NAME("name"),
-        NUMBER("number");
-        
+        NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME),
+        TIME("time", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME),
+        DIRECTION("direction", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION),
+        CALL_TYPE("call type", null),
+        NUMBER("number", null),
+        TEL("tel", null),
+        TO("to", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO),
+        FROM("from", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM),
+        DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED),
+        DURATION("duration", null),
+        STORAGE("storage", null),
+        INDEX("index", null),
+        TYPE("type", null),
+        NAME("name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME);
+
         private final String name;
-        XryKey(String name) {
+        private final BlackboardAttribute.ATTRIBUTE_TYPE type;
+
+        XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) {
             this.name = name;
+            this.type = type;
         }
-        
+
+        public BlackboardAttribute.ATTRIBUTE_TYPE getType() {
+            return type;
+        }
+
         /**
          * Indicates if the display name of the XRY key is a recognized type.
-         * 
-         * @param xryKey
-         * @return 
          */
-        public static boolean contains(String xryKey) {
-            String normalizedKey = xryKey.trim().toLowerCase();
-            for(XryKey keyChoice : XryKey.values()) {
-                if(keyChoice.name.equals(normalizedKey)) {
-                    return true;
-                }
+        public static boolean contains(String key) {
+            try {
+                XryKey.fromDisplayName(key);
+                return true;
+            } catch (IllegalArgumentException ex) {
+                return false;
             }
-            
-            return false;
         }
-        
+
         /**
          * Matches the display name of the xry key to the appropriate enum type.
-         * 
-         * It is assumed that XRY key string is recognized. Otherwise,
-         * an IllegalArgumentException is thrown. Test all membership
-         * with contains() before hand.
-         * 
-         * @param xryKey
-         * @return 
+         *
+         * It is assumed that XRY key string is recognized. Otherwise, an
+         * IllegalArgumentException is thrown. Test all membership with
+         * contains() before hand.
          */
-        public static XryKey fromDisplayName(String xryKey) {
-            String normalizedKey = xryKey.trim().toLowerCase();
-            for(XryKey keyChoice : XryKey.values()) {
-                if(keyChoice.name.equals(normalizedKey)) {
+        public static XryKey fromDisplayName(String key) {
+            String normalizedKey = key.trim().toLowerCase();
+            for (XryKey keyChoice : XryKey.values()) {
+                if (normalizedKey.equals(keyChoice.name)) {
                     return keyChoice;
                 }
             }
-            
+
             throw new IllegalArgumentException(String.format("Key [%s] was not found."
-                    + " All keys should be tested with contains.", xryKey));
+                    + " All keys should be tested with contains.", key));
         }
     }
 
@@ -108,55 +123,50 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser {
         TO("to"),
         FROM("from"),
         NONE(null);
-        
+
         private final String name;
+
         XryNamespace(String name) {
             this.name = name;
         }
-        
+
         /**
-         * Indicates if the display name of the XRY namespace is a recognized type.
-         * 
-         * @param xryNamespace
-         * @return 
+         * Indicates if the display name of the XRY namespace is a recognized
+         * type.
          */
         public static boolean contains(String xryNamespace) {
-            String normalizedNamespace = xryNamespace.trim().toLowerCase();
-            for(XryNamespace keyChoice : XryNamespace.values()) {
-                if(normalizedNamespace.equals(keyChoice.name)) {
-                    return true;
-                }
+            try {
+                XryNamespace.fromDisplayName(xryNamespace);
+                return true;
+            } catch (IllegalArgumentException ex) {
+                return false;
             }
-            
-            return false;
         }
-        
+
         /**
-         * Matches the display name of the xry namespace to the appropriate enum type.
-         * 
-         * It is assumed that XRY namespace string is recognized. Otherwise,
-         * an IllegalArgumentException is thrown. Test all membership
-         * with contains() before hand.
-         * 
-         * @param xryNamespace
-         * @return 
+         * Matches the display name of the xry namespace to the appropriate enum
+         * type.
+         *
+         * It is assumed that XRY namespace string is recognized. Otherwise, an
+         * IllegalArgumentException is thrown. Test all membership with
+         * contains() before hand.
          */
         public static XryNamespace fromDisplayName(String xryNamespace) {
             String normalizedNamespace = xryNamespace.trim().toLowerCase();
-            for(XryNamespace keyChoice : XryNamespace.values()) {
-                if(normalizedNamespace.equals(keyChoice.name)) {
+            for (XryNamespace keyChoice : XryNamespace.values()) {
+                if (normalizedNamespace.equals(keyChoice.name)) {
                     return keyChoice;
                 }
             }
-            
+
             throw new IllegalArgumentException(String.format("Key [%s] was not found."
                     + " All keys should be tested with contains.", xryNamespace));
         }
     }
 
     @Override
-    boolean isKey(String key) {
-        return XryKey.contains(key);
+    boolean canProcess(XRYKeyValuePair pair) {
+        return XryKey.contains(pair.getKey());
     }
 
     @Override
@@ -165,77 +175,75 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser {
     }
 
     @Override
-    Optional<BlackboardAttribute> makeAttribute(String nameSpace, String key, String value) {
-        XryKey xryKey = XryKey.fromDisplayName(key);
+    void makeArtifact(List<XRYKeyValuePair> keyValuePairs, Content parent) throws TskCoreException {
+        List<BlackboardAttribute> attributes = new ArrayList<>();
+        for(XRYKeyValuePair pair : keyValuePairs) {
+            Optional<BlackboardAttribute> attribute = getBlackboardAttribute(pair);
+            if(attribute.isPresent()) {
+                attributes.add(attribute.get());
+            }
+        }
+        if(!attributes.isEmpty()) {
+            BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG);
+            artifact.addAttributes(attributes);
+        }
+    }
+    
+    /**
+     * Creates the appropriate blackboard attribute given a single XRY Key Value
+     * pair, if any. Most XRY keys are mapped to an attribute type in the enum above.
+     */
+    private Optional<BlackboardAttribute> getBlackboardAttribute(XRYKeyValuePair pair) {
+        XryKey xryKey = XryKey.fromDisplayName(pair.getKey());
         XryNamespace xryNamespace = XryNamespace.NONE;
-        if(XryNamespace.contains(nameSpace)) {
-            xryNamespace = XryNamespace.fromDisplayName(nameSpace);
+        if (XryNamespace.contains(pair.getNamespace())) {
+            xryNamespace = XryNamespace.fromDisplayName(pair.getNamespace());
         }
 
         switch (xryKey) {
-            case DIRECTION:
-                return Optional.of(new BlackboardAttribute(
-                        BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, 
-                        PARSER_NAME, value));
-            case NAME_MATCHED:
-                return Optional.of(new BlackboardAttribute(
-                        BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, 
-                        PARSER_NAME, value));
-            case NUMBER:
-                return Optional.of(new BlackboardAttribute(
-                        BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, 
-                        PARSER_NAME, value));
             case TEL:
+            case NUMBER:
                 //Apply the namespace
                 switch (xryNamespace) {
                     case FROM:
                         return Optional.of(new BlackboardAttribute(
                                 BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM,
-                                PARSER_NAME, value));
+                                PARSER_NAME, pair.getValue()));
                     case TO:
                         return Optional.of(new BlackboardAttribute(
                                 BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO,
-                                PARSER_NAME, value));
+                                PARSER_NAME, pair.getValue()));
                     default:
                         return Optional.of(new BlackboardAttribute(
                                 BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
-                                PARSER_NAME, value));
+                                PARSER_NAME, pair.getValue()));
                 }
             case TIME:
                 try {
                     //Tranform value to seconds since epoch
-                    long dateTimeSinceEpoch = calculateSecondsSinceEpoch(value);
-                    return Optional.of(new BlackboardAttribute(
-                            BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, 
+                    long dateTimeSinceEpoch = calculateSecondsSinceEpoch(pair.getValue());
+                    return Optional.of(new BlackboardAttribute(xryKey.getType(),
                             PARSER_NAME, dateTimeSinceEpoch));
                 } catch (DateTimeParseException ex) {
                     logger.log(Level.WARNING, String.format("[XRY DSP] Assumption"
                             + " about the date time formatting of call logs is "
-                            + "not right. Here is the value [ %s ]", value), ex);
+                            + "not right. Here is the value [ %s ]", pair.getValue()), ex);
                     return Optional.empty();
                 }
-            case DURATION:
-            case STORAGE:
-            case INDEX:
-            case CALL_TYPE:
-                //Ignore for now, don't need more data.
-                return Optional.empty();
-            case NAME:
-                logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was "
-                    + "recognized but more examples of its values are needed "
-                    + "to make a decision on an appropriate TSK attribute. "
-                        + "Here is the value [%s].", key, value));
-                return Optional.empty();
             default:
-                throw new IllegalArgumentException(String.format("Key [ %s ] "
-                        + "passed the isKey() test but was not matched.", key));
-        }
-    }
+                //Otherwise, the XryKey enum contains the correct BlackboardAttribute
+                //type.
+                if (xryKey.getType() != null) {
+                    return Optional.of(new BlackboardAttribute(xryKey.getType(),
+                            PARSER_NAME, pair.getValue()));
+                }
 
-    @Override
-    void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException {
-        BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG);
-        artifact.addAttributes(attributes);
+                logger.log(Level.INFO, String.format("[XRY DSP] Key value pair "
+                        + "(in brackets) [ %s ] was recognized but "
+                        + "more data or time is needed to finish implementation. Discarding... ",
+                        pair));
+                return Optional.empty();
+        }
     }
 
     /**
@@ -247,12 +255,16 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser {
      * @return A purer date time value.
      */
     private String removeDateTimeLocale(String dateTime) {
-        int index = dateTime.indexOf('(');
-        if (index == -1) {
-            return dateTime;
+        String result = dateTime;
+        int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
+        if (deviceIndex != -1) {
+            result = result.substring(0, deviceIndex);
         }
-
-        return dateTime.substring(0, index);
+        int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
+        if (networkIndex != -1) {
+            result = result.substring(0, networkIndex);
+        }
+        return result;
     }
 
     /**
@@ -265,42 +277,48 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser {
         String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
         /**
          * The format of time in XRY Messages reports is of the form:
-         * 
-         *      1/3/1990 1:23:54 AM UTC+4
-         * 
-         * In our current version of Java (openjdk-1.8.0.222), there is
-         * a bug with having the timezone offset (UTC+4 or GMT-7) at the 
-         * end of the date time input. This is fixed in later versions
-         * of the JDK (9 and beyond). 
-         * https://bugs.openjdk.java.net/browse/JDK-8154050
-         * Rather than update the JDK to accommodate this, the components of 
-         * the date time string are reversed:
-         * 
-         *      UTC+4 AM 1:23:54 1/3/1990
-         * 
+         *
+         * 1/3/1990 1:23:54 AM UTC+4
+         *
+         * In our current version of Java (openjdk-1.8.0.222), there is a bug
+         * with having the timezone offset (UTC+4 or GMT-7) at the end of the
+         * date time input. This is fixed in later versions of the JDK (9 and
+         * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than
+         * update the JDK to accommodate this, the components of the date time
+         * string are reversed:
+         *
+         * UTC+4 AM 1:23:54 1/3/1990
+         *
          * The java time package will correctly parse this date time format.
          */
         String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
         /**
-         * Furthermore, the DateTimeFormatter's timezone offset letter ('O') does
-         * not recognize UTC but recognizes GMT. According to 
-         * https://en.wikipedia.org/wiki/Coordinated_Universal_Time,
-         * GMT only differs from UTC by at most 1 second and so substitution
-         * will only introduce a trivial amount of error.
+         * Furthermore, the DateTimeFormatter's timezone offset letter ('O')
+         * does not recognize UTC but recognizes GMT. According to
+         * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only
+         * differs from UTC by at most 1 second and so substitution will only
+         * introduce a trivial amount of error.
          */
         String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
-        ZonedDateTime zonedDateTime = ZonedDateTime.parse(reversedDateTimeWithGMT, DATE_TIME_PARSER);
-        return zonedDateTime.toEpochSecond();
+        TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
+                ZonedDateTime::from,
+                LocalDateTime::from,
+                OffsetDateTime::from);
+        //Query for the ZoneID
+        if (result.query(TemporalQueries.zoneId()) == null) {
+            //If none, assumed GMT+0.
+            return ZonedDateTime.of(LocalDateTime.from(result),
+                    ZoneId.of("GMT")).toEpochSecond();
+        } else {
+            return Instant.from(result).getEpochSecond();
+        }
     }
-    
+
     /**
      * Reverses the order of the date time components.
-     * 
-     * Example: 
-     *  1/3/1990 1:23:54 AM UTC+4 
-     * becomes
-     *  UTC+4 AM 1:23:54 1/3/1990
-     * 
+     *
+     * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990
+     *
      * @param dateTime
      * @return
      */

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java

@@ -18,6 +18,7 @@
  */
 package org.sleuthkit.autopsy.datasourceprocessors.xry;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,7 +33,7 @@ import org.sleuthkit.datamodel.TskCoreException;
 /**
  * Parses XRY Contacts-Contacts files and creates artifacts.
  */
-final class XRYContactsFileParser extends AbstractSingleKeyValueParser {
+final class XRYContactsFileParser extends AbstractSingleEntityParser {
     
     private static final Logger logger = Logger.getLogger(XRYContactsFileParser.class.getName());
 
@@ -42,9 +43,11 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser {
         put("name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME);
         put("tel", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER);
         put("mobile", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_MOBILE);
+        put("home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_HOME);
         put("related application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME);
         put("address home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LOCATION);
         put("email home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME);
+        put("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED);
         
         //Ignoring or need more information to decide.
         put("storage", null);
@@ -54,10 +57,10 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser {
         put("account name", null);
         
     }};
-
+    
     @Override
-    boolean isKey(String key) {
-        String normalizedKey = key.toLowerCase();
+    boolean canProcess(XRYKeyValuePair pair) {
+        String normalizedKey = pair.getKey().toLowerCase();
         return XRY_KEYS.containsKey(normalizedKey);
     }
 
@@ -67,29 +70,36 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser {
         return false;
     }
 
-    @Override
-    Optional<BlackboardAttribute> makeAttribute(String nameSpace, String key, String value) {
-        String normalizedKey = key.toLowerCase();
-        if(XRY_KEYS.containsKey(normalizedKey)) {
-            BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey);
-            if(attrType != null) {
-                return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, value));
-            }
-            
-            logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was "
-                    + "recognized but more examples of its values are needed "
-                    + "to make a decision on an appropriate TSK attribute. "
-                        + "Here is the value [%s].", key, value));
-            return Optional.empty();
+    /**
+     * Creates the appropriate blackboard attribute given a single XRY Key Value
+     * pair. 
+     */
+    private Optional<BlackboardAttribute> getBlackboardAttribute(XRYKeyValuePair pair) {
+        String normalizedKey = pair.getKey().toLowerCase();
+        BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey);
+        if(attrType != null) {
+            return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, pair.getValue()));
         }
-        
-        throw new IllegalArgumentException(String.format("Key [ %s ] passed the "
-                + "isKey() test but was not matched.", key));
+
+        logger.log(Level.INFO, String.format("[XRY DSP] Key value pair "
+               + "(in brackets) [ %s ] was recognized but we need "
+               + "more data or time to finish implementation. Discarding... ", 
+               pair));
+        return Optional.empty();
     }
     
     @Override
-    void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException {
-        BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT);
-        artifact.addAttributes(attributes);
+    void makeArtifact(List<XRYKeyValuePair> keyValuePairs, Content parent) throws TskCoreException {
+        List<BlackboardAttribute> attributes = new ArrayList<>();
+        for(XRYKeyValuePair pair : keyValuePairs) {
+            Optional<BlackboardAttribute> attribute = getBlackboardAttribute(pair);
+            if(attribute.isPresent()) {
+                attributes.add(attribute.get());
+            }
+        }
+        if(!attributes.isEmpty()) {
+            BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT);
+            artifact.addAttributes(attributes);
+        }
     }
 }

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java

@@ -21,9 +21,15 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry;
 import java.beans.PropertyChangeListener;
 import java.beans.PropertyChangeSupport;
 import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Optional;
 import javax.swing.JFileChooser;
 import javax.swing.JPanel;
+import org.apache.commons.lang3.StringUtils;
 import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor;
+import org.sleuthkit.autopsy.coreutils.ModuleSettings;
 
 /**
  * Allows an examiner to configure the XRY Data source processor.
@@ -32,10 +38,15 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor;
 final class XRYDataSourceProcessorConfigPanel extends JPanel {
 
     private static final long serialVersionUID = 1L;
+    
+    private static final String PROP_LAST_USED_PATH = "LAST_USED_PATH";
+    private static final String SETTINGS_CONTEXT = "XRYDataSourceProcessorConfigPanel_Settings";
+    
     private static final XRYDataSourceProcessorConfigPanel INSTANCE = 
             new XRYDataSourceProcessorConfigPanel();
     
-    //Communicates 
+    //Used to communicate with the DSP infrastructure. This config
+    //panel will indicate when it is ready for an update.
     private final PropertyChangeSupport pcs;
     
     /**
@@ -47,6 +58,31 @@ final class XRYDataSourceProcessorConfigPanel extends JPanel {
         pcs = new PropertyChangeSupport(this);
     }
     
+    /**
+     * Persists the last used path between application runs.
+     */
+    private void setLastUsedPath(Path selection) {
+        Path parent = selection.getParent();
+        ModuleSettings.setConfigSetting(SETTINGS_CONTEXT, 
+                PROP_LAST_USED_PATH, parent.toString());
+    }
+
+    /**
+     * Retrieves the last used path, if any. This path will be saved across 
+     * application runs.
+     */
+    private Optional<Path> getLastUsedPath() {
+        String lastFolderPath = ModuleSettings.getConfigSetting(
+                SETTINGS_CONTEXT, PROP_LAST_USED_PATH);
+        if (StringUtils.isNotBlank(lastFolderPath)) {
+            Path lastPath = Paths.get(lastFolderPath);
+            if (Files.exists(lastPath)) {
+                return Optional.of(lastPath);
+            }
+        }
+        return Optional.empty();
+    }
+    
     /**
      * Gets the singleton XRYDataSourceProcessorConfigPanel.
      */
@@ -158,9 +194,14 @@ final class XRYDataSourceProcessorConfigPanel extends JPanel {
         JFileChooser fileChooser = new JFileChooser();
         fileChooser.setMultiSelectionEnabled(false);
         fileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
+        Optional<Path> lastUsedPath = getLastUsedPath();
+        if(lastUsedPath.isPresent()) {
+            fileChooser.setCurrentDirectory(lastUsedPath.get().toFile());
+        }
         int returnVal = fileChooser.showOpenDialog(this);
         if (returnVal == JFileChooser.APPROVE_OPTION) {
             File selection = fileChooser.getSelectedFile();
+            setLastUsedPath(selection.toPath());
             filePathTextField.setText(selection.getAbsolutePath());
             
             //This will notify the wizard to revalidate the data source processor.

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java

@@ -18,8 +18,6 @@
  */
 package org.sleuthkit.autopsy.datasourceprocessors.xry;
 
-import java.io.IOException;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -35,38 +33,38 @@ import org.sleuthkit.datamodel.TskCoreException;
 /**
  * Parses XRY Device-General Information files and creates artifacts.
  */
-final class XRYDeviceGenInfoFileParser implements XRYFileParser {
+final class XRYDeviceGenInfoFileParser extends AbstractSingleEntityParser {
 
     private static final Logger logger = Logger.getLogger(XRYDeviceGenInfoFileParser.class.getName());
 
-    //Human readable name of this parser.
-    private static final String PARSER_NAME = "XRY DSP";
-    private static final char KEY_VALUE_DELIMITER = ':';
-
     //All known XRY keys for Device Gen Info reports.
     private static final String ATTRIBUTE_KEY = "attribute";
     private static final String DATA_KEY = "data";
 
     //All of the known XRY Attribute values for device gen info. The value of the
-    //attribute keys are actionable for this parser. See parse() header for more
-    //details.
-    private static final Map<String, BlackboardAttribute.ATTRIBUTE_TYPE> KEY_TO_TYPE
+    //attribute keys are actionable for this parser.
+    //Ex:
+    // Data:        Nokia
+    // Attribute:   Device Type
+    private static final Map<String, BlackboardAttribute.ATTRIBUTE_TYPE> XRY_ATTRIBUTE_VALUES
             = new HashMap<String, BlackboardAttribute.ATTRIBUTE_TYPE>() {
         {
             put("device name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_NAME);
             put("device type", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MAKE);
             put("mobile id (imei)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI);
             put("security code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD);
+            put("unlock code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD);
             put("imei/meid", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI);
             put("model", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MODEL);
             put("wifi address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS);
+            put("subscriber id (imsi)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI);
 
             //There could be two of these on an artifact, not aware of a way
             //to distinguish between two DATE_TIMEs such as the ones below.
             put("device clock", null);
             put("pc clock", null);
 
-            //Ignore these for now, need more data.
+            //Ignore these for now, need more data or time to finish implementation.
             put("device family", null);
             put("advertising id", null);
             put("device status", null);
@@ -76,192 +74,98 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser {
             put("revision", null);
         }
     };
-
-    /**
-     * Device-General Information reports have 2 key value pairs for every
-     * attribute. The two only known keys are "Data" and "Attribute", where data
-     * is some generic information that the Attribute key describes.
-     *
-     * Example:
-     *
-     * Data:	        Nokia XYZ 
-     * Attribute:	Device Name
-     *
-     * This parse implementation assumes that the data field does not span
-     * multiple lines. If the data does span multiple lines, it will log an
-     * error describing an expectation for an "Attribute" key that is not found.
-     *
-     * @param reader The XRYFileReader that reads XRY entities from the
-     * Device-General Information report.
-     * @param parent The parent Content to create artifacts from.
-     * @throws IOException If an I/O error is encountered during report reading
-     * @throws TskCoreException If an error during artifact creation is
-     * encountered.
-     */
+    
+    
     @Override
-    public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException {
-        Path reportPath = reader.getReportPath();
-        logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString()));
-
-        while (reader.hasNextEntity()) {
-            String xryEntity = reader.nextEntity();
-            //Extract attributes from this entity.
-            List<BlackboardAttribute> attributes = createTSKAttributes(xryEntity);
-            if (!attributes.isEmpty()) {
-                //Save the artifact.
-                BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO);
-                artifact.addAttributes(attributes);
-            }
-        }
+    boolean canProcess(XRYKeyValuePair pair) {
+        String key = pair.getKey().trim().toLowerCase();
+        return key.equals(DATA_KEY) || key.equals(ATTRIBUTE_KEY);
     }
 
-    /**
-     * Parses the XRY entity, extracts key value pairs and creates blackboard
-     * attributes from these key value pairs.
-     * 
-     * @param xryEntity
-     * @return A collection of attributes from the XRY entity.
-     */
-    private List<BlackboardAttribute> createTSKAttributes(String xryEntity) {
-        //Examine this XRY entity line by line.
-        String[] xryLines = xryEntity.split("\n");
+    @Override
+    boolean isNamespace(String nameSpace) {
+        //No known namespaces
+        return false;
+    }
+    
+    @Override
+    void makeArtifact(List<XRYKeyValuePair> keyValuePairs, Content parent) throws TskCoreException {
         List<BlackboardAttribute> attributes = new ArrayList<>();
-
-        //First line of the entity is the title, the entity will always be non-empty.
-        logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0]));
-
-        for (int i = 1; i < xryLines.length; i++) {
-            String xryLine = xryLines[i];
-
-            //Expecting to see a "Data" key.
-            if (!hasDataKey(xryLine)) {
-                logger.log(Level.WARNING, String.format("[XRY DSP] Expected a "
-                        + "'Data' key on this line (in brackets) [ %s ], but none "
-                        + "was found. Discarding... Here is the previous line for"
-                        + " context [ %s ]. What does this mean?", 
-                        xryLine, xryLines[i - 1]));
-                continue;
+        for(int i = 0; i < keyValuePairs.size(); i+=2) {
+            Optional<BlackboardAttribute> attribute;
+            if(i + 1 == keyValuePairs.size()) {
+                attribute = getBlackboardAttribute(keyValuePairs.get(i));
+            } else {
+                attribute = getBlackboardAttribute(keyValuePairs.get(i), keyValuePairs.get(i+1));
             }
-
-            int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER);
-            String dataValue = xryLine.substring(dataKeyIndex + 1).trim();
-
-            /**
-             * If there is only a Data key in the XRY Entity, then assume it is
-             * the path to the device.
-             */
-            if (i + 1 == xryLines.length) {
-                attributes.add(new BlackboardAttribute(
-                        BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH,
-                        PARSER_NAME, dataValue));
-                continue;
-            }
-
-            String nextXryLine = xryLines[++i];
-
-            //Expecting to see an "Attribute" key
-            if (!hasXRYAttributeKey(nextXryLine)) {
-                logger.log(Level.WARNING, String.format("[XRY DSP] Expected an "
-                        + "'Attribute' key on this line (in brackets) [ %s ], "
-                        + "but none was found. Discarding... Here is the previous "
-                        + "line for context [ %s ]. What does this mean?", 
-                        nextXryLine, xryLine));
-                continue;
-            }
-
-            int attributeKeyIndex = nextXryLine.indexOf(KEY_VALUE_DELIMITER);
-            String attributeValue = nextXryLine.substring(attributeKeyIndex + 1).trim();
-            String normalizedAttributeValue = attributeValue.toLowerCase();
-
-            //Check if this value is known.
-            if (!isXRYAttributeValueRecognized(normalizedAttributeValue)) {
-                logger.log(Level.WARNING, String.format("[XRY DSP] Attribute value "
-                        + "(in brackets) [ %s ] was not recognized. Discarding... "
-                        + "Here is the data field for context [ %s ]. "
-                        + "What does this mean?", attributeValue, dataValue));
-                continue;
-            }
-
-            Optional<BlackboardAttribute> attribute = createTSKAttribute(
-                    normalizedAttributeValue, dataValue);
-            if (attribute.isPresent()) {
+            if(attribute.isPresent()) {
                 attributes.add(attribute.get());
             }
         }
-        return attributes;
+        if(!attributes.isEmpty()) {
+            BlackboardArtifact artifact = parent.newArtifact(
+                    BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO);
+            artifact.addAttributes(attributes);
+        }
     }
 
     /**
-     * Creates the appropriate blackboard attribute given the XRY Key Value pair.
-     * If the value is recognized but has no corresponding Blackboard
-     * attribute type, the Optional will be empty.
-     *
-     * A WARNING message will be logged for all recognized values that don't have
-     * a type. More data is needed to make a decision about the appropriate type.
-     *
-     * @param normalizedAttributeValue Normalized (trimmed and lowercased)
-     * attribute value to map.
-     * @param dataValue The value of the blackboard attribute.
-     * @return Corresponding BlackboardAttribute, if any.
+     * Creates the appropriate blackboard attribute given a single XRY Key Value
+     * pair. It is assumed that only 'Data' keys can appear by themselves.
+     * If a Data key is by itself, its value most closely resembles a TSK_PATH attribute.
      */
-    private Optional<BlackboardAttribute> createTSKAttribute(
-            String normalizedAttributeValue, String dataValue) {
-        BlackboardAttribute.ATTRIBUTE_TYPE attrType = KEY_TO_TYPE.get(normalizedAttributeValue);
+    private Optional<BlackboardAttribute> getBlackboardAttribute(XRYKeyValuePair pair) {
+        if (pair.hasKey(DATA_KEY)) {
+            return Optional.of(new BlackboardAttribute(
+                    BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH,
+                    PARSER_NAME, pair.getValue()));
+        }
+
+        logger.log(Level.WARNING, "Expected a 'Data' key value pair, but [ %s ] "
+                + "was found.", pair);
+
+        return Optional.empty();
+    }
+
+    /**
+     * Creates the appropriate blackboard attribute given two XRY Key Value
+     * pairs. The expectation is that one pair is the 'Data' key and the other is
+     * an 'Attribute' key. If the attribute value is recognized but has no corresponding
+     * Blackboard attribute type, the Optional will be empty.
+     */
+    private Optional<BlackboardAttribute> getBlackboardAttribute(XRYKeyValuePair firstPair, XRYKeyValuePair secondPair) {
+        String attributeValue;
+        String dataValue;
+        if (firstPair.hasKey(DATA_KEY) && secondPair.hasKey(ATTRIBUTE_KEY)) {
+            dataValue = firstPair.getValue();
+            attributeValue = secondPair.getValue();
+        } else if (firstPair.hasKey(ATTRIBUTE_KEY) && secondPair.hasKey(DATA_KEY)) {
+            dataValue = secondPair.getValue();
+            attributeValue = firstPair.getValue();
+        } else {
+            logger.log(Level.WARNING, String.format("[XRY DSP] Expected these key value"
+                    + " pairs (in brackets) [ %s ], [ %s ] to be an 'Attribute' and 'Data' "
+                    + "pair.", firstPair, secondPair));
+            return Optional.empty();
+        }
+
+        String normalizedAttributeValue = attributeValue.toLowerCase();
+        if (!XRY_ATTRIBUTE_VALUES.containsKey(normalizedAttributeValue)) {
+            logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair "
+                    + "(in brackets) [ %s : %s ] was not recognized. Discarding... ", 
+                    attributeValue, dataValue));
+            return Optional.empty();
+        }
+
+        BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_ATTRIBUTE_VALUES.get(normalizedAttributeValue);
         if (attrType == null) {
-            logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was "
-                    + "recognized but more examples of its values are needed "
-                    + "to make a decision on an appropriate TSK attribute. "
-                        + "Here is the value [%s].", normalizedAttributeValue, dataValue));
+            logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair "
+                    + "(in brackets) [ %s : %s ] was recognized but we need "
+                    + "more data or time to finish implementation. Discarding... ", 
+                    attributeValue, dataValue));
             return Optional.empty();
         }
 
         return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, dataValue));
     }
-
-    /**
-     * Tests if the attribute value is a recognized type.
-     *
-     * @param normalizedAttributeValue Normalized (trimmed and lowercased) value
-     * to test.
-     * @return True if the attribute value is known, False otherwise.
-     */
-    private boolean isXRYAttributeValueRecognized(String normalizedAttributeValue) {
-        return KEY_TO_TYPE.containsKey(normalizedAttributeValue);
-    }
-
-    /**
-     * Tests if the XRY line has a data key on it.
-     *
-     * @param xryLine
-     * @return
-     */
-    private boolean hasDataKey(String xryLine) {
-        int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER);
-        //No key structure found.
-        if (dataKeyIndex == -1) {
-            return false;
-        }
-
-        String normalizedDataKey = xryLine.substring(0,
-                dataKeyIndex).trim().toLowerCase();
-        return normalizedDataKey.equals(DATA_KEY);
-    }
-
-    /**
-     * Tests if the XRY line has an attribute key on it.
-     *
-     * @param xryLine
-     * @return
-     */
-    private boolean hasXRYAttributeKey(String xryLine) {
-        int attributeKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER);
-        //No key structure found.
-        if (attributeKeyIndex == -1) {
-            return false;
-        }
-
-        String normalizedDataKey = xryLine.substring(0,
-                attributeKeyIndex).trim().toLowerCase();
-        return normalizedDataKey.equals(ATTRIBUTE_KEY);
-    }
 }

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java

@@ -46,6 +46,7 @@ final class XRYFileParserFactory {
             case "calls":
                 return new XRYCallsFileParser();
             case "contacts/contacts":
+            case "contacts":
                 return new XRYContactsFileParser();
             case "device/general information":
                 return new XRYDeviceGenInfoFileParser();

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java

@@ -0,0 +1,132 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2019 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.datasourceprocessors.xry;
+
+/**
+ * A XRY Key Value pair. XRY Key Value pairs make up the body of XRY entities.
+ * 
+ * Example:
+ * 
+ * Attribute:                   Device Name
+ * Time:			3/20/2012 12:06:30 AM
+ * Status:			Read
+ */
+class XRYKeyValuePair {
+  
+    private static final char KEY_VALUE_DELIMITER = ':';
+    
+    private final String key;
+    private final String value;
+    private final String namespace;
+
+    XRYKeyValuePair(String key, String value, String namespace) {
+        this.key = key.trim();
+        this.value = value.trim();
+        this.namespace = namespace.trim();
+    }
+
+    /**
+     * Tests if the key of this pair matches some expected value.
+     * 
+     * The expected value and the key of this pair will both be 
+     * normalized (trimmed and lowercased).
+     *
+     * @param targetKey Key name to test.
+     */
+    boolean hasKey(String targetKey) {
+        String normalizedKey = key.toLowerCase();
+        String normalizedTargetKey = targetKey.trim().toLowerCase();
+        return normalizedKey.equals(normalizedTargetKey);
+    }
+
+    /**
+     * Retrieves the value contained within this pair.
+     */
+    String getValue() {
+        return value;
+    }
+    
+    /**
+     * Retrieves the key contained within this pair.
+     */
+    String getKey() {
+        return key;
+    }
+    
+    /**
+     * Retrieves the namespace contained within this pair.
+     */
+    String getNamespace() {
+        return namespace;
+    }
+    
+    /**
+     * Tests if the input has the structure of a key value pair.
+     *
+     * @param xryLine XRY entity line to test.
+     */
+    static boolean isPair(String input) {
+        int dataKeyIndex = input.indexOf(KEY_VALUE_DELIMITER);
+        //No key structure found.
+        return dataKeyIndex != -1;
+    }
+
+    /**
+     * Extracts a key value pair from the input.
+     *
+     * This function assumes that there is a key value structure on the line. It
+     * can be verified using hasKeyValueForm().
+     *
+     * @param input Input key value string to parse.
+     * @param namespace Namespace to assign to this pair.
+     * @return
+     */
+    static XRYKeyValuePair from(String input, String namespace) {
+        if(!isPair(input)) {
+            throw new IllegalArgumentException("Input does not have the structure"
+                    + " of a key value pair");
+        }
+        
+        int keyIndex = input.indexOf(KEY_VALUE_DELIMITER);
+        String parsedKey = input.substring(0, keyIndex);
+        String parsedValue = input.substring(keyIndex + 1);
+        return new XRYKeyValuePair(parsedKey, parsedValue, namespace);
+    }
+    
+    /**
+     * Extracts a key value pair from the input.
+     *
+     * This function assumes that there is a key value structure on the line. It
+     * can be verified using hasKeyValueForm().
+     *
+     * @param input Input key value string to parse.
+     * @return
+     */
+    static XRYKeyValuePair from(String input) {
+        return from(input, "");
+    }
+    
+    @Override
+    public String toString() {
+        if(namespace.isEmpty()) {
+            return key + " : " + value;
+        }
+        return "(Namespace: " + namespace +") " + key + " : " + value;
+    }
+}

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java

@@ -54,7 +54,7 @@ final class XRYReportProcessor {
                     XRYFileParser parser = XRYFileParserFactory.get(reportType);
                     parser.parse(xryFileReader, parent);
                 } else {
-                    logger.log(Level.SEVERE, String.format("[XRY DSP] XRY File (in brackets) "
+                    logger.log(Level.WARNING, String.format("[XRY DSP] XRY File (in brackets) "
                             + "[ %s ] was found, but no parser to support its report type exists. "
                             + "Report type is [ %s ]", xryFileReader.getReportPath().toString(), reportType));
                 }

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java

@@ -18,6 +18,7 @@
  */
 package org.sleuthkit.autopsy.datasourceprocessors.xry;
 
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.HashMap;
 import java.util.List;
@@ -30,21 +31,22 @@ import org.sleuthkit.datamodel.TskCoreException;
 /**
  * Parses XRY Web-Bookmark files and creates artifacts.
  */
-final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser {
+final class XRYWebBookmarksFileParser extends AbstractSingleEntityParser {
 
     //All known XRY keys for web bookmarks.
-    private static final Map<String, BlackboardAttribute.ATTRIBUTE_TYPE> KEY_TO_TYPE
+    private static final Map<String, BlackboardAttribute.ATTRIBUTE_TYPE> XRY_KEYS
             = new HashMap<String, BlackboardAttribute.ATTRIBUTE_TYPE>() {
         {
             put("web address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL);
             put("domain", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN);
+            put("application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME);
         }
     };
 
     @Override
-    boolean isKey(String key) {
-        String normalizedKey = key.toLowerCase();
-        return KEY_TO_TYPE.containsKey(normalizedKey);
+    boolean canProcess(XRYKeyValuePair pair) {
+        String normalizedKey = pair.getKey().toLowerCase();
+        return XRY_KEYS.containsKey(normalizedKey);
     }
 
     @Override
@@ -53,17 +55,29 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser {
         return false;
     }
 
-    @Override
-    Optional<BlackboardAttribute> makeAttribute(String nameSpace, String key, String value) {
-        String normalizedKey = key.toLowerCase();
+    /**
+     * Creates the appropriate blackboard attribute given a single XRY Key Value
+     * pair.
+     */
+    private Optional<BlackboardAttribute> getBlackboardAttribute(XRYKeyValuePair pair) {
+        String normalizedKey = pair.getKey().toLowerCase();
         return Optional.of(new BlackboardAttribute(
-                KEY_TO_TYPE.get(normalizedKey), 
-                PARSER_NAME, value));
+                XRY_KEYS.get(normalizedKey), 
+                PARSER_NAME, pair.getValue()));
     }
     
     @Override
-    void makeArtifact(List<BlackboardAttribute> attributes, Content parent) throws TskCoreException {
-        BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
-        artifact.addAttributes(attributes);
+    void makeArtifact(List<XRYKeyValuePair> keyValuePairs, Content parent) throws TskCoreException {
+        List<BlackboardAttribute> attributes = new ArrayList<>();
+        for(XRYKeyValuePair pair : keyValuePairs) {
+            Optional<BlackboardAttribute> attribute = getBlackboardAttribute(pair);
+            if(attribute.isPresent()) {
+                attributes.add(attribute.get());
+            }
+        }
+        if(!attributes.isEmpty()) {
+            BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
+            artifact.addAttributes(attributes);
+        }
     }
 }

Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java

@@ -41,7 +41,6 @@ import org.sleuthkit.autopsy.actions.DeleteFileBlackboardArtifactTagAction;
 import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction;
 import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint;
 import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType;
 import org.sleuthkit.autopsy.datamodel.AbstractFsContentNode;
 import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode;
 import org.sleuthkit.autopsy.datamodel.DataModelActionsFactory;
@@ -259,15 +258,20 @@ public class DataResultFilterNode extends FilterNode {
 
         @Override
         protected Node[] createNodes(Node key) {
-            // filter out all non-message artifacts, if displaying the results from the Data Source tree
+            // if displaying the results from the Data Source tree
+            // filter out artifacts
+          
+            // In older versions of Autopsy,  attachments were children of email/message artifacts
+            // and hence email/messages with attachments are shown in the tree data source tree,
             BlackboardArtifact art = key.getLookup().lookup(BlackboardArtifact.class);
-            if (art != null
-                    && filterArtifacts
-                    && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID()
-                    && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) {
+            if (art != null && filterArtifacts
+                    && ((FilterNodeUtils.showMessagesInDatasourceTree() == false)
+                         || (FilterNodeUtils.showMessagesInDatasourceTree()
+                                && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID()
+                                && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()))) {
                 return new Node[]{};
             }
-
+                
             return new Node[]{new DataResultFilterNode(key, sourceEm)};
         }
     }

Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java

@@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.directorytree;
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.logging.Level;
 import javax.swing.Action;
@@ -33,17 +32,12 @@ import org.sleuthkit.autopsy.core.UserPreferences;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.datamodel.AbstractContentNode;
 import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode;
-import org.sleuthkit.autopsy.ingest.runIngestModuleWizard.RunIngestModulesAction;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.BlackboardArtifact;
 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
 import org.sleuthkit.datamodel.Content;
-import org.sleuthkit.datamodel.Directory;
-import org.sleuthkit.datamodel.Image;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
-import org.sleuthkit.datamodel.VirtualDirectory;
-import org.sleuthkit.datamodel.Volume;
 
 /**
  * A node filter (decorator) that sets the actions for a node in the tree view
@@ -137,11 +131,18 @@ class DirectoryTreeFilterNode extends FilterNode {
                         numVisibleChildren--;
                     }
                 } else if (child instanceof BlackboardArtifact) {
-                    BlackboardArtifact bba = (BlackboardArtifact) child;
-
-                    // Only message type artifacts are displayed in the tree
-                    if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID())
-                            && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) {
+                    
+                    if (FilterNodeUtils.showMessagesInDatasourceTree()) {
+                        // In older versions of Autopsy,  attachments were children of email/message artifacts
+                        // and hence email/messages with attachments are shown in the directory tree.
+                        BlackboardArtifact bba = (BlackboardArtifact) child;
+                        // Only message type artifacts are displayed in the tree
+                        if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID())
+                                && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) {
+                            numVisibleChildren--;
+                        }
+                    }
+                    else {
                         numVisibleChildren--;
                     }
                 }

Core/src/org/sleuthkit/autopsy/directorytree/FilterNodeUtils.java

@@ -0,0 +1,66 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2019 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.sleuthkit.autopsy.directorytree;
+
+import org.sleuthkit.autopsy.casemodule.Case;
+import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
+
+/**
+ * Utility class for Directory tree.
+ * 
+ */
+final class FilterNodeUtils {
+    
+    private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER = 8;
+    private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER = 4;
+    
+    /**
+     * Empty private constructor
+     */
+    private FilterNodeUtils() {
+        
+    }
+    
+     /**
+     * Prior to schema version 8.4, attachments were children of messages and
+     * hence messages with any attachment children are shown in the directory
+     * tree.
+     *
+     * At 8.4 and later, attachments are tracked as an attribute, and the message 
+     * artifacts don't need to be shown in the directory tree.
+     *
+     * This method may be used to check the schema version and behave
+     * accordingly, in order to maintain backward compatibility.
+     *
+     * @return True if messages with attachment children should be shown in
+     * directory tree.
+     */
+    static boolean showMessagesInDatasourceTree() {
+        boolean showMessagesInDatasourceTree = true;
+        if (Case.isCaseOpen()) {
+            CaseDbSchemaVersionNumber version = Case.getCurrentCase().getSleuthkitCase().getDBSchemaCreationVersion();
+            showMessagesInDatasourceTree
+                    = ((version.getMajor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER)
+                    || (version.getMajor() == ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER && version.getMinor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER));
+        }
+        return showMessagesInDatasourceTree;
+    }
+    
+}

Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java

@@ -32,7 +32,7 @@ import org.sleuthkit.autopsy.datamodel.FileNode;
 import org.sleuthkit.datamodel.AbstractFile;
 
 /**
- * Create a node containing the children for the to display in the
+ * Create a node containing the children to display in the
  * DataResultViewerThumbnail
  */
 class DiscoveryThumbnailChildren extends Children.Keys<AbstractFile> {

Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java

@@ -977,16 +977,6 @@ class FileSearch {
         GroupKey getGroupKey(ResultFile file) {
             return new FileTypeGroupKey(file);
         }
-
-        @Override
-        void addAttributeToResultFiles(List<ResultFile> files, SleuthkitCase caseDb,
-                EamDb centralRepoDb) throws FileSearchException {
-            for (ResultFile file : files) {
-                if (file.getFileType().equals(FileType.OTHER)) {
-                    file.setFileType(FileType.fromMIMEtype(file.getFirstInstance().getMIMEType()));
-                }
-            }
-        }
     }
 
     /**

Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java

@@ -72,7 +72,7 @@ class ResultFile {
         tagNames = new ArrayList<>();
         interestingSetNames = new ArrayList<>();
         objectDetectedNames = new ArrayList<>();
-        fileType = FileType.OTHER;
+        fileType = FileType.fromMIMEtype(abstractFile.getMIMEType());
     }
 
     /**
@@ -103,6 +103,9 @@ class ResultFile {
         if (deleted && !duplicate.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC)) {
             deleted = false;
         }
+        if (fileType == FileType.OTHER) {
+            fileType = FileType.fromMIMEtype(duplicate.getMIMEType());
+        }
         updateScoreAndDescription(duplicate);
         instances.add(duplicate);
     }
@@ -156,15 +159,6 @@ class ResultFile {
         return fileType;
     }
 
-    /**
-     * Set the file type
-     *
-     * @param fileType the type
-     */
-    void setFileType(FileType fileType) {
-        this.fileType = fileType;
-    }
-
     /**
      * Add a keyword list name that matched this file.
      *

Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java

@@ -88,17 +88,21 @@ public class ResultsPanel extends javax.swing.JPanel {
         imageThumbnailViewer = new ImageThumbnailViewer();
         videoThumbnailViewer = new VideoThumbnailViewer();
         videoThumbnailViewer.addListSelectionListener((e) -> {
-            if (!e.getValueIsAdjusting()) {
-                populateInstancesList();
-            } else {
-                instancesList.clearSelection();
+            if (resultType == FileSearchData.FileType.VIDEO) {
+                if (!e.getValueIsAdjusting()) {
+                    populateInstancesList();
+                } else {
+                    instancesList.clearSelection();
+                }
             }
         });
         imageThumbnailViewer.addListSelectionListener((e) -> {
-            if (!e.getValueIsAdjusting()) {
-                populateInstancesList();
-            } else {
-                instancesList.clearSelection();
+            if (resultType == FileSearchData.FileType.IMAGE) {
+                if (!e.getValueIsAdjusting()) {
+                    populateInstancesList();
+                } else {
+                    instancesList.clearSelection();
+                }
             }
         });
         //Add the context menu when right clicking

Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties

@@ -24,7 +24,7 @@ GeolocationSettingsPanel.mbtileFileField.toolTipText=
 GeolocationSettingsPanel.mbtileFileField.text=
 GeolocationSettingsPanel.defaultDataSource.text=Default online tile server (bing.com/maps)
 GeolocationSettingsPanel.osmServerRBnt.text=OpenStreetMap server
-GeolocationSettingsPanel.zipFileRBnt.text=OpenStreeMap zip file
+GeolocationSettingsPanel.zipFileRBnt.text=OpenStreetMap zip file
 GeolocationSettingsPanel.zipFileRBnt.actionCommand=OpenStreeMap tile ZIP file
 GeolocationSettingsPanel.mbtilesRBtn.text=MBTiles file
 GeolocationSettingsPanel.osmServerAddressField.text=

Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED

@@ -59,7 +59,7 @@ GeolocationSettingsPanel.mbtileFileField.toolTipText=
 GeolocationSettingsPanel.mbtileFileField.text=
 GeolocationSettingsPanel.defaultDataSource.text=Default online tile server (bing.com/maps)
 GeolocationSettingsPanel.osmServerRBnt.text=OpenStreetMap server
-GeolocationSettingsPanel.zipFileRBnt.text=OpenStreeMap zip file
+GeolocationSettingsPanel.zipFileRBnt.text=OpenStreetMap zip file
 GeolocationSettingsPanel.zipFileRBnt.actionCommand=OpenStreeMap tile ZIP file
 GeolocationSettingsPanel.mbtilesRBtn.text=MBTiles file
 GeolocationSettingsPanel.osmServerAddressField.text=

Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form

@@ -145,7 +145,7 @@
           </Events>
           <Constraints>
             <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
-              <GridBagConstraints gridX="2" gridY="2" gridWidth="1" gridHeight="1" fill="0" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="9" insetsBottom="9" insetsRight="9" anchor="18" weightX="0.0" weightY="0.0"/>
+              <GridBagConstraints gridX="2" gridY="2" gridWidth="1" gridHeight="1" fill="0" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="9" insetsBottom="9" insetsRight="9" anchor="17" weightX="0.0" weightY="0.0"/>
             </Constraint>
           </Constraints>
         </Component>
@@ -160,7 +160,7 @@
           </Events>
           <Constraints>
             <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
-              <GridBagConstraints gridX="2" gridY="1" gridWidth="1" gridHeight="1" fill="0" ipadX="20" ipadY="0" insetsTop="0" insetsLeft="9" insetsBottom="9" insetsRight="9" anchor="18" weightX="0.0" weightY="0.0"/>
+              <GridBagConstraints gridX="2" gridY="1" gridWidth="1" gridHeight="1" fill="2" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="9" insetsBottom="9" insetsRight="9" anchor="17" weightX="0.0" weightY="0.0"/>
             </Constraint>
           </Constraints>
         </Component>
@@ -200,36 +200,45 @@
             </Constraint>
           </Constraints>
         </Component>
-        <Component class="javax.swing.JButton" name="mbtilesBrowseBtn">
-          <Properties>
-            <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
-              <ResourceString bundle="org/sleuthkit/autopsy/geolocation/Bundle.properties" key="GeolocationSettingsPanel.mbtilesBrowseBtn.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
-            </Property>
-          </Properties>
-          <Events>
-            <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="mbtilesBrowseBtnActionPerformed"/>
-          </Events>
+        <Container class="javax.swing.JPanel" name="MBTilesBtnPanel">
+          <AuxValues>
+            <AuxValue name="JavaCodeGenerator_VariableLocal" type="java.lang.Boolean" value="true"/>
+            <AuxValue name="JavaCodeGenerator_VariableModifier" type="java.lang.Integer" value="0"/>
+          </AuxValues>
           <Constraints>
             <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
-              <GridBagConstraints gridX="2" gridY="3" gridWidth="1" gridHeight="1" fill="0" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="9" insetsBottom="9" insetsRight="9" anchor="18" weightX="0.0" weightY="0.0"/>
+              <GridBagConstraints gridX="2" gridY="3" gridWidth="2" gridHeight="1" fill="0" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="9" insetsBottom="9" insetsRight="9" anchor="17" weightX="0.0" weightY="0.0"/>
             </Constraint>
           </Constraints>
-        </Component>
-        <Component class="javax.swing.JButton" name="mbtileTestBtn">
-          <Properties>
-            <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
-              <ResourceString bundle="org/sleuthkit/autopsy/geolocation/Bundle.properties" key="GeolocationSettingsPanel.mbtileTestBtn.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
-            </Property>
-          </Properties>
-          <Events>
-            <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="mbtileTestBtnActionPerformed"/>
-          </Events>
-          <Constraints>
-            <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
-              <GridBagConstraints gridX="3" gridY="3" gridWidth="1" gridHeight="1" fill="0" ipadX="20" ipadY="0" insetsTop="0" insetsLeft="0" insetsBottom="0" insetsRight="0" anchor="18" weightX="1.0" weightY="0.0"/>
-            </Constraint>
-          </Constraints>
-        </Component>
+
+          <Layout class="org.netbeans.modules.form.compat2.layouts.DesignGridLayout">
+            <Property name="columns" type="int" value="0"/>
+            <Property name="horizontalGap" type="int" value="5"/>
+            <Property name="rows" type="int" value="1"/>
+          </Layout>
+          <SubComponents>
+            <Component class="javax.swing.JButton" name="mbtilesBrowseBtn">
+              <Properties>
+                <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
+                  <ResourceString bundle="org/sleuthkit/autopsy/geolocation/Bundle.properties" key="GeolocationSettingsPanel.mbtilesBrowseBtn.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
+                </Property>
+              </Properties>
+              <Events>
+                <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="mbtilesBrowseBtnActionPerformed"/>
+              </Events>
+            </Component>
+            <Component class="javax.swing.JButton" name="mbtileTestBtn">
+              <Properties>
+                <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
+                  <ResourceString bundle="org/sleuthkit/autopsy/geolocation/Bundle.properties" key="GeolocationSettingsPanel.mbtileTestBtn.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
+                </Property>
+              </Properties>
+              <Events>
+                <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="mbtileTestBtnActionPerformed"/>
+              </Events>
+            </Component>
+          </SubComponents>
+        </Container>
       </SubComponents>
     </Container>
   </SubComponents>

Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java

@@ -159,6 +159,7 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio
         serverTestBtn = new javax.swing.JButton();
         mbtilesRBtn = new javax.swing.JRadioButton();
         mbtileFileField = new javax.swing.JTextField();
+        javax.swing.JPanel MBTilesBtnPanel = new javax.swing.JPanel();
         mbtilesBrowseBtn = new javax.swing.JButton();
         mbtileTestBtn = new javax.swing.JButton();
 
@@ -236,7 +237,7 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio
         gridBagConstraints = new java.awt.GridBagConstraints();
         gridBagConstraints.gridx = 2;
         gridBagConstraints.gridy = 2;
-        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST;
         gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9);
         tilePane.add(zipFileBrowseBnt, gridBagConstraints);
 
@@ -249,8 +250,8 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio
         gridBagConstraints = new java.awt.GridBagConstraints();
         gridBagConstraints.gridx = 2;
         gridBagConstraints.gridy = 1;
-        gridBagConstraints.ipadx = 20;
-        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST;
         gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9);
         tilePane.add(serverTestBtn, gridBagConstraints);
 
@@ -277,18 +278,15 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio
         gridBagConstraints.insets = new java.awt.Insets(0, 0, 9, 0);
         tilePane.add(mbtileFileField, gridBagConstraints);
 
+        MBTilesBtnPanel.setLayout(new java.awt.GridLayout(1, 0, 5, 0));
+
         org.openide.awt.Mnemonics.setLocalizedText(mbtilesBrowseBtn, org.openide.util.NbBundle.getMessage(GeolocationSettingsPanel.class, "GeolocationSettingsPanel.mbtilesBrowseBtn.text")); // NOI18N
         mbtilesBrowseBtn.addActionListener(new java.awt.event.ActionListener() {
             public void actionPerformed(java.awt.event.ActionEvent evt) {
                 mbtilesBrowseBtnActionPerformed(evt);
             }
         });
-        gridBagConstraints = new java.awt.GridBagConstraints();
-        gridBagConstraints.gridx = 2;
-        gridBagConstraints.gridy = 3;
-        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
-        gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9);
-        tilePane.add(mbtilesBrowseBtn, gridBagConstraints);
+        MBTilesBtnPanel.add(mbtilesBrowseBtn);
 
         org.openide.awt.Mnemonics.setLocalizedText(mbtileTestBtn, org.openide.util.NbBundle.getMessage(GeolocationSettingsPanel.class, "GeolocationSettingsPanel.mbtileTestBtn.text")); // NOI18N
         mbtileTestBtn.addActionListener(new java.awt.event.ActionListener() {
@@ -296,13 +294,15 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio
                 mbtileTestBtnActionPerformed(evt);
             }
         });
+        MBTilesBtnPanel.add(mbtileTestBtn);
+
         gridBagConstraints = new java.awt.GridBagConstraints();
-        gridBagConstraints.gridx = 3;
+        gridBagConstraints.gridx = 2;
         gridBagConstraints.gridy = 3;
-        gridBagConstraints.ipadx = 20;
-        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
-        gridBagConstraints.weightx = 1.0;
-        tilePane.add(mbtileTestBtn, gridBagConstraints);
+        gridBagConstraints.gridwidth = 2;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST;
+        gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9);
+        tilePane.add(MBTilesBtnPanel, gridBagConstraints);
 
         gridBagConstraints = new java.awt.GridBagConstraints();
         gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;

Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java

@@ -431,6 +431,8 @@ public final class GeolocationTopComponent extends TopComponent {
                                 Bundle.GeoTopComponent_filter_exception_Title(),
                                 Bundle.GeoTopComponent_filter_exception_msg(),
                                 JOptionPane.ERROR_MESSAGE);
+                        
+                        setWaypointLoading(false);
                     }
                 });
             }

Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java

@@ -417,8 +417,6 @@ public class KdTree<T extends KdTree.XYZPoint> implements Iterable<T> {
         }
         Double nodeDistance = node.id.euclideanDistance(value);
         if (nodeDistance.compareTo(lastDistance) < 0) {
-            if (results.size() == K && lastNode != null)
-                results.remove(lastNode);
             results.add(node);
         } else if (nodeDistance.equals(lastDistance)) {
             results.add(node);

Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java

@@ -70,6 +70,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
 import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationDataException;
 import org.sleuthkit.datamodel.TskCoreException;
 import javax.imageio.ImageIO;
+import javax.swing.SwingUtilities;
 import org.jxmapviewer.viewer.DefaultWaypointRenderer;
 
 /**
@@ -204,13 +205,12 @@ final public class MapPanel extends javax.swing.JPanel {
                     Iterator<MapWaypoint> iterator = waypointTree.iterator();
                     while (iterator.hasNext()) {
                         MapWaypoint point = iterator.next();
-                        if (point != currentlySelectedWaypoint) {
-                            set.add(point);
-                        }
+                        set.add(point);
                     }
                     // Add the currentlySelectedWaypoint to the end so that
                     // it will be painted last.
                     if (currentlySelectedWaypoint != null) {
+                        set.remove(currentlySelectedWaypoint);
                         set.add(currentlySelectedWaypoint);
                     }
                 }
@@ -342,7 +342,11 @@ final public class MapPanel extends javax.swing.JPanel {
      */
     private void showPopupMenu(Point point) {
         try {
-            MapWaypoint waypoint = findClosestWaypoint(point);
+            List<MapWaypoint> waypoints = findClosestWaypoint(point);
+            MapWaypoint waypoint = null;
+            if(waypoints.size() > 0) {
+                waypoint = waypoints.get(0);
+            }
             showPopupMenu(waypoint, point);
             // Change the details popup to the currently selected point only if 
             // it the popup is currently visible
@@ -410,6 +414,7 @@ final public class MapPanel extends javax.swing.JPanel {
             currentPopup = popupFactory.getPopup(this, detailPane, popupLocation.x, popupLocation.y);
             currentPopup.show();
             
+            mapViewer.revalidate();
             mapViewer.repaint();
         }
     }
@@ -437,7 +442,7 @@ final public class MapPanel extends javax.swing.JPanel {
      * @return A waypoint that is within 10 pixels of the given point, or null
      *         if none was found.
      */
-    private MapWaypoint findClosestWaypoint(Point mouseClickPoint) {
+    private List<MapWaypoint> findClosestWaypoint(Point mouseClickPoint) {
         if (waypointTree == null) {
             return null;
         }
@@ -446,7 +451,7 @@ final public class MapPanel extends javax.swing.JPanel {
         GeoPosition geopos = mapViewer.getTileFactory().pixelToGeo(mouseClickPoint, mapViewer.getZoom());
 
         // Get the 5 nearest neightbors to the point
-        Collection<MapWaypoint> waypoints = waypointTree.nearestNeighbourSearch(20, MapWaypoint.getDummyWaypoint(geopos));
+        Collection<MapWaypoint> waypoints = waypointTree.nearestNeighbourSearch(10, MapWaypoint.getDummyWaypoint(geopos));
 
         if (waypoints == null || waypoints.isEmpty()) {
             return null;
@@ -456,6 +461,7 @@ final public class MapPanel extends javax.swing.JPanel {
 
         // These maybe the points closest to lat/log was clicked but 
         // that doesn't mean they are close in terms of pixles.
+        List<MapWaypoint> closestPoints = new ArrayList<>();
         while (iterator.hasNext()) {
             MapWaypoint nextWaypoint = iterator.next();
 
@@ -466,11 +472,11 @@ final public class MapPanel extends javax.swing.JPanel {
                     (int) point.getY() - rect.y);
 
             if (converted_gp_pt.distance(mouseClickPoint) < 10) {
-                return nextWaypoint;
+                closestPoints.add(nextWaypoint);
             }
         }
 
-        return null;
+        return closestPoints;
     }
 
     /**
@@ -629,8 +635,14 @@ final public class MapPanel extends javax.swing.JPanel {
     }//GEN-LAST:event_mapViewerMouseMoved
 
     private void mapViewerMouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_mapViewerMouseClicked
-        if(!evt.isPopupTrigger() && (evt.getButton() == MouseEvent.BUTTON1)) {
-            currentlySelectedWaypoint = findClosestWaypoint(evt.getPoint());
+        if(!evt.isPopupTrigger() && SwingUtilities.isLeftMouseButton(evt)) {
+            List<MapWaypoint> waypoints = findClosestWaypoint(evt.getPoint());
+            if(waypoints.size() > 0) {
+                currentlySelectedWaypoint = waypoints.get(0);
+            } 
+            
+            
+//            currentlySelectedWaypoint = findClosestWaypoint(evt.getPoint());
             showDetailsPopup();
         }
     }//GEN-LAST:event_mapViewerMouseClicked

Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java

@@ -0,0 +1,79 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2019 Basis Technology Corp.
+ * contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.geolocation.datamodel;
+
+import java.util.Map;
+import org.sleuthkit.datamodel.BlackboardArtifact;
+import org.sleuthkit.datamodel.BlackboardAttribute;
+
+/**
+ * Class wraps any artifact that is not one of the known types, but have the 
+ * TSK_GEO_LONGITUDE and TSK_GEO_LATITUDE attributes.
+ *
+ */
+final class CustomArtifactWaypoint extends Waypoint {
+
+    /**
+     * Constructs a new waypoint from the given artifact.
+     *
+     * @param artifact BlackboardArtifact for this waypoint
+     *
+     * @throws GeoLocationDataException
+     */
+    CustomArtifactWaypoint(BlackboardArtifact artifact) throws GeoLocationDataException {
+        this(artifact, getAttributesFromArtifactAsMap(artifact));
+    }
+
+    /**
+     * Constructs a new CustomArtifactWaypoint.
+     *
+     * @param artifact     BlackboardArtifact for this waypoint
+     * @param attributeMap A Map of the BlackboardAttributes for the given
+     *                     artifact.
+     *
+     * @throws GeoLocationDataException
+     */
+    private CustomArtifactWaypoint(BlackboardArtifact artifact, Map<BlackboardAttribute.ATTRIBUTE_TYPE, BlackboardAttribute> attributeMap) throws GeoLocationDataException {
+        super(artifact,
+                getLabelFromArtifact(attributeMap),
+                attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME).getValueLong() : null,
+                attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE).getValueDouble() : null,
+                attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE).getValueDouble() : null,
+                attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE).getValueDouble() : null,
+                null, attributeMap, null);
+    }
+
+    /**
+     * Gets the label for this waypoint.
+     *
+     * @param artifact BlackboardArtifact for waypoint
+     *
+     * @return Returns a label for the waypoint, or empty string if no label was
+     *         found.
+     */
+    private static String getLabelFromArtifact(Map<BlackboardAttribute.ATTRIBUTE_TYPE, BlackboardAttribute> attributeMap) {
+        BlackboardAttribute attribute = attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME);
+        if (attribute != null) {
+            return attribute.getDisplayString();
+        }
+
+        return "";
+    }
+
+}

Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java

@@ -71,7 +71,7 @@ final class LastKnownWaypoint extends Waypoint {
     "LastKnownWaypoint_Label=Last Known Location",})
     private static String getLabelFromArtifact(Map<BlackboardAttribute.ATTRIBUTE_TYPE, BlackboardAttribute> attributeMap) throws GeoLocationDataException {
         BlackboardAttribute attribute = attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME);
-        String label = attribute.getDisplayString();
+        String label = attribute != null ? attribute.getDisplayString() : Bundle.LastKnownWaypoint_Label();
 
         if (label == null || label.isEmpty()) {
             label = Bundle.LastKnownWaypoint_Label();

Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java

@@ -592,8 +592,11 @@ public final class WaypointBuilder {
                 Route route = new Route(artifact);
                 waypoints.addAll(route.getRoute());
                 break;
+            case TSK_GPS_LAST_KNOWN_LOCATION:
+                waypoints.add(new LastKnownWaypoint(artifact));
+                break;
             default:
-                throw new GeoLocationDataException(String.format("Unable to create waypoint for artifact of type %s", type.toString()));
+                 waypoints.add(new CustomArtifactWaypoint(artifact));
         }
 
         return waypoints;

Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java

@@ -209,7 +209,6 @@ public final class EventsModel {
      * data source data in the case database.
      */
     synchronized private void populateDataSourcesCache() throws TskCoreException {
-        datasourceIDsToNamesMap.clear();
         SleuthkitCase skCase = currentCase.getSleuthkitCase();
         for (DataSource ds : skCase.getDataSources()) {
             datasourceIDsToNamesMap.putIfAbsent(ds.getId(), ds.getName());

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java

@@ -578,8 +578,16 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
 
             TskData.TSK_DB_FILES_TYPE_ENUM aType = aFile.getType();
 
-            // unallocated and unused blocks can only have strings extracted from them. 
-            if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) || aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))) {
+            /**
+             * Extract unicode strings from unallocated and unused blocks and
+             * carved text files. The reason for performing string extraction
+             * on these is because they all may contain multiple encodings which
+             * can cause text to be missed by the more specialized text extractors
+             * used below.
+             */
+            if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)
+                    || aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))
+                    || (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.CARVED) && aFile.getNameExtension().equalsIgnoreCase("txt"))) {
                 if (context.fileIngestIsCancelled()) {
                     return;
                 }

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED

@@ -15,6 +15,7 @@ ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index e
 # {0} - file name
 # {1} - file ID
 ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse.
+ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message.
 ThunderbirdMboxFileIngestModule.moduleName=Email Parser
 ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case.
 ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace=Out of disk space. Cannot copy {0} to parse.

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -58,6 +59,9 @@ import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 import org.sleuthkit.datamodel.TskDataException;
 import org.sleuthkit.datamodel.TskException;
+import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper;
+import org.sleuthkit.datamodel.blackboardutils.FileAttachment;
+import org.sleuthkit.datamodel.blackboardutils.MessageAttachments;
 
 /**
  * File-level ingest module that detects MBOX, PST, and vCard files based on
@@ -70,6 +74,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
     private FileManager fileManager;
     private IngestJobContext context;
     private Blackboard blackboard;
+    private CommunicationArtifactsHelper communicationArtifactsHelper;
     
     private Case currentCase;
 
@@ -129,6 +134,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
             logger.log(Level.WARNING, null, ex);
         }
 
+        try {
+            communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(),
+                    EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL);
+        } catch (TskCoreException ex) {
+            logger.log(Level.SEVERE, String.format("Failed to create CommunicationArtifactsHelper for file with object id = %d", abstractFile.getId()), ex);
+            return ProcessResult.ERROR;
+        }
+
+       
         if (isMbox) {
             return processMBox(abstractFile);
         }
@@ -267,7 +281,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
         } else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS
             emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS
         }
-        emailFolder = emailFolder + mboxFileName;
+        emailFolder += mboxFileName;
         emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS
 
         String fileName;
@@ -487,8 +501,12 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
      *
      * @return List of attachments
      */
+    @NbBundle.Messages({
+    "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message."
+})
     private List<AbstractFile> handleAttachments(List<EmailMessage.Attachment> attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) {
         List<AbstractFile> files = new ArrayList<>();
+        List<FileAttachment> fileAttachments = new ArrayList<>();
         for (EmailMessage.Attachment attach : attachments) {
             String filename = attach.getName();
             long crTime = attach.getCrTime();
@@ -501,12 +519,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
 
             try {
                 DerivedFile df = fileManager.addDerivedFile(filename, relPath,
-                        size, cTime, crTime, aTime, mTime, true, messageArtifact, "",
+                        size, cTime, crTime, aTime, mTime, true, abstractFile, "",
                         EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType);
                 
                 associateAttachmentWithMesssge(messageArtifact, df);
                 
                 files.add(df);
+                
+                fileAttachments.add(new FileAttachment(df));
             } catch (TskCoreException ex) {
                 postErrorMessage(
                         NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.errMsg",
@@ -516,6 +536,17 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
                 logger.log(Level.INFO, "", ex);
             }
         }
+        
+       
+        try {
+            communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList()));
+        } catch (TskCoreException ex) {
+             postErrorMessage(
+                        NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"),
+                        "");
+                logger.log(Level.INFO, "Failed to add attachments to email message.", ex);
+        }
+       
         return files;
     }