diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 83aefea7c5..17e5d82284 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -345,6 +345,7 @@ org.sleuthkit.autopsy.textextractors.configs org.sleuthkit.autopsy.texttranslation org.sleuthkit.datamodel + org.sleuthkit.datamodel.blackboardutils ext/commons-lang3-3.8.1.jar diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java index d97e980a8b..9de0e95462 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java @@ -227,6 +227,7 @@ public class ExtractedContent implements AutopsyVisitableItem { // maps the artifact type to its child node private final HashMap typeNodeList = new HashMap<>(); + @SuppressWarnings("deprecation") TypeFactory() { super(); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java similarity index 51% rename from Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java rename to Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java index 93b5051657..cdd84fa06e 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java @@ -22,25 +22,19 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; /** - * Template parse method for reports that make blackboard attributes from a - * single key value pair. - * - * This parse implementation will create 1 artifact per XRY entity. + * Template parse method for reports that make artifacts from a single XRY + * Entity. */ -abstract class AbstractSingleKeyValueParser implements XRYFileParser { +abstract class AbstractSingleEntityParser implements XRYFileParser { - private static final Logger logger = Logger.getLogger(AbstractSingleKeyValueParser.class.getName()); + private static final Logger logger = Logger.getLogger(AbstractSingleEntityParser.class.getName()); - private static final char KEY_VALUE_DELIMITER = ':'; - protected static final String PARSER_NAME = "XRY DSP"; @Override @@ -52,14 +46,13 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { String xryEntity = reader.nextEntity(); String[] xryLines = xryEntity.split("\n"); - List attributes = new ArrayList<>(); + List keyValuePairs = new ArrayList<>(); //First line of the entity is the title, the entity will always be non-empty. logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); String namespace = ""; //Process each line, searching for a key value pair or a namespace. - //If neither are found, an error message is logged. for (int i = 1; i < xryLines.length; i++) { String xryLine = xryLines[i]; @@ -71,60 +64,46 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { continue; } - //Find the XRY key on this line. Assume key is the value between - //the start of the line and the first delimiter. - int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (keyDelimiter == -1) { + //Check if this line resembles a Key Value pair. + if(!XRYKeyValuePair.isPair(xryLine)) { logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value " - + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Here is the previous line [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); + + "pair on this line (in brackets) [ %s ], but one was not detected.", + xryLine)); continue; } - String key = xryLine.substring(0, keyDelimiter).trim(); - String value = xryLine.substring(keyDelimiter + 1).trim(); + + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine, namespace); - if (!isKey(key)) { + //Verify the implementation recognizes the key. + if (!canProcess(pair)) { logger.log(Level.WARNING, String.format("[XRY DSP] The following key, " - + "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..." - + " Here is the previous line [ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); + + "value pair (in brackets) [ %s ] was not recognized. Discarding...", + pair)); continue; } - if (value.isEmpty()) { - logger.log(Level.WARNING, String.format("[XRY DSP] The following key " - + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding..." - + " Here is the previous line for context [ %s ]. What does this mean?", key, xryLines[i - 1])); + //Empty values are meaningless for blackboard attributes. + if (pair.getValue().isEmpty()) { + logger.log(Level.WARNING, String.format("[XRY DSP] The following key value pair" + + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding...", + pair)); continue; } - - //Create the attribute, if any. - Optional attribute = makeAttribute(namespace, key, value); - if(attribute.isPresent()) { - attributes.add(attribute.get()); - } + + keyValuePairs.add(pair); } - - //Only create artifacts with non-empty attributes. - if (!attributes.isEmpty()) { - makeArtifact(attributes, parent); + + if(!keyValuePairs.isEmpty()) { + makeArtifact(keyValuePairs, parent); } } } /** - * Determines if the key candidate is a known key. A key candidate is a - * string literal that begins a line and is terminated by a semi-colon. - * - * Ex: - * - * Call Type : Missed - * - * "Call Type" would be the key candidate that was extracted. - * - * @param key Key to test. These keys are trimmed of whitespace only. - * @return Indication if this key can be processed. + * Determines if the XRY key value pair can be processed by the + * implementation. */ - abstract boolean isKey(String key); + abstract boolean canProcess(XRYKeyValuePair pair); /** * Determines if the namespace candidate is a known namespace. A namespace @@ -144,19 +123,8 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { abstract boolean isNamespace(String nameSpace); /** - * Creates an attribute from the extracted key value pair. - * - * @param nameSpace The namespace of this key value pair. - * It will have been verified with isNamespace, otherwise it will be empty. - * @param key The key that was verified with isKey. - * @param value The value associated with that key. - * @return The corresponding blackboard attribute, if any. + * Makes an artifact from the parsed key value pairs. */ - abstract Optional makeAttribute(String nameSpace, String key, String value); - - /** - * Makes an artifact from the parsed attributes. - */ - abstract void makeArtifact(List attributes, Content parent) throws TskCoreException; + abstract void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException; } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index e0ae43721a..ef31f5f417 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -18,9 +18,16 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -33,71 +40,79 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Calls files and creates artifacts. */ -final class XRYCallsFileParser extends AbstractSingleKeyValueParser { +final class XRYCallsFileParser extends AbstractSingleEntityParser { private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName()); //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() //function for more details. private static final DateTimeFormatter DATE_TIME_PARSER - = DateTimeFormatter.ofPattern("O a h:m:s M/d/y"); + = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); + + private static final String DEVICE_LOCALE = "(device)"; + private static final String NETWORK_LOCALE = "(network)"; + /** - * All of the known XRY keys for call reports. + * All of the known XRY keys for call reports and their corresponding + * blackboard attribute types, if any. */ private enum XryKey { - TEL("tel"), - NAME_MATCHED("name (matched)"), - TIME("time"), - DIRECTION("direction"), - CALL_TYPE("call type"), - DURATION("duration"), - STORAGE("storage"), - INDEX("index"), - NAME("name"), - NUMBER("number"); - + NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME), + TIME("time", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME), + DIRECTION("direction", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION), + CALL_TYPE("call type", null), + NUMBER("number", null), + TEL("tel", null), + TO("to", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO), + FROM("from", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM), + DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED), + DURATION("duration", null), + STORAGE("storage", null), + INDEX("index", null), + TYPE("type", null), + NAME("name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); + private final String name; - XryKey(String name) { + private final BlackboardAttribute.ATTRIBUTE_TYPE type; + + XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) { this.name = name; + this.type = type; } - + + public BlackboardAttribute.ATTRIBUTE_TYPE getType() { + return type; + } + /** * Indicates if the display name of the XRY key is a recognized type. - * - * @param xryKey - * @return */ - public static boolean contains(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); - for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { - return true; - } + public static boolean contains(String key) { + try { + XryKey.fromDisplayName(key); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } - + /** * Matches the display name of the xry key to the appropriate enum type. - * - * It is assumed that XRY key string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * - * @param xryKey - * @return + * + * It is assumed that XRY key string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. */ - public static XryKey fromDisplayName(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); - for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { + public static XryKey fromDisplayName(String key) { + String normalizedKey = key.trim().toLowerCase(); + for (XryKey keyChoice : XryKey.values()) { + if (normalizedKey.equals(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Key [%s] was not found." - + " All keys should be tested with contains.", xryKey)); + + " All keys should be tested with contains.", key)); } } @@ -108,55 +123,50 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { TO("to"), FROM("from"), NONE(null); - + private final String name; + XryNamespace(String name) { this.name = name; } - + /** - * Indicates if the display name of the XRY namespace is a recognized type. - * - * @param xryNamespace - * @return + * Indicates if the display name of the XRY namespace is a recognized + * type. */ public static boolean contains(String xryNamespace) { - String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { - return true; - } + try { + XryNamespace.fromDisplayName(xryNamespace); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } - + /** - * Matches the display name of the xry namespace to the appropriate enum type. - * - * It is assumed that XRY namespace string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * - * @param xryNamespace - * @return + * Matches the display name of the xry namespace to the appropriate enum + * type. + * + * It is assumed that XRY namespace string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. */ public static XryNamespace fromDisplayName(String xryNamespace) { String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { + for (XryNamespace keyChoice : XryNamespace.values()) { + if (normalizedNamespace.equals(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Key [%s] was not found." + " All keys should be tested with contains.", xryNamespace)); } } @Override - boolean isKey(String key) { - return XryKey.contains(key); + boolean canProcess(XRYKeyValuePair pair) { + return XryKey.contains(pair.getKey()); } @Override @@ -165,77 +175,75 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } @Override - Optional makeAttribute(String nameSpace, String key, String value) { - XryKey xryKey = XryKey.fromDisplayName(key); + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { + List attributes = new ArrayList<>(); + for(XRYKeyValuePair pair : keyValuePairs) { + Optional attribute = getBlackboardAttribute(pair); + if(attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); + artifact.addAttributes(attributes); + } + } + + /** + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair, if any. Most XRY keys are mapped to an attribute type in the enum above. + */ + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { + XryKey xryKey = XryKey.fromDisplayName(pair.getKey()); XryNamespace xryNamespace = XryNamespace.NONE; - if(XryNamespace.contains(nameSpace)) { - xryNamespace = XryNamespace.fromDisplayName(nameSpace); + if (XryNamespace.contains(pair.getNamespace())) { + xryNamespace = XryNamespace.fromDisplayName(pair.getNamespace()); } switch (xryKey) { - case DIRECTION: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, - PARSER_NAME, value)); - case NAME_MATCHED: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, - PARSER_NAME, value)); - case NUMBER: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, - PARSER_NAME, value)); case TEL: + case NUMBER: //Apply the namespace switch (xryNamespace) { case FROM: return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); case TO: return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); default: return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); } case TIME: try { //Tranform value to seconds since epoch - long dateTimeSinceEpoch = calculateSecondsSinceEpoch(value); - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, + long dateTimeSinceEpoch = calculateSecondsSinceEpoch(pair.getValue()); + return Optional.of(new BlackboardAttribute(xryKey.getType(), PARSER_NAME, dateTimeSinceEpoch)); } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[XRY DSP] Assumption" + " about the date time formatting of call logs is " - + "not right. Here is the value [ %s ]", value), ex); + + "not right. Here is the value [ %s ]", pair.getValue()), ex); return Optional.empty(); } - case DURATION: - case STORAGE: - case INDEX: - case CALL_TYPE: - //Ignore for now, don't need more data. - return Optional.empty(); - case NAME: - logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was " - + "recognized but more examples of its values are needed " - + "to make a decision on an appropriate TSK attribute. " - + "Here is the value [%s].", key, value)); - return Optional.empty(); default: - throw new IllegalArgumentException(String.format("Key [ %s ] " - + "passed the isKey() test but was not matched.", key)); - } - } + //Otherwise, the XryKey enum contains the correct BlackboardAttribute + //type. + if (xryKey.getType() != null) { + return Optional.of(new BlackboardAttribute(xryKey.getType(), + PARSER_NAME, pair.getValue())); + } - @Override - void makeArtifact(List attributes, Content parent) throws TskCoreException { - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); - artifact.addAttributes(attributes); + logger.log(Level.INFO, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s ] was recognized but " + + "more data or time is needed to finish implementation. Discarding... ", + pair)); + return Optional.empty(); + } } /** @@ -247,12 +255,16 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @return A purer date time value. */ private String removeDateTimeLocale(String dateTime) { - int index = dateTime.indexOf('('); - if (index == -1) { - return dateTime; + String result = dateTime; + int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE); + if (deviceIndex != -1) { + result = result.substring(0, deviceIndex); } - - return dateTime.substring(0, index); + int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); + if (networkIndex != -1) { + result = result.substring(0, networkIndex); + } + return result; } /** @@ -265,42 +277,48 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim(); /** * The format of time in XRY Messages reports is of the form: - * - * 1/3/1990 1:23:54 AM UTC+4 - * - * In our current version of Java (openjdk-1.8.0.222), there is - * a bug with having the timezone offset (UTC+4 or GMT-7) at the - * end of the date time input. This is fixed in later versions - * of the JDK (9 and beyond). - * https://bugs.openjdk.java.net/browse/JDK-8154050 - * Rather than update the JDK to accommodate this, the components of - * the date time string are reversed: - * - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * 1/3/1990 1:23:54 AM UTC+4 + * + * In our current version of Java (openjdk-1.8.0.222), there is a bug + * with having the timezone offset (UTC+4 or GMT-7) at the end of the + * date time input. This is fixed in later versions of the JDK (9 and + * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than + * update the JDK to accommodate this, the components of the date time + * string are reversed: + * + * UTC+4 AM 1:23:54 1/3/1990 + * * The java time package will correctly parse this date time format. */ String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale); /** - * Furthermore, the DateTimeFormatter's timezone offset letter ('O') does - * not recognize UTC but recognizes GMT. According to - * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, - * GMT only differs from UTC by at most 1 second and so substitution - * will only introduce a trivial amount of error. + * Furthermore, the DateTimeFormatter's timezone offset letter ('O') + * does not recognize UTC but recognizes GMT. According to + * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only + * differs from UTC by at most 1 second and so substitution will only + * introduce a trivial amount of error. */ String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); - ZonedDateTime zonedDateTime = ZonedDateTime.parse(reversedDateTimeWithGMT, DATE_TIME_PARSER); - return zonedDateTime.toEpochSecond(); + TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, + ZonedDateTime::from, + LocalDateTime::from, + OffsetDateTime::from); + //Query for the ZoneID + if (result.query(TemporalQueries.zoneId()) == null) { + //If none, assumed GMT+0. + return ZonedDateTime.of(LocalDateTime.from(result), + ZoneId.of("GMT")).toEpochSecond(); + } else { + return Instant.from(result).getEpochSecond(); + } } - + /** * Reverses the order of the date time components. - * - * Example: - * 1/3/1990 1:23:54 AM UTC+4 - * becomes - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990 + * * @param dateTime * @return */ diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index 496d13d2ad..b2dae939e3 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,7 +33,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Contacts-Contacts files and creates artifacts. */ -final class XRYContactsFileParser extends AbstractSingleKeyValueParser { +final class XRYContactsFileParser extends AbstractSingleEntityParser { private static final Logger logger = Logger.getLogger(XRYContactsFileParser.class.getName()); @@ -42,9 +43,11 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { put("name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); put("tel", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER); put("mobile", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_MOBILE); + put("home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_HOME); put("related application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME); put("address home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LOCATION); put("email home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME); + put("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED); //Ignoring or need more information to decide. put("storage", null); @@ -54,10 +57,10 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { put("account name", null); }}; - + @Override - boolean isKey(String key) { - String normalizedKey = key.toLowerCase(); + boolean canProcess(XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); return XRY_KEYS.containsKey(normalizedKey); } @@ -67,29 +70,36 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { return false; } - @Override - Optional makeAttribute(String nameSpace, String key, String value) { - String normalizedKey = key.toLowerCase(); - if(XRY_KEYS.containsKey(normalizedKey)) { - BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey); - if(attrType != null) { - return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, value)); - } - - logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was " - + "recognized but more examples of its values are needed " - + "to make a decision on an appropriate TSK attribute. " - + "Here is the value [%s].", key, value)); - return Optional.empty(); + /** + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair. + */ + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); + BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey); + if(attrType != null) { + return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, pair.getValue())); } - - throw new IllegalArgumentException(String.format("Key [ %s ] passed the " - + "isKey() test but was not matched.", key)); + + logger.log(Level.INFO, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s ] was recognized but we need " + + "more data or time to finish implementation. Discarding... ", + pair)); + return Optional.empty(); } @Override - void makeArtifact(List attributes, Content parent) throws TskCoreException { - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); - artifact.addAttributes(attributes); + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { + List attributes = new ArrayList<>(); + for(XRYKeyValuePair pair : keyValuePairs) { + Optional attribute = getBlackboardAttribute(pair); + if(attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); + artifact.addAttributes(attributes); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java index c35666d9c1..a641ea1f73 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java @@ -21,9 +21,15 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; import javax.swing.JFileChooser; import javax.swing.JPanel; +import org.apache.commons.lang3.StringUtils; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; /** * Allows an examiner to configure the XRY Data source processor. @@ -32,10 +38,15 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; final class XRYDataSourceProcessorConfigPanel extends JPanel { private static final long serialVersionUID = 1L; + + private static final String PROP_LAST_USED_PATH = "LAST_USED_PATH"; + private static final String SETTINGS_CONTEXT = "XRYDataSourceProcessorConfigPanel_Settings"; + private static final XRYDataSourceProcessorConfigPanel INSTANCE = new XRYDataSourceProcessorConfigPanel(); - //Communicates + //Used to communicate with the DSP infrastructure. This config + //panel will indicate when it is ready for an update. private final PropertyChangeSupport pcs; /** @@ -47,6 +58,31 @@ final class XRYDataSourceProcessorConfigPanel extends JPanel { pcs = new PropertyChangeSupport(this); } + /** + * Persists the last used path between application runs. + */ + private void setLastUsedPath(Path selection) { + Path parent = selection.getParent(); + ModuleSettings.setConfigSetting(SETTINGS_CONTEXT, + PROP_LAST_USED_PATH, parent.toString()); + } + + /** + * Retrieves the last used path, if any. This path will be saved across + * application runs. + */ + private Optional getLastUsedPath() { + String lastFolderPath = ModuleSettings.getConfigSetting( + SETTINGS_CONTEXT, PROP_LAST_USED_PATH); + if (StringUtils.isNotBlank(lastFolderPath)) { + Path lastPath = Paths.get(lastFolderPath); + if (Files.exists(lastPath)) { + return Optional.of(lastPath); + } + } + return Optional.empty(); + } + /** * Gets the singleton XRYDataSourceProcessorConfigPanel. */ @@ -158,9 +194,14 @@ final class XRYDataSourceProcessorConfigPanel extends JPanel { JFileChooser fileChooser = new JFileChooser(); fileChooser.setMultiSelectionEnabled(false); fileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); + Optional lastUsedPath = getLastUsedPath(); + if(lastUsedPath.isPresent()) { + fileChooser.setCurrentDirectory(lastUsedPath.get().toFile()); + } int returnVal = fileChooser.showOpenDialog(this); if (returnVal == JFileChooser.APPROVE_OPTION) { File selection = fileChooser.getSelectedFile(); + setLastUsedPath(selection.toPath()); filePathTextField.setText(selection.getAbsolutePath()); //This will notify the wizard to revalidate the data source processor. diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index 0665eabc64..14d8d54660 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; -import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -35,38 +33,38 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Device-General Information files and creates artifacts. */ -final class XRYDeviceGenInfoFileParser implements XRYFileParser { +final class XRYDeviceGenInfoFileParser extends AbstractSingleEntityParser { private static final Logger logger = Logger.getLogger(XRYDeviceGenInfoFileParser.class.getName()); - //Human readable name of this parser. - private static final String PARSER_NAME = "XRY DSP"; - private static final char KEY_VALUE_DELIMITER = ':'; - //All known XRY keys for Device Gen Info reports. private static final String ATTRIBUTE_KEY = "attribute"; private static final String DATA_KEY = "data"; //All of the known XRY Attribute values for device gen info. The value of the - //attribute keys are actionable for this parser. See parse() header for more - //details. - private static final Map KEY_TO_TYPE + //attribute keys are actionable for this parser. + //Ex: + // Data: Nokia + // Attribute: Device Type + private static final Map XRY_ATTRIBUTE_VALUES = new HashMap() { { put("device name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_NAME); put("device type", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MAKE); put("mobile id (imei)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI); put("security code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD); + put("unlock code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD); put("imei/meid", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI); put("model", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MODEL); put("wifi address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS); + put("subscriber id (imsi)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI); //There could be two of these on an artifact, not aware of a way //to distinguish between two DATE_TIMEs such as the ones below. put("device clock", null); put("pc clock", null); - //Ignore these for now, need more data. + //Ignore these for now, need more data or time to finish implementation. put("device family", null); put("advertising id", null); put("device status", null); @@ -76,192 +74,98 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { put("revision", null); } }; - - /** - * Device-General Information reports have 2 key value pairs for every - * attribute. The two only known keys are "Data" and "Attribute", where data - * is some generic information that the Attribute key describes. - * - * Example: - * - * Data: Nokia XYZ - * Attribute: Device Name - * - * This parse implementation assumes that the data field does not span - * multiple lines. If the data does span multiple lines, it will log an - * error describing an expectation for an "Attribute" key that is not found. - * - * @param reader The XRYFileReader that reads XRY entities from the - * Device-General Information report. - * @param parent The parent Content to create artifacts from. - * @throws IOException If an I/O error is encountered during report reading - * @throws TskCoreException If an error during artifact creation is - * encountered. - */ + + @Override - public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { - Path reportPath = reader.getReportPath(); - logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString())); - - while (reader.hasNextEntity()) { - String xryEntity = reader.nextEntity(); - //Extract attributes from this entity. - List attributes = createTSKAttributes(xryEntity); - if (!attributes.isEmpty()) { - //Save the artifact. - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO); - artifact.addAttributes(attributes); - } - } + boolean canProcess(XRYKeyValuePair pair) { + String key = pair.getKey().trim().toLowerCase(); + return key.equals(DATA_KEY) || key.equals(ATTRIBUTE_KEY); } - /** - * Parses the XRY entity, extracts key value pairs and creates blackboard - * attributes from these key value pairs. - * - * @param xryEntity - * @return A collection of attributes from the XRY entity. - */ - private List createTSKAttributes(String xryEntity) { - //Examine this XRY entity line by line. - String[] xryLines = xryEntity.split("\n"); + @Override + boolean isNamespace(String nameSpace) { + //No known namespaces + return false; + } + + @Override + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { List attributes = new ArrayList<>(); - - //First line of the entity is the title, the entity will always be non-empty. - logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); - - for (int i = 1; i < xryLines.length; i++) { - String xryLine = xryLines[i]; - - //Expecting to see a "Data" key. - if (!hasDataKey(xryLine)) { - logger.log(Level.WARNING, String.format("[XRY DSP] Expected a " - + "'Data' key on this line (in brackets) [ %s ], but none " - + "was found. Discarding... Here is the previous line for" - + " context [ %s ]. What does this mean?", - xryLine, xryLines[i - 1])); - continue; + for(int i = 0; i < keyValuePairs.size(); i+=2) { + Optional attribute; + if(i + 1 == keyValuePairs.size()) { + attribute = getBlackboardAttribute(keyValuePairs.get(i)); + } else { + attribute = getBlackboardAttribute(keyValuePairs.get(i), keyValuePairs.get(i+1)); } - - int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); - String dataValue = xryLine.substring(dataKeyIndex + 1).trim(); - - /** - * If there is only a Data key in the XRY Entity, then assume it is - * the path to the device. - */ - if (i + 1 == xryLines.length) { - attributes.add(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, - PARSER_NAME, dataValue)); - continue; - } - - String nextXryLine = xryLines[++i]; - - //Expecting to see an "Attribute" key - if (!hasXRYAttributeKey(nextXryLine)) { - logger.log(Level.WARNING, String.format("[XRY DSP] Expected an " - + "'Attribute' key on this line (in brackets) [ %s ], " - + "but none was found. Discarding... Here is the previous " - + "line for context [ %s ]. What does this mean?", - nextXryLine, xryLine)); - continue; - } - - int attributeKeyIndex = nextXryLine.indexOf(KEY_VALUE_DELIMITER); - String attributeValue = nextXryLine.substring(attributeKeyIndex + 1).trim(); - String normalizedAttributeValue = attributeValue.toLowerCase(); - - //Check if this value is known. - if (!isXRYAttributeValueRecognized(normalizedAttributeValue)) { - logger.log(Level.WARNING, String.format("[XRY DSP] Attribute value " - + "(in brackets) [ %s ] was not recognized. Discarding... " - + "Here is the data field for context [ %s ]. " - + "What does this mean?", attributeValue, dataValue)); - continue; - } - - Optional attribute = createTSKAttribute( - normalizedAttributeValue, dataValue); - if (attribute.isPresent()) { + if(attribute.isPresent()) { attributes.add(attribute.get()); } } - return attributes; + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact( + BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO); + artifact.addAttributes(attributes); + } } /** - * Creates the appropriate blackboard attribute given the XRY Key Value pair. - * If the value is recognized but has no corresponding Blackboard - * attribute type, the Optional will be empty. - * - * A WARNING message will be logged for all recognized values that don't have - * a type. More data is needed to make a decision about the appropriate type. - * - * @param normalizedAttributeValue Normalized (trimmed and lowercased) - * attribute value to map. - * @param dataValue The value of the blackboard attribute. - * @return Corresponding BlackboardAttribute, if any. + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair. It is assumed that only 'Data' keys can appear by themselves. + * If a Data key is by itself, its value most closely resembles a TSK_PATH attribute. */ - private Optional createTSKAttribute( - String normalizedAttributeValue, String dataValue) { - BlackboardAttribute.ATTRIBUTE_TYPE attrType = KEY_TO_TYPE.get(normalizedAttributeValue); + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { + if (pair.hasKey(DATA_KEY)) { + return Optional.of(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, + PARSER_NAME, pair.getValue())); + } + + logger.log(Level.WARNING, "Expected a 'Data' key value pair, but [ %s ] " + + "was found.", pair); + + return Optional.empty(); + } + + /** + * Creates the appropriate blackboard attribute given two XRY Key Value + * pairs. The expectation is that one pair is the 'Data' key and the other is + * an 'Attribute' key. If the attribute value is recognized but has no corresponding + * Blackboard attribute type, the Optional will be empty. + */ + private Optional getBlackboardAttribute(XRYKeyValuePair firstPair, XRYKeyValuePair secondPair) { + String attributeValue; + String dataValue; + if (firstPair.hasKey(DATA_KEY) && secondPair.hasKey(ATTRIBUTE_KEY)) { + dataValue = firstPair.getValue(); + attributeValue = secondPair.getValue(); + } else if (firstPair.hasKey(ATTRIBUTE_KEY) && secondPair.hasKey(DATA_KEY)) { + dataValue = secondPair.getValue(); + attributeValue = firstPair.getValue(); + } else { + logger.log(Level.WARNING, String.format("[XRY DSP] Expected these key value" + + " pairs (in brackets) [ %s ], [ %s ] to be an 'Attribute' and 'Data' " + + "pair.", firstPair, secondPair)); + return Optional.empty(); + } + + String normalizedAttributeValue = attributeValue.toLowerCase(); + if (!XRY_ATTRIBUTE_VALUES.containsKey(normalizedAttributeValue)) { + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s : %s ] was not recognized. Discarding... ", + attributeValue, dataValue)); + return Optional.empty(); + } + + BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_ATTRIBUTE_VALUES.get(normalizedAttributeValue); if (attrType == null) { - logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was " - + "recognized but more examples of its values are needed " - + "to make a decision on an appropriate TSK attribute. " - + "Here is the value [%s].", normalizedAttributeValue, dataValue)); + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s : %s ] was recognized but we need " + + "more data or time to finish implementation. Discarding... ", + attributeValue, dataValue)); return Optional.empty(); } return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, dataValue)); } - - /** - * Tests if the attribute value is a recognized type. - * - * @param normalizedAttributeValue Normalized (trimmed and lowercased) value - * to test. - * @return True if the attribute value is known, False otherwise. - */ - private boolean isXRYAttributeValueRecognized(String normalizedAttributeValue) { - return KEY_TO_TYPE.containsKey(normalizedAttributeValue); - } - - /** - * Tests if the XRY line has a data key on it. - * - * @param xryLine - * @return - */ - private boolean hasDataKey(String xryLine) { - int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); - //No key structure found. - if (dataKeyIndex == -1) { - return false; - } - - String normalizedDataKey = xryLine.substring(0, - dataKeyIndex).trim().toLowerCase(); - return normalizedDataKey.equals(DATA_KEY); - } - - /** - * Tests if the XRY line has an attribute key on it. - * - * @param xryLine - * @return - */ - private boolean hasXRYAttributeKey(String xryLine) { - int attributeKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); - //No key structure found. - if (attributeKeyIndex == -1) { - return false; - } - - String normalizedDataKey = xryLine.substring(0, - attributeKeyIndex).trim().toLowerCase(); - return normalizedDataKey.equals(ATTRIBUTE_KEY); - } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java index 06492de07b..f7a28d6bff 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java @@ -46,6 +46,7 @@ final class XRYFileParserFactory { case "calls": return new XRYCallsFileParser(); case "contacts/contacts": + case "contacts": return new XRYContactsFileParser(); case "device/general information": return new XRYDeviceGenInfoFileParser(); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java new file mode 100755 index 0000000000..a14f571060 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java @@ -0,0 +1,132 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +/** + * A XRY Key Value pair. XRY Key Value pairs make up the body of XRY entities. + * + * Example: + * + * Attribute: Device Name + * Time: 3/20/2012 12:06:30 AM + * Status: Read + */ +class XRYKeyValuePair { + + private static final char KEY_VALUE_DELIMITER = ':'; + + private final String key; + private final String value; + private final String namespace; + + XRYKeyValuePair(String key, String value, String namespace) { + this.key = key.trim(); + this.value = value.trim(); + this.namespace = namespace.trim(); + } + + /** + * Tests if the key of this pair matches some expected value. + * + * The expected value and the key of this pair will both be + * normalized (trimmed and lowercased). + * + * @param targetKey Key name to test. + */ + boolean hasKey(String targetKey) { + String normalizedKey = key.toLowerCase(); + String normalizedTargetKey = targetKey.trim().toLowerCase(); + return normalizedKey.equals(normalizedTargetKey); + } + + /** + * Retrieves the value contained within this pair. + */ + String getValue() { + return value; + } + + /** + * Retrieves the key contained within this pair. + */ + String getKey() { + return key; + } + + /** + * Retrieves the namespace contained within this pair. + */ + String getNamespace() { + return namespace; + } + + /** + * Tests if the input has the structure of a key value pair. + * + * @param xryLine XRY entity line to test. + */ + static boolean isPair(String input) { + int dataKeyIndex = input.indexOf(KEY_VALUE_DELIMITER); + //No key structure found. + return dataKeyIndex != -1; + } + + /** + * Extracts a key value pair from the input. + * + * This function assumes that there is a key value structure on the line. It + * can be verified using hasKeyValueForm(). + * + * @param input Input key value string to parse. + * @param namespace Namespace to assign to this pair. + * @return + */ + static XRYKeyValuePair from(String input, String namespace) { + if(!isPair(input)) { + throw new IllegalArgumentException("Input does not have the structure" + + " of a key value pair"); + } + + int keyIndex = input.indexOf(KEY_VALUE_DELIMITER); + String parsedKey = input.substring(0, keyIndex); + String parsedValue = input.substring(keyIndex + 1); + return new XRYKeyValuePair(parsedKey, parsedValue, namespace); + } + + /** + * Extracts a key value pair from the input. + * + * This function assumes that there is a key value structure on the line. It + * can be verified using hasKeyValueForm(). + * + * @param input Input key value string to parse. + * @return + */ + static XRYKeyValuePair from(String input) { + return from(input, ""); + } + + @Override + public String toString() { + if(namespace.isEmpty()) { + return key + " : " + value; + } + return "(Namespace: " + namespace +") " + key + " : " + value; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java index 941b26ecc2..74242ddca5 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -20,12 +20,19 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; import java.io.IOException; import java.nio.file.Path; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -42,199 +49,216 @@ final class XRYMessagesFileParser implements XRYFileParser { private static final Logger logger = Logger.getLogger( XRYMessagesFileParser.class.getName()); - + private static final String PARSER_NAME = "XRY DSP"; - private static final char KEY_VALUE_DELIMITER = ':'; - + //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() //function for more details. private static final DateTimeFormatter DATE_TIME_PARSER - = DateTimeFormatter.ofPattern("O a h:m:s M/d/y"); + = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); + + private static final String DEVICE_LOCALE = "(device)"; + private static final String NETWORK_LOCALE = "(network)"; - //A more readable version of these values. Referring to if the user - //has read the message. private static final int READ = 1; private static final int UNREAD = 0; - + /** - * All of the known XRY keys for message reports. + * All of the known XRY keys for message reports and their corresponding + * blackboard attribute types, if any. */ private enum XryKey { - TEXT("text"), - DIRECTION("direction"), - TIME("time"), - STATUS("status"), - TEL("tel"), - STORAGE("storage"), - INDEX("index"), - FOLDER("folder"), - SERVICE_CENTER("service center"), - TYPE("type"), - NAME("name"), - NAME_MATCHED("name (matched)"); - + DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED), + DIRECTION("direction", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION), + MESSAGE("message", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT), + NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON), + TEXT("text", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT), + TIME("time", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME), + SERVICE_CENTER("service center", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER), + FROM("from", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM), + TO("to", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO), + //The following keys either need special processing or more time and data to find a type. + STORAGE("storage", null), + NUMBER("number", null), + TYPE("type", null), + TEL("tel", null), + FOLDER("folder", null), + NAME("name", null), + INDEX("index", null), + STATUS("status", null); + private final String name; - - XryKey(String name) { + private final BlackboardAttribute.ATTRIBUTE_TYPE type; + + XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) { this.name = name; + this.type = type; } - + + public BlackboardAttribute.ATTRIBUTE_TYPE getType() { + return type; + } + + public String getDisplayName() { + return name; + } + /** * Indicates if the display name of the XRY key is a recognized type. - * + * * @param xryKey - * @return + * @return */ - public static boolean contains(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); - for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { - return true; - } + public static boolean contains(String name) { + try { + XryKey.fromDisplayName(name); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } - + /** * Matches the display name of the xry key to the appropriate enum type. - * - * It is assumed that XRY key string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * + * + * It is assumed that XRY key string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * * @param xryKey - * @return + * @return */ - public static XryKey fromDisplayName(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); - for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { + public static XryKey fromDisplayName(String name) { + String normalizedName = name.trim().toLowerCase(); + for (XryKey keyChoice : XryKey.values()) { + if (normalizedName.equals(keyChoice.name)) { return keyChoice; } } - - throw new IllegalArgumentException(String.format("Key [%s] was not found." - + " All keys should be tested with contains.", xryKey)); + + throw new IllegalArgumentException(String.format("Key [ %s ] was not found." + + " All keys should be tested with contains.", name)); } } - + /** * All of the known XRY namespaces for message reports. */ private enum XryNamespace { - TO("to"), FROM("from"), PARTICIPANT("participant"), + TO("to"), NONE(null); - + private final String name; - + XryNamespace(String name) { this.name = name; } - + /** - * Indicates if the display name of the XRY namespace is a recognized type. - * + * Indicates if the display name of the XRY namespace is a recognized + * type. + * * @param xryNamespace - * @return + * @return */ public static boolean contains(String xryNamespace) { - String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { - return true; - } + try { + XryNamespace.fromDisplayName(xryNamespace); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } - + /** - * Matches the display name of the xry namespace to the appropriate enum type. - * - * It is assumed that XRY namespace string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * + * Matches the display name of the xry namespace to the appropriate enum + * type. + * + * It is assumed that XRY namespace string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * * @param xryNamespace - * @return + * @return */ public static XryNamespace fromDisplayName(String xryNamespace) { String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { + for (XryNamespace keyChoice : XryNamespace.values()) { + if (normalizedNamespace.equals(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Namespace [%s] was not found." + " All namespaces should be tested with contains.", xryNamespace)); } } - + /** * All known XRY meta keys for message reports. */ private enum XryMetaKey { REFERENCE_NUMBER("reference number"), - SEGMENT_NUMBER("segment number"), - SEGMENT_COUNT("segments"); - + SEGMENT_COUNT("segments"), + SEGMENT_NUMBER("segment number"); + private final String name; - + XryMetaKey(String name) { this.name = name; } - - /** - * Indicates if the display name of the XRY meta key is a recognized type. - * - * @param xryMetaKey - * @return - */ - public static boolean contains(String xryMetaKey) { - String normalizedMetaKey = xryMetaKey.trim().toLowerCase(); - for(XryMetaKey keyChoice : XryMetaKey.values()) { - if(keyChoice.name.equals(normalizedMetaKey)) { - return true; - } - } - - return false; + + public String getDisplayName() { + return name; } - + /** - * Matches the display name of the xry meta key to the appropriate enum type. - * - * It is assumed that XRY meta key string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * - * @param xryMetaKey - * @return + * Indicates if the display name of the XRY key is a recognized type. + * + * @param xryKey + * @return */ - public static XryMetaKey fromDisplayName(String xryMetaKey) { - String normalizedMetaKey = xryMetaKey.trim().toLowerCase(); - for(XryMetaKey keyChoice : XryMetaKey.values()) { - if(keyChoice.name.equals(normalizedMetaKey)) { + public static boolean contains(String name) { + try { + XryMetaKey.fromDisplayName(name); + return true; + } catch (IllegalArgumentException ex) { + return false; + } + } + + /** + * Matches the display name of the xry key to the appropriate enum type. + * + * It is assumed that XRY key string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * + * @param xryKey + * @return + */ + public static XryMetaKey fromDisplayName(String name) { + String normalizedName = name.trim().toLowerCase(); + for (XryMetaKey keyChoice : XryMetaKey.values()) { + if (normalizedName.equals(keyChoice.name)) { return keyChoice; } } - - throw new IllegalArgumentException(String.format("Meta key [%s] was not found." - + " All meta keys should be tested with contains.", xryMetaKey)); + + throw new IllegalArgumentException(String.format("Key [ %s ] was not found." + + " All keys should be tested with contains.", name)); } } /** * Message-SMS report artifacts can span multiple XRY entities and their - * attributes can span multiple lines. The "Text" key is the only known key - * value pair that can span multiple lines. Messages can be segmented, - * meaning that their "Text" content can appear in multiple XRY entities. - * Our goal for a segmented message is to aggregate all of the text pieces and - * create 1 artifact. + * attributes can span multiple lines. The "Text" and "Message" keys are the + * only known key value pair that can span multiple lines. Messages can be + * segmented, meaning that their "Text" and "Message" content can appear in + * multiple XRY entities. Our goal for a segmented message is to aggregate + * all of the text pieces and create 1 artifact. * * This parse implementation assumes that segments are contiguous and that * they ascend incrementally. There are checks in place to verify this @@ -258,112 +282,9 @@ final class XRYMessagesFileParser implements XRYFileParser { while (reader.hasNextEntity()) { String xryEntity = reader.nextEntity(); - String[] xryLines = xryEntity.split("\n"); - - //First line of the entity is the title, each XRY entity is non-empty. - logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); - - List attributes = new ArrayList<>(); - - XryNamespace namespace = XryNamespace.NONE; - for (int i = 1; i < xryLines.length; i++) { - String xryLine = xryLines[i]; - - if (XryNamespace.contains(xryLine)) { - namespace = XryNamespace.fromDisplayName(xryLine); - continue; - } - - //Find the XRY key on this line. - int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (keyDelimiter == -1) { - logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value " - + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Is this the continuation of a previous line?" - + " Here is the previous line (in brackets) [ %s ]. " - + "What does this key mean?", xryLine, xryLines[i - 1])); - continue; - } - - //Extract the key value pair - String key = xryLine.substring(0, keyDelimiter); - String value = xryLine.substring(keyDelimiter + 1).trim(); - - if (XryMetaKey.contains(key)) { - //Skip meta keys, they are being handled seperately. - continue; - } - - if (!XryKey.contains(key)) { - logger.log(Level.SEVERE, String.format("[XRY DSP] The following key, " - + "value pair (in brackets, respectively) [ %s ], [ %s ] " - + "was not recognized. Discarding... Here is the previous line " - + "[ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); - continue; - } - - if (value.isEmpty()) { - logger.log(Level.SEVERE, String.format("[XRY DSP] The following key " - + "(in brackets) [ %s ] was recognized, but the value " - + "was empty. Discarding... Here is the previous line " - + "for context [ %s ]. Is this a continuation of this line? " - + "What does an empty key mean?", key, xryLines[i - 1])); - continue; - } - - XryKey xryKey = XryKey.fromDisplayName(key); - - //Assume text is the only field that can span multiple lines. - if (xryKey.equals(XryKey.TEXT)) { - //Build up multiple lines. - for (; (i + 1) < xryLines.length - && !hasKey(xryLines[i + 1]) - && !hasNamespace(xryLines[i + 1]); i++) { - String continuedValue = xryLines[i + 1].trim(); - //Assume multi lined values are split by word. - value = value + " " + continuedValue; - } - - Optional referenceNumber = getMetaInfo(xryLines, XryMetaKey.REFERENCE_NUMBER); - //Check if there is any segmented text. - if (referenceNumber.isPresent()) { - logger.log(Level.INFO, String.format("[XRY DSP] Message entity " - + "appears to be segmented with reference number [ %d ]", referenceNumber.get())); - - if (referenceNumbersSeen.contains(referenceNumber.get())) { - logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already " - + "been seen. This means that the segments are not " - + "contiguous. Any segments contiguous with this " - + "one will be aggregated and another " - + "(otherwise duplicate) artifact will be created.", referenceNumber.get())); - } - - referenceNumbersSeen.add(referenceNumber.get()); - - Optional segmentNumber = getMetaInfo(xryLines, XryMetaKey.SEGMENT_NUMBER); - if(segmentNumber.isPresent()) { - //Unify segmented text - String segmentedText = getSegmentedText(referenceNumber.get(), - segmentNumber.get(), reader); - //Assume it was segmented by word. - value = value + " " + segmentedText; - } else { - logger.log(Level.SEVERE, String.format("No segment " - + "number was found on the message entity" - + "with reference number [%d]", referenceNumber.get())); - } - } - } - - //Get the corresponding blackboard attribute, if any. - Optional attribute = makeAttribute(namespace, xryKey, value); - if (attribute.isPresent()) { - attributes.add(attribute.get()); - } - } - + List attributes = getBlackboardAttributes(xryEntity, reader, referenceNumbersSeen); //Only create artifacts with non-empty attributes. - if(!attributes.isEmpty()) { + if (!attributes.isEmpty()) { BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); artifact.addAttributes(attributes); } @@ -371,26 +292,127 @@ final class XRYMessagesFileParser implements XRYFileParser { } /** - * Builds up segmented message entities so that the text is unified in the - * artifact. - * - * @param referenceNumber Reference number that messages are group by - * @param segmentNumber Segment number of the starting segment. - * @param reader + * Extracts all blackboard attributes from the XRY Entity. This function will + * unify any segmented text, if need be. + */ + private List getBlackboardAttributes(String xryEntity, + XRYFileReader reader, Set referenceValues) throws IOException { + String[] xryLines = xryEntity.split("\n"); + //First line of the entity is the title, each XRY entity is non-empty. + logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); + + List attributes = new ArrayList<>(); + + //Count the key value pairs in the XRY entity. + int keyCount = getCountOfKeyValuePairs(xryLines); + for (int i = 1; i <= keyCount; i++) { + //Get the ith key value pair in the entity. Always expect to have + //a valid value. + XRYKeyValuePair pair = getKeyValuePairByIndex(xryLines, i).get(); + if (XryMetaKey.contains(pair.getKey())) { + //Skip meta keys, they are being handled seperately. + continue; + } + + if (!XryKey.contains(pair.getKey())) { + logger.log(Level.WARNING, String.format("[XRY DSP] The following key, " + + "value pair (in brackets) [ %s ], " + + "was not recognized. Discarding...", pair)); + continue; + } + + if (pair.getValue().isEmpty()) { + logger.log(Level.WARNING, String.format("[XRY DSP] The following key " + + "(in brackets) [ %s ] was recognized, but the value " + + "was empty. Discarding...", pair.getKey())); + continue; + } + + //Assume text and message are the only fields that can be segmented + //among multiple XRY entities. + if (pair.hasKey(XryKey.TEXT.getDisplayName()) + || pair.hasKey(XryKey.MESSAGE.getDisplayName())) { + String segmentedText = getSegmentedText(xryLines, reader, referenceValues); + pair = new XRYKeyValuePair(pair.getKey(), + //Assume text is segmented by word. + pair.getValue() + " " + segmentedText, + pair.getNamespace()); + } + + //Get the corresponding blackboard attribute, if any. + Optional attribute = getBlackboardAttribute(pair); + if (attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + + return attributes; + } + + /** + * Counts the key value pairs in an XRY entity. Skips counting the first + * line as it is assumed to be the title. + */ + private Integer getCountOfKeyValuePairs(String[] xryEntity) { + int count = 0; + for (int i = 1; i < xryEntity.length; i++) { + if (XRYKeyValuePair.isPair(xryEntity[i])) { + count++; + } + } + return count; + } + + /** + * Builds up segmented message entities so that the text is unified for a + * single artifact. + * + * @param reader File reader that is producing XRY entities. + * @param referenceNumbersSeen All known references numbers up until this point. + * @param xryEntity The source XRY entity. * @return * @throws IOException */ - private String getSegmentedText(int referenceNumber, int segmentNumber, XRYFileReader reader) throws IOException { + private String getSegmentedText(String[] xryEntity, XRYFileReader reader, + Set referenceNumbersSeen) throws IOException { + Optional referenceNumber = getMetaKeyValue(xryEntity, XryMetaKey.REFERENCE_NUMBER); + //Check if there is any segmented text. + if (!referenceNumber.isPresent()) { + return ""; + } + + logger.log(Level.INFO, String.format("[XRY DSP] Message entity " + + "appears to be segmented with reference number [ %d ]", referenceNumber.get())); + + if (referenceNumbersSeen.contains(referenceNumber.get())) { + logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already " + + "been seen. This means that the segments are not " + + "contiguous. Any segments contiguous with this " + + "one will be aggregated and another " + + "(otherwise duplicate) artifact will be created.", referenceNumber.get())); + } + + referenceNumbersSeen.add(referenceNumber.get()); + + Optional segmentNumber = getMetaKeyValue(xryEntity, XryMetaKey.SEGMENT_NUMBER); + if (!segmentNumber.isPresent()) { + logger.log(Level.SEVERE, String.format("No segment " + + "number was found on the message entity" + + "with reference number [%d]", referenceNumber.get())); + return ""; + } + StringBuilder segmentedText = new StringBuilder(); - int currentSegmentNumber = segmentNumber; + int currentSegmentNumber = segmentNumber.get(); while (reader.hasNextEntity()) { //Peek at the next to see if it has the same reference number. String nextEntity = reader.peek(); String[] nextEntityLines = nextEntity.split("\n"); - Optional nextReferenceNumber = getMetaInfo(nextEntityLines, XryMetaKey.REFERENCE_NUMBER); + Optional nextReferenceNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.REFERENCE_NUMBER); - if (!nextReferenceNumber.isPresent() || nextReferenceNumber.get() != referenceNumber) { + if (!nextReferenceNumber.isPresent() + || !Objects.equals(nextReferenceNumber, referenceNumber)) { //Don't consume the next entity. It is not related //to the current message thread. break; @@ -399,48 +421,32 @@ final class XRYMessagesFileParser implements XRYFileParser { //Consume the entity, it is a part of the message thread. reader.nextEntity(); - Optional nextSegmentNumber = getMetaInfo(nextEntityLines, XryMetaKey.SEGMENT_NUMBER); + Optional nextSegmentNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.SEGMENT_NUMBER); logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] " - + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber)); + + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber.get())); - if(!nextSegmentNumber.isPresent()) { + if (!nextSegmentNumber.isPresent()) { logger.log(Level.SEVERE, String.format("[XRY DSP] Segment with reference" + " number [ %d ] did not have a segment number associated with it." - + " It cannot be determined if the reconstructed text will be in order.", referenceNumber)); + + " It cannot be determined if the reconstructed text will be in order.", referenceNumber.get())); } else if (nextSegmentNumber.get() != currentSegmentNumber + 1) { logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous " + "segments are not ascending incrementally. Encountered " + "segment [ %d ] after segment [ %d ]. This means the reconstructed " - + "text will be out of order.", nextSegmentNumber, currentSegmentNumber)); + + "text will be out of order.", nextSegmentNumber.get(), currentSegmentNumber)); } - for (int i = 1; i < nextEntityLines.length; i++) { - String xryLine = nextEntityLines[i]; - //Find the XRY key on this line. - int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (keyDelimiter == -1) { - //Skip this line, we are searching only for a text key-value pair. - continue; - } - - //Extract the text key from the entity - String key = xryLine.substring(0, keyDelimiter); - if(XryKey.contains(key) && XryKey.fromDisplayName(key).equals(XryKey.TEXT)) { - String value = xryLine.substring(keyDelimiter + 1).trim(); - segmentedText.append(value).append(' '); - - //Build up multiple lines. - for (; (i + 1) < nextEntityLines.length - && !hasKey(nextEntityLines[i + 1]) - && !hasNamespace(nextEntityLines[i + 1]); i++) { - String continuedValue = nextEntityLines[i + 1].trim(); - segmentedText.append(continuedValue).append(' '); - } + int keyCount = getCountOfKeyValuePairs(nextEntityLines); + for (int i = 1; i <= keyCount; i++) { + XRYKeyValuePair pair = getKeyValuePairByIndex(nextEntityLines, i).get(); + if (pair.hasKey(XryKey.TEXT.getDisplayName()) + || pair.hasKey(XryKey.MESSAGE.getDisplayName())) { + segmentedText.append(pair.getValue()).append(' '); } } - if(nextSegmentNumber.isPresent()) { + if (nextSegmentNumber.isPresent()) { currentSegmentNumber = nextSegmentNumber.get(); } } @@ -449,121 +455,138 @@ final class XRYMessagesFileParser implements XRYFileParser { if (segmentedText.length() > 0) { segmentedText.setLength(segmentedText.length() - 1); } + return segmentedText.toString(); } /** - * Determines if the line has recognized key value on it. - * - * @param xryLine - * @return - */ - private boolean hasKey(String xryLine) { - int delimiter = xryLine.indexOf(':'); - if(delimiter == -1) { - return false; - } - - String key = xryLine.substring(0, delimiter); - return XryKey.contains(key); - } - - /** - * Determines if the line is a recognized namespace. - * - * @param xryLine - * @return - */ - private boolean hasNamespace(String xryLine) { - return XryNamespace.contains(xryLine); - } - - /** - * Extracts meta keys from the XRY entity. All of the known meta - * keys are assumed integers and part of the XRY_META_KEY enum. + * Extracts the value of the XRY meta key, if any. * - * @param xryLines Current XRY entity - * @param expectedKey The meta key to search for - * @return The interpreted integer value or Integer.MIN_VALUE if - * no meta key was found. + * @param xryLines XRY entity to extract from. + * @param metaKey The key type to extract. + * @return */ - private Optional getMetaInfo(String[] xryLines, XryMetaKey metaKey) { - for (int i = 0; i < xryLines.length; i++) { - String xryLine = xryLines[i]; + private Optional getMetaKeyValue(String[] xryLines, XryMetaKey metaKey) { + for (String xryLine : xryLines) { + if (!XRYKeyValuePair.isPair(xryLine)) { + continue; + } - int firstDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (firstDelimiter != -1) { - String key = xryLine.substring(0, firstDelimiter); - if(!XryMetaKey.contains(key)) { - continue; - } - - XryMetaKey currentMetaKey = XryMetaKey.fromDisplayName(key); - if (currentMetaKey.equals(metaKey)) { - String value = xryLine.substring(firstDelimiter + 1).trim(); - try { - return Optional.of(Integer.parseInt(value)); - } catch (NumberFormatException ex) { - logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for " - + "meta key [ %s ] was not an integer.", value, metaKey), ex); - } + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine); + if (pair.hasKey(metaKey.getDisplayName())) { + try { + return Optional.of(Integer.parseInt(pair.getValue())); + } catch (NumberFormatException ex) { + logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for " + + "meta key [ %s ] was not an integer.", pair.getValue(), metaKey), ex); } } } + return Optional.empty(); + } + + /** + * Extracts the ith XRY Key Value pair in the XRY Entity. + * + * The total number of pairs can be determined via getCountOfKeyValuePairs(). + * + * @param xryLines XRY entity. + * @param index The requested Key Value pair. + * @return + */ + private Optional getKeyValuePairByIndex(String[] xryLines, int index) { + int pairsParsed = 0; + String namespace = ""; + for (int i = 1; i < xryLines.length; i++) { + String xryLine = xryLines[i]; + if (XryNamespace.contains(xryLine)) { + namespace = xryLine.trim(); + continue; + } + + if (!XRYKeyValuePair.isPair(xryLine)) { + logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value " + + "pair on this line (in brackets) [ %s ], but one was not detected." + + " Discarding...", xryLine)); + continue; + } + + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine); + String value = pair.getValue(); + //Build up multiple lines. + for (; (i + 1) < xryLines.length + && !XRYKeyValuePair.isPair(xryLines[i + 1]) + && !XryNamespace.contains(xryLines[i + 1]); i++) { + String continuedValue = xryLines[i + 1].trim(); + //Assume multi lined values are split by word. + value = value + " " + continuedValue; + } + + pair = new XRYKeyValuePair(pair.getKey(), value, namespace); + pairsParsed++; + if (pairsParsed == index) { + return Optional.of(pair); + } + } return Optional.empty(); } /** * Creates an attribute from the extracted key value pair. - * - * @param nameSpace The namespace of this key value pair. - * It will have been verified beforehand, otherwise it will be NONE. + * + * @param nameSpace The namespace of this key value pair. It will have been + * verified beforehand, otherwise it will be NONE. * @param key The recognized XRY key. * @param value The value associated with that key. * @return Corresponding blackboard attribute, if any. */ - private Optional makeAttribute(XryNamespace namespace, XryKey key, String value) { - String normalizedValue = value.toLowerCase().trim(); + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { + XryNamespace namespace = XryNamespace.NONE; + if (XryNamespace.contains(pair.getNamespace())) { + namespace = XryNamespace.fromDisplayName(pair.getNamespace()); + } + XryKey key = XryKey.fromDisplayName(pair.getKey()); + String normalizedValue = pair.getValue().toLowerCase().trim(); + switch (key) { - case DIRECTION: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, - PARSER_NAME, value)); - case NAME_MATCHED: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON, - PARSER_NAME, value)); case TEL: - if(namespace.equals(XryNamespace.FROM)) { - return Optional.of(new BlackboardAttribute( + case NUMBER: + switch (namespace) { + case FROM: + return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, - PARSER_NAME, value)); - } else { - //Assume TO and PARTICIPANT are TSK_PHONE_NUMBER_TOs - return Optional.of(new BlackboardAttribute( + PARSER_NAME, pair.getValue())); + case TO: + case PARTICIPANT: + return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); + default: + return Optional.of(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, + PARSER_NAME, pair.getValue())); } - case TEXT: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT, - PARSER_NAME, value)); case TIME: try { //Tranform value to seconds since epoch - long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(value); + long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(pair.getValue()); return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeSinceInEpoch)); } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[XRY DSP] Assumption" + " about the date time formatting of messages is " - + "not right. Here is the value [ %s ]", value), ex); + + "not right. Here is the pair [ %s ]", pair), ex); return Optional.empty(); } case TYPE: switch (normalizedValue) { + case "incoming": + case "outgoing": + return Optional.of(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, + PARSER_NAME, pair.getValue())); case "deliver": case "submit": case "status report": @@ -571,22 +594,18 @@ final class XRYMessagesFileParser implements XRYFileParser { return Optional.empty(); default: logger.log(Level.WARNING, String.format("[XRY DSP] Unrecognized " - + "type value [ %s ]", value)); + + " value for key pair [ %s ].", pair)); return Optional.empty(); } - case SERVICE_CENTER: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, - PARSER_NAME, value)); case STATUS: switch (normalizedValue) { case "read": return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, READ)); case "unread": return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, UNREAD)); case "sending failed": case "deleted": @@ -596,18 +615,22 @@ final class XRYMessagesFileParser implements XRYFileParser { return Optional.empty(); default: logger.log(Level.WARNING, String.format("[XRY DSP] Unrecognized " - + "status value [ %s ].", value)); + + " value for key pair [ %s ].", pair)); return Optional.empty(); } - case STORAGE: - case INDEX: - case FOLDER: - case NAME: - //Ignore for now. - return Optional.empty(); default: - throw new IllegalArgumentException(String.format("Key [ %s ] " - + "passed the isKey() test but was not matched.", key)); + //Otherwise, the XryKey enum contains the correct BlackboardAttribute + //type. + if (key.getType() != null) { + return Optional.of(new BlackboardAttribute(key.getType(), + PARSER_NAME, pair.getValue())); + } + + logger.log(Level.INFO, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s ] was recognized but " + + "more data or time is needed to finish implementation. Discarding... ", pair)); + + return Optional.empty(); } } @@ -620,12 +643,16 @@ final class XRYMessagesFileParser implements XRYFileParser { * @return A purer date time value. */ private String removeDateTimeLocale(String dateTime) { - int index = dateTime.indexOf('('); - if (index == -1) { - return dateTime; + String result = dateTime; + int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE); + if (deviceIndex != -1) { + result = result.substring(0, deviceIndex); } - - return dateTime.substring(0, index); + int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); + if (networkIndex != -1) { + result = result.substring(0, networkIndex); + } + return result; } /** @@ -638,42 +665,48 @@ final class XRYMessagesFileParser implements XRYFileParser { String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim(); /** * The format of time in XRY Messages reports is of the form: - * - * 1/3/1990 1:23:54 AM UTC+4 - * - * In our current version of Java (openjdk-1.8.0.222), there is - * a bug with having the timezone offset (UTC+4 or GMT-7) at the - * end of the date time input. This is fixed in later versions - * of the JDK (9 and beyond). - * https://bugs.openjdk.java.net/browse/JDK-8154050 - * Rather than update the JDK to accommodate this, the components of - * the date time string are reversed: - * - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * 1/3/1990 1:23:54 AM UTC+4 + * + * In our current version of Java (openjdk-1.8.0.222), there is a bug + * with having the timezone offset (UTC+4 or GMT-7) at the end of the + * date time input. This is fixed in later versions of the JDK (9 and + * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than + * update the JDK to accommodate this, the components of the date time + * string are reversed: + * + * UTC+4 AM 1:23:54 1/3/1990 + * * The java time package will correctly parse this date time format. */ String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale); /** - * Furthermore, the DateTimeFormatter's timezone offset letter ('O') does - * not recognize UTC but recognizes GMT. According to - * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, - * GMT only differs from UTC by at most 1 second and so substitution - * will only introduce a trivial amount of error. + * Furthermore, the DateTimeFormatter's timezone offset letter ('O') + * does not recognize UTC but recognizes GMT. According to + * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only + * differs from UTC by at most 1 second and so substitution will only + * introduce a trivial amount of error. */ String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); - ZonedDateTime zonedDateTime = ZonedDateTime.parse(reversedDateTimeWithGMT, DATE_TIME_PARSER); - return zonedDateTime.toEpochSecond(); + TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, + ZonedDateTime::from, + LocalDateTime::from, + OffsetDateTime::from); + //Query for the ZoneID + if (result.query(TemporalQueries.zoneId()) == null) { + //If none, assumed GMT+0. + return ZonedDateTime.of(LocalDateTime.from(result), + ZoneId.of("GMT")).toEpochSecond(); + } else { + return Instant.from(result).getEpochSecond(); + } } - + /** * Reverses the order of the date time components. - * - * Example: - * 1/3/1990 1:23:54 AM UTC+4 - * becomes - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990 + * * @param dateTime * @return */ diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java index 98ee693101..a67ce772f4 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java @@ -54,7 +54,7 @@ final class XRYReportProcessor { XRYFileParser parser = XRYFileParserFactory.get(reportType); parser.parse(xryFileReader, parent); } else { - logger.log(Level.SEVERE, String.format("[XRY DSP] XRY File (in brackets) " + logger.log(Level.WARNING, String.format("[XRY DSP] XRY File (in brackets) " + "[ %s ] was found, but no parser to support its report type exists. " + "Report type is [ %s ]", xryFileReader.getReportPath().toString(), reportType)); } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index 6f21048230..9678db0274 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; +import java.util.ArrayList; import java.util.Map; import java.util.HashMap; import java.util.List; @@ -30,21 +31,22 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Web-Bookmark files and creates artifacts. */ -final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { +final class XRYWebBookmarksFileParser extends AbstractSingleEntityParser { //All known XRY keys for web bookmarks. - private static final Map KEY_TO_TYPE + private static final Map XRY_KEYS = new HashMap() { { put("web address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL); put("domain", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN); + put("application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME); } }; @Override - boolean isKey(String key) { - String normalizedKey = key.toLowerCase(); - return KEY_TO_TYPE.containsKey(normalizedKey); + boolean canProcess(XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); + return XRY_KEYS.containsKey(normalizedKey); } @Override @@ -53,17 +55,29 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { return false; } - @Override - Optional makeAttribute(String nameSpace, String key, String value) { - String normalizedKey = key.toLowerCase(); + /** + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair. + */ + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); return Optional.of(new BlackboardAttribute( - KEY_TO_TYPE.get(normalizedKey), - PARSER_NAME, value)); + XRY_KEYS.get(normalizedKey), + PARSER_NAME, pair.getValue())); } @Override - void makeArtifact(List attributes, Content parent) throws TskCoreException { - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK); - artifact.addAttributes(attributes); + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { + List attributes = new ArrayList<>(); + for(XRYKeyValuePair pair : keyValuePairs) { + Optional attribute = getBlackboardAttribute(pair); + if(attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK); + artifact.addAttributes(attributes); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java index bb43f64c84..522d3836c3 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java @@ -41,7 +41,6 @@ import org.sleuthkit.autopsy.actions.DeleteFileBlackboardArtifactTagAction; import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction; import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType; import org.sleuthkit.autopsy.datamodel.AbstractFsContentNode; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; import org.sleuthkit.autopsy.datamodel.DataModelActionsFactory; @@ -259,15 +258,20 @@ public class DataResultFilterNode extends FilterNode { @Override protected Node[] createNodes(Node key) { - // filter out all non-message artifacts, if displaying the results from the Data Source tree + // if displaying the results from the Data Source tree + // filter out artifacts + + // In older versions of Autopsy, attachments were children of email/message artifacts + // and hence email/messages with attachments are shown in the tree data source tree, BlackboardArtifact art = key.getLookup().lookup(BlackboardArtifact.class); - if (art != null - && filterArtifacts - && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() - && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) { + if (art != null && filterArtifacts + && ((FilterNodeUtils.showMessagesInDatasourceTree() == false) + || (FilterNodeUtils.showMessagesInDatasourceTree() + && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() + && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()))) { return new Node[]{}; } - + return new Node[]{new DataResultFilterNode(key, sourceEm)}; } } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java index c9070015f2..06a0b18c10 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.directorytree; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.logging.Level; import javax.swing.Action; @@ -33,17 +32,12 @@ import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.AbstractContentNode; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; -import org.sleuthkit.autopsy.ingest.runIngestModuleWizard.RunIngestModulesAction; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.Directory; -import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.VirtualDirectory; -import org.sleuthkit.datamodel.Volume; /** * A node filter (decorator) that sets the actions for a node in the tree view @@ -137,11 +131,18 @@ class DirectoryTreeFilterNode extends FilterNode { numVisibleChildren--; } } else if (child instanceof BlackboardArtifact) { - BlackboardArtifact bba = (BlackboardArtifact) child; - - // Only message type artifacts are displayed in the tree - if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID()) - && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) { + + if (FilterNodeUtils.showMessagesInDatasourceTree()) { + // In older versions of Autopsy, attachments were children of email/message artifacts + // and hence email/messages with attachments are shown in the directory tree. + BlackboardArtifact bba = (BlackboardArtifact) child; + // Only message type artifacts are displayed in the tree + if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID()) + && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) { + numVisibleChildren--; + } + } + else { numVisibleChildren--; } } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/FilterNodeUtils.java b/Core/src/org/sleuthkit/autopsy/directorytree/FilterNodeUtils.java new file mode 100644 index 0000000000..ce8344862c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/FilterNodeUtils.java @@ -0,0 +1,66 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.directorytree; + +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; + +/** + * Utility class for Directory tree. + * + */ +final class FilterNodeUtils { + + private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER = 8; + private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER = 4; + + /** + * Empty private constructor + */ + private FilterNodeUtils() { + + } + + /** + * Prior to schema version 8.4, attachments were children of messages and + * hence messages with any attachment children are shown in the directory + * tree. + * + * At 8.4 and later, attachments are tracked as an attribute, and the message + * artifacts don't need to be shown in the directory tree. + * + * This method may be used to check the schema version and behave + * accordingly, in order to maintain backward compatibility. + * + * @return True if messages with attachment children should be shown in + * directory tree. + */ + static boolean showMessagesInDatasourceTree() { + boolean showMessagesInDatasourceTree = true; + if (Case.isCaseOpen()) { + CaseDbSchemaVersionNumber version = Case.getCurrentCase().getSleuthkitCase().getDBSchemaCreationVersion(); + showMessagesInDatasourceTree + = ((version.getMajor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER) + || (version.getMajor() == ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER && version.getMinor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER)); + } + return showMessagesInDatasourceTree; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java b/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java index 1fad5d82ae..35ee60ac5c 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java @@ -32,7 +32,7 @@ import org.sleuthkit.autopsy.datamodel.FileNode; import org.sleuthkit.datamodel.AbstractFile; /** - * Create a node containing the children for the to display in the + * Create a node containing the children to display in the * DataResultViewerThumbnail */ class DiscoveryThumbnailChildren extends Children.Keys { diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java index 648eb6457a..495b70f4e4 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java @@ -977,16 +977,6 @@ class FileSearch { GroupKey getGroupKey(ResultFile file) { return new FileTypeGroupKey(file); } - - @Override - void addAttributeToResultFiles(List files, SleuthkitCase caseDb, - EamDb centralRepoDb) throws FileSearchException { - for (ResultFile file : files) { - if (file.getFileType().equals(FileType.OTHER)) { - file.setFileType(FileType.fromMIMEtype(file.getFirstInstance().getMIMEType())); - } - } - } } /** diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java index 3b28a5e97d..8b903bf8b9 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java @@ -72,7 +72,7 @@ class ResultFile { tagNames = new ArrayList<>(); interestingSetNames = new ArrayList<>(); objectDetectedNames = new ArrayList<>(); - fileType = FileType.OTHER; + fileType = FileType.fromMIMEtype(abstractFile.getMIMEType()); } /** @@ -103,6 +103,9 @@ class ResultFile { if (deleted && !duplicate.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC)) { deleted = false; } + if (fileType == FileType.OTHER) { + fileType = FileType.fromMIMEtype(duplicate.getMIMEType()); + } updateScoreAndDescription(duplicate); instances.add(duplicate); } @@ -156,15 +159,6 @@ class ResultFile { return fileType; } - /** - * Set the file type - * - * @param fileType the type - */ - void setFileType(FileType fileType) { - this.fileType = fileType; - } - /** * Add a keyword list name that matched this file. * diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java index 27086c731c..3e4f8b7a68 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java @@ -88,17 +88,21 @@ public class ResultsPanel extends javax.swing.JPanel { imageThumbnailViewer = new ImageThumbnailViewer(); videoThumbnailViewer = new VideoThumbnailViewer(); videoThumbnailViewer.addListSelectionListener((e) -> { - if (!e.getValueIsAdjusting()) { - populateInstancesList(); - } else { - instancesList.clearSelection(); + if (resultType == FileSearchData.FileType.VIDEO) { + if (!e.getValueIsAdjusting()) { + populateInstancesList(); + } else { + instancesList.clearSelection(); + } } }); imageThumbnailViewer.addListSelectionListener((e) -> { - if (!e.getValueIsAdjusting()) { - populateInstancesList(); - } else { - instancesList.clearSelection(); + if (resultType == FileSearchData.FileType.IMAGE) { + if (!e.getValueIsAdjusting()) { + populateInstancesList(); + } else { + instancesList.clearSelection(); + } } }); //Add the context menu when right clicking diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties index 5525fed84c..e21db15ee9 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties @@ -24,7 +24,7 @@ GeolocationSettingsPanel.mbtileFileField.toolTipText= GeolocationSettingsPanel.mbtileFileField.text= GeolocationSettingsPanel.defaultDataSource.text=Default online tile server (bing.com/maps) GeolocationSettingsPanel.osmServerRBnt.text=OpenStreetMap server -GeolocationSettingsPanel.zipFileRBnt.text=OpenStreeMap zip file +GeolocationSettingsPanel.zipFileRBnt.text=OpenStreetMap zip file GeolocationSettingsPanel.zipFileRBnt.actionCommand=OpenStreeMap tile ZIP file GeolocationSettingsPanel.mbtilesRBtn.text=MBTiles file GeolocationSettingsPanel.osmServerAddressField.text= diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED index 19ce2c9a59..1fbba07220 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED @@ -59,7 +59,7 @@ GeolocationSettingsPanel.mbtileFileField.toolTipText= GeolocationSettingsPanel.mbtileFileField.text= GeolocationSettingsPanel.defaultDataSource.text=Default online tile server (bing.com/maps) GeolocationSettingsPanel.osmServerRBnt.text=OpenStreetMap server -GeolocationSettingsPanel.zipFileRBnt.text=OpenStreeMap zip file +GeolocationSettingsPanel.zipFileRBnt.text=OpenStreetMap zip file GeolocationSettingsPanel.zipFileRBnt.actionCommand=OpenStreeMap tile ZIP file GeolocationSettingsPanel.mbtilesRBtn.text=MBTiles file GeolocationSettingsPanel.osmServerAddressField.text= diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form index e2121d0772..ed4de8758a 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form @@ -145,7 +145,7 @@ - + @@ -160,7 +160,7 @@ - + @@ -200,36 +200,45 @@ - - - - - - - - - + + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java index f66d05c08d..c2819d4a4f 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java @@ -159,6 +159,7 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio serverTestBtn = new javax.swing.JButton(); mbtilesRBtn = new javax.swing.JRadioButton(); mbtileFileField = new javax.swing.JTextField(); + javax.swing.JPanel MBTilesBtnPanel = new javax.swing.JPanel(); mbtilesBrowseBtn = new javax.swing.JButton(); mbtileTestBtn = new javax.swing.JButton(); @@ -236,7 +237,7 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 2; gridBagConstraints.gridy = 2; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); tilePane.add(zipFileBrowseBnt, gridBagConstraints); @@ -249,8 +250,8 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 2; gridBagConstraints.gridy = 1; - gridBagConstraints.ipadx = 20; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); tilePane.add(serverTestBtn, gridBagConstraints); @@ -277,18 +278,15 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio gridBagConstraints.insets = new java.awt.Insets(0, 0, 9, 0); tilePane.add(mbtileFileField, gridBagConstraints); + MBTilesBtnPanel.setLayout(new java.awt.GridLayout(1, 0, 5, 0)); + org.openide.awt.Mnemonics.setLocalizedText(mbtilesBrowseBtn, org.openide.util.NbBundle.getMessage(GeolocationSettingsPanel.class, "GeolocationSettingsPanel.mbtilesBrowseBtn.text")); // NOI18N mbtilesBrowseBtn.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { mbtilesBrowseBtnActionPerformed(evt); } }); - gridBagConstraints = new java.awt.GridBagConstraints(); - gridBagConstraints.gridx = 2; - gridBagConstraints.gridy = 3; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; - gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); - tilePane.add(mbtilesBrowseBtn, gridBagConstraints); + MBTilesBtnPanel.add(mbtilesBrowseBtn); org.openide.awt.Mnemonics.setLocalizedText(mbtileTestBtn, org.openide.util.NbBundle.getMessage(GeolocationSettingsPanel.class, "GeolocationSettingsPanel.mbtileTestBtn.text")); // NOI18N mbtileTestBtn.addActionListener(new java.awt.event.ActionListener() { @@ -296,13 +294,15 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio mbtileTestBtnActionPerformed(evt); } }); + MBTilesBtnPanel.add(mbtileTestBtn); + gridBagConstraints = new java.awt.GridBagConstraints(); - gridBagConstraints.gridx = 3; + gridBagConstraints.gridx = 2; gridBagConstraints.gridy = 3; - gridBagConstraints.ipadx = 20; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; - gridBagConstraints.weightx = 1.0; - tilePane.add(mbtileTestBtn, gridBagConstraints); + gridBagConstraints.gridwidth = 2; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); + tilePane.add(MBTilesBtnPanel, gridBagConstraints); gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java index b1dee1cbb4..aafd468957 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java @@ -431,6 +431,8 @@ public final class GeolocationTopComponent extends TopComponent { Bundle.GeoTopComponent_filter_exception_Title(), Bundle.GeoTopComponent_filter_exception_msg(), JOptionPane.ERROR_MESSAGE); + + setWaypointLoading(false); } }); } diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java b/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java index 656abc9489..3a2f305083 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java @@ -417,8 +417,6 @@ public class KdTree implements Iterable { } Double nodeDistance = node.id.euclideanDistance(value); if (nodeDistance.compareTo(lastDistance) < 0) { - if (results.size() == K && lastNode != null) - results.remove(lastNode); results.add(node); } else if (nodeDistance.equals(lastDistance)) { results.add(node); diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java index 7be6ac25a8..a90dac796b 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java @@ -70,6 +70,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationDataException; import org.sleuthkit.datamodel.TskCoreException; import javax.imageio.ImageIO; +import javax.swing.SwingUtilities; import org.jxmapviewer.viewer.DefaultWaypointRenderer; /** @@ -204,13 +205,12 @@ final public class MapPanel extends javax.swing.JPanel { Iterator iterator = waypointTree.iterator(); while (iterator.hasNext()) { MapWaypoint point = iterator.next(); - if (point != currentlySelectedWaypoint) { - set.add(point); - } + set.add(point); } // Add the currentlySelectedWaypoint to the end so that // it will be painted last. if (currentlySelectedWaypoint != null) { + set.remove(currentlySelectedWaypoint); set.add(currentlySelectedWaypoint); } } @@ -342,7 +342,11 @@ final public class MapPanel extends javax.swing.JPanel { */ private void showPopupMenu(Point point) { try { - MapWaypoint waypoint = findClosestWaypoint(point); + List waypoints = findClosestWaypoint(point); + MapWaypoint waypoint = null; + if(waypoints.size() > 0) { + waypoint = waypoints.get(0); + } showPopupMenu(waypoint, point); // Change the details popup to the currently selected point only if // it the popup is currently visible @@ -410,6 +414,7 @@ final public class MapPanel extends javax.swing.JPanel { currentPopup = popupFactory.getPopup(this, detailPane, popupLocation.x, popupLocation.y); currentPopup.show(); + mapViewer.revalidate(); mapViewer.repaint(); } } @@ -437,7 +442,7 @@ final public class MapPanel extends javax.swing.JPanel { * @return A waypoint that is within 10 pixels of the given point, or null * if none was found. */ - private MapWaypoint findClosestWaypoint(Point mouseClickPoint) { + private List findClosestWaypoint(Point mouseClickPoint) { if (waypointTree == null) { return null; } @@ -446,7 +451,7 @@ final public class MapPanel extends javax.swing.JPanel { GeoPosition geopos = mapViewer.getTileFactory().pixelToGeo(mouseClickPoint, mapViewer.getZoom()); // Get the 5 nearest neightbors to the point - Collection waypoints = waypointTree.nearestNeighbourSearch(20, MapWaypoint.getDummyWaypoint(geopos)); + Collection waypoints = waypointTree.nearestNeighbourSearch(10, MapWaypoint.getDummyWaypoint(geopos)); if (waypoints == null || waypoints.isEmpty()) { return null; @@ -456,6 +461,7 @@ final public class MapPanel extends javax.swing.JPanel { // These maybe the points closest to lat/log was clicked but // that doesn't mean they are close in terms of pixles. + List closestPoints = new ArrayList<>(); while (iterator.hasNext()) { MapWaypoint nextWaypoint = iterator.next(); @@ -466,11 +472,11 @@ final public class MapPanel extends javax.swing.JPanel { (int) point.getY() - rect.y); if (converted_gp_pt.distance(mouseClickPoint) < 10) { - return nextWaypoint; + closestPoints.add(nextWaypoint); } } - return null; + return closestPoints; } /** @@ -629,8 +635,14 @@ final public class MapPanel extends javax.swing.JPanel { }//GEN-LAST:event_mapViewerMouseMoved private void mapViewerMouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_mapViewerMouseClicked - if(!evt.isPopupTrigger() && (evt.getButton() == MouseEvent.BUTTON1)) { - currentlySelectedWaypoint = findClosestWaypoint(evt.getPoint()); + if(!evt.isPopupTrigger() && SwingUtilities.isLeftMouseButton(evt)) { + List waypoints = findClosestWaypoint(evt.getPoint()); + if(waypoints.size() > 0) { + currentlySelectedWaypoint = waypoints.get(0); + } + + +// currentlySelectedWaypoint = findClosestWaypoint(evt.getPoint()); showDetailsPopup(); } }//GEN-LAST:event_mapViewerMouseClicked diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java new file mode 100755 index 0000000000..bc3480ea40 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java @@ -0,0 +1,79 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.geolocation.datamodel; + +import java.util.Map; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; + +/** + * Class wraps any artifact that is not one of the known types, but have the + * TSK_GEO_LONGITUDE and TSK_GEO_LATITUDE attributes. + * + */ +final class CustomArtifactWaypoint extends Waypoint { + + /** + * Constructs a new waypoint from the given artifact. + * + * @param artifact BlackboardArtifact for this waypoint + * + * @throws GeoLocationDataException + */ + CustomArtifactWaypoint(BlackboardArtifact artifact) throws GeoLocationDataException { + this(artifact, getAttributesFromArtifactAsMap(artifact)); + } + + /** + * Constructs a new CustomArtifactWaypoint. + * + * @param artifact BlackboardArtifact for this waypoint + * @param attributeMap A Map of the BlackboardAttributes for the given + * artifact. + * + * @throws GeoLocationDataException + */ + private CustomArtifactWaypoint(BlackboardArtifact artifact, Map attributeMap) throws GeoLocationDataException { + super(artifact, + getLabelFromArtifact(attributeMap), + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME).getValueLong() : null, + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE).getValueDouble() : null, + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE).getValueDouble() : null, + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE).getValueDouble() : null, + null, attributeMap, null); + } + + /** + * Gets the label for this waypoint. + * + * @param artifact BlackboardArtifact for waypoint + * + * @return Returns a label for the waypoint, or empty string if no label was + * found. + */ + private static String getLabelFromArtifact(Map attributeMap) { + BlackboardAttribute attribute = attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); + if (attribute != null) { + return attribute.getDisplayString(); + } + + return ""; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java index 7bf85874ff..483dfd4689 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java @@ -71,7 +71,7 @@ final class LastKnownWaypoint extends Waypoint { "LastKnownWaypoint_Label=Last Known Location",}) private static String getLabelFromArtifact(Map attributeMap) throws GeoLocationDataException { BlackboardAttribute attribute = attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); - String label = attribute.getDisplayString(); + String label = attribute != null ? attribute.getDisplayString() : Bundle.LastKnownWaypoint_Label(); if (label == null || label.isEmpty()) { label = Bundle.LastKnownWaypoint_Label(); diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java index 86539412be..bccf5118a5 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java @@ -592,8 +592,11 @@ public final class WaypointBuilder { Route route = new Route(artifact); waypoints.addAll(route.getRoute()); break; + case TSK_GPS_LAST_KNOWN_LOCATION: + waypoints.add(new LastKnownWaypoint(artifact)); + break; default: - throw new GeoLocationDataException(String.format("Unable to create waypoint for artifact of type %s", type.toString())); + waypoints.add(new CustomArtifactWaypoint(artifact)); } return waypoints; diff --git a/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java b/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java index c75c7c814e..146bc156ed 100755 --- a/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java @@ -209,7 +209,6 @@ public final class EventsModel { * data source data in the case database. */ synchronized private void populateDataSourcesCache() throws TskCoreException { - datasourceIDsToNamesMap.clear(); SleuthkitCase skCase = currentCase.getSleuthkitCase(); for (DataSource ds : skCase.getDataSources()) { datasourceIDsToNamesMap.putIfAbsent(ds.getId(), ds.getName()); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 9866c500a0..4dcd7ff984 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -578,8 +578,16 @@ public final class KeywordSearchIngestModule implements FileIngestModule { TskData.TSK_DB_FILES_TYPE_ENUM aType = aFile.getType(); - // unallocated and unused blocks can only have strings extracted from them. - if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) || aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))) { + /** + * Extract unicode strings from unallocated and unused blocks and + * carved text files. The reason for performing string extraction + * on these is because they all may contain multiple encodings which + * can cause text to be missed by the more specialized text extractors + * used below. + */ + if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) + || (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.CARVED) && aFile.getNameExtension().equalsIgnoreCase("txt"))) { if (context.fileIngestIsCancelled()) { return; } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED index cdfd241886..90ce00170c 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED @@ -15,6 +15,7 @@ ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index e # {0} - file name # {1} - file ID ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse. +ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message. ThunderbirdMboxFileIngestModule.moduleName=Email Parser ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case. ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace=Out of disk space. Cannot copy {0} to parse. diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index f1260f269f..983e59e6b4 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -58,6 +59,9 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskDataException; import org.sleuthkit.datamodel.TskException; +import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper; +import org.sleuthkit.datamodel.blackboardutils.FileAttachment; +import org.sleuthkit.datamodel.blackboardutils.MessageAttachments; /** * File-level ingest module that detects MBOX, PST, and vCard files based on @@ -70,6 +74,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private FileManager fileManager; private IngestJobContext context; private Blackboard blackboard; + private CommunicationArtifactsHelper communicationArtifactsHelper; private Case currentCase; @@ -129,6 +134,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.WARNING, null, ex); } + try { + communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(), + EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Failed to create CommunicationArtifactsHelper for file with object id = %d", abstractFile.getId()), ex); + return ProcessResult.ERROR; + } + + if (isMbox) { return processMBox(abstractFile); } @@ -267,7 +281,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS } - emailFolder = emailFolder + mboxFileName; + emailFolder += mboxFileName; emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS String fileName; @@ -487,8 +501,12 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * * @return List of attachments */ + @NbBundle.Messages({ + "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message." +}) private List handleAttachments(List attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) { List files = new ArrayList<>(); + List fileAttachments = new ArrayList<>(); for (EmailMessage.Attachment attach : attachments) { String filename = attach.getName(); long crTime = attach.getCrTime(); @@ -501,12 +519,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { try { DerivedFile df = fileManager.addDerivedFile(filename, relPath, - size, cTime, crTime, aTime, mTime, true, messageArtifact, "", + size, cTime, crTime, aTime, mTime, true, abstractFile, "", EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType); associateAttachmentWithMesssge(messageArtifact, df); files.add(df); + + fileAttachments.add(new FileAttachment(df)); } catch (TskCoreException ex) { postErrorMessage( NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.errMsg", @@ -516,6 +536,17 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.INFO, "", ex); } } + + + try { + communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList())); + } catch (TskCoreException ex) { + postErrorMessage( + NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"), + ""); + logger.log(Level.INFO, "Failed to add attachments to email message.", ex); + } + return files; }