diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 04ab92821a..da599ea7d7 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -18,15 +18,7 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.time.temporal.TemporalAccessor; -import java.time.temporal.TemporalQueries; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -50,14 +42,6 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName()); - //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() - //function for more details. - private static final DateTimeFormatter DATE_TIME_PARSER - = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); - - private static final String DEVICE_LOCALE = "(device)"; - private static final String NETWORK_LOCALE = "(network)"; - /** * All of the known XRY keys for call reports and their corresponding * blackboard attribute types, if any. @@ -202,6 +186,10 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { switch (xryKey) { case TEL: case NUMBER: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + // Apply namespace or direction if (xryNamespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) { callerId = pair.getValue(); @@ -216,15 +204,23 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { // Although confusing, as these are also 'name spaces', it appears // later versions of XRY just made these standardized lines. case TO: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + calleeList.add(pair.getValue()); break; case FROM: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + callerId = pair.getValue(); break; case TIME: try { //Tranform value to seconds since epoch - long dateTimeSinceEpoch = calculateSecondsSinceEpoch(pair.getValue()); + long dateTimeSinceEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue()); startTime = dateTimeSinceEpoch; } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[XRY DSP] Assumption" @@ -322,89 +318,4 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { endTime, callType, otherAttributes); } } - - /** - * Removes the locale from the date time value. - * - * Locale in this case being (Device) or (Network). - * - * @param dateTime XRY datetime value to be sanitized. - * @return A purer date time value. - */ - private String removeDateTimeLocale(String dateTime) { - String result = dateTime; - int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE); - if (deviceIndex != -1) { - result = result.substring(0, deviceIndex); - } - int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); - if (networkIndex != -1) { - result = result.substring(0, networkIndex); - } - return result; - } - - /** - * Parses the date time value and calculates seconds since epoch. - * - * @param dateTime - * @return - */ - private long calculateSecondsSinceEpoch(String dateTime) { - String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim(); - /** - * The format of time in XRY Messages reports is of the form: - * - * 1/3/1990 1:23:54 AM UTC+4 - * - * In our current version of Java (openjdk-1.8.0.222), there is a bug - * with having the timezone offset (UTC+4 or GMT-7) at the end of the - * date time input. This is fixed in later versions of the JDK (9 and - * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than - * update the JDK to accommodate this, the components of the date time - * string are reversed: - * - * UTC+4 AM 1:23:54 1/3/1990 - * - * The java time package will correctly parse this date time format. - */ - String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale); - /** - * Furthermore, the DateTimeFormatter's timezone offset letter ('O') - * does not recognize UTC but recognizes GMT. According to - * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only - * differs from UTC by at most 1 second and so substitution will only - * introduce a trivial amount of error. - */ - String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); - TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, - ZonedDateTime::from, - LocalDateTime::from, - OffsetDateTime::from); - //Query for the ZoneID - if (result.query(TemporalQueries.zoneId()) == null) { - //If none, assumed GMT+0. - return ZonedDateTime.of(LocalDateTime.from(result), - ZoneId.of("GMT")).toEpochSecond(); - } else { - return Instant.from(result).getEpochSecond(); - } - } - - /** - * Reverses the order of the date time components. - * - * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990 - * - * @param dateTime - * @return - */ - private String reverseOrderOfDateTimeComponents(String dateTime) { - StringBuilder reversedDateTime = new StringBuilder(dateTime.length()); - String[] dateTimeComponents = dateTime.split(" "); - for (String component : dateTimeComponents) { - reversedDateTime.insert(0, " ").insert(0, component); - } - return reversedDateTime.toString().trim(); - } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index 4b539adddb..bf73b2b4d7 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -74,6 +74,10 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser { } break; case TEL: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + if(phoneNumber != null) { additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, pair.getValue())); } else { @@ -81,6 +85,10 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser { } break; case MOBILE: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + if(mobilePhoneNumber != null) { additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_MOBILE, PARSER_NAME, pair.getValue())); } else { @@ -88,22 +96,32 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser { } break; case HOME: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + if(homePhoneNumber != null) { additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_HOME, PARSER_NAME, pair.getValue())); } else { homePhoneNumber = pair.getValue(); } break; + case EMAIL_HOME: + if(!XRYUtils.isEmailValid(pair.getValue())) { + continue; + } + + hasAnEmail = true; + additionalAttributes.add(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME, + PARSER_NAME, pair.getValue())); + break; default: //Otherwise, the XryKey enum contains the correct BlackboardAttribute //type. if (xryKey.getType() != null) { additionalAttributes.add(new BlackboardAttribute(xryKey.getType(), PARSER_NAME, pair.getValue())); - - if(xryKey == XryKey.EMAIL_HOME) { - hasAnEmail = true; - } } logger.log(Level.INFO, String.format("[XRY DSP] Key value pair " @@ -139,7 +157,7 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser { HOME("home", null), RELATED_APPLICATION("related application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME), ADDRESS_HOME("address home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LOCATION), - EMAIL_HOME("email home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME), + EMAIL_HOME("email home", null), DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED), //Ignoring or need more information to decide. STORAGE("storage", null), diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java index 356ac95230..b3cd172f4a 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -20,15 +20,7 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; import java.io.IOException; import java.nio.file.Path; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.time.temporal.TemporalAccessor; -import java.time.temporal.TemporalQueries; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -40,7 +32,6 @@ import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.Blackboard.BlackboardException; -import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.SleuthkitCase; @@ -59,14 +50,6 @@ final class XRYMessagesFileParser implements XRYFileParser { private static final String PARSER_NAME = "XRY DSP"; - //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() - //function for more details. - private static final DateTimeFormatter DATE_TIME_PARSER - = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); - - private static final String DEVICE_LOCALE = "(device)"; - private static final String NETWORK_LOCALE = "(network)"; - /** * All of the known XRY keys for message reports and their corresponding * blackboard attribute types, if any. @@ -78,7 +61,7 @@ final class XRYMessagesFileParser implements XRYFileParser { NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON), TEXT("text", null), TIME("time", null), - SERVICE_CENTER("service center", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER), + SERVICE_CENTER("service center", null), FROM("from", null), TO("to", null), //The following keys either need special processing or more time and data to find a type. @@ -314,6 +297,10 @@ final class XRYMessagesFileParser implements XRYFileParser { switch (key) { case TEL: case NUMBER: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + // Apply namespace or direction if(namespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) { senderId = pair.getValue(); @@ -330,15 +317,23 @@ final class XRYMessagesFileParser implements XRYFileParser { // Although confusing, as these are also 'name spaces', it appears // later versions of XRY just made these standardized lines. case FROM: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + senderId = pair.getValue(); break; case TO: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + recipientIdsList.add(pair.getValue()); break; case TIME: try { //Tranform value to seconds since epoch - long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(pair.getValue()); + long dateTimeSinceInEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue()); dateTime = dateTimeSinceInEpoch; } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[%s] Assumption" @@ -404,6 +399,14 @@ final class XRYMessagesFileParser implements XRYFileParser { break; } break; + case SERVICE_CENTER: + if(!XRYUtils.isPhoneValid(pair.getValue())) { + continue; + } + + otherAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, + PARSER_NAME, pair.getValue())); + break; default: //Otherwise, the XryKey enum contains the correct BlackboardAttribute //type. @@ -663,89 +666,4 @@ final class XRYMessagesFileParser implements XRYFileParser { return Optional.empty(); } - - /** - * Removes the locale from the date time value. - * - * Locale in this case being (Device) or (Network). - * - * @param dateTime XRY datetime value to be sanitized. - * @return A purer date time value. - */ - private String removeDateTimeLocale(String dateTime) { - String result = dateTime; - int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE); - if (deviceIndex != -1) { - result = result.substring(0, deviceIndex); - } - int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); - if (networkIndex != -1) { - result = result.substring(0, networkIndex); - } - return result; - } - - /** - * Parses the date time value and calculates seconds since epoch. - * - * @param dateTime - * @return - */ - private long calculateSecondsSinceEpoch(String dateTime) { - String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim(); - /** - * The format of time in XRY Messages reports is of the form: - * - * 1/3/1990 1:23:54 AM UTC+4 - * - * In our current version of Java (openjdk-1.8.0.222), there is a bug - * with having the timezone offset (UTC+4 or GMT-7) at the end of the - * date time input. This is fixed in later versions of the JDK (9 and - * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than - * update the JDK to accommodate this, the components of the date time - * string are reversed: - * - * UTC+4 AM 1:23:54 1/3/1990 - * - * The java time package will correctly parse this date time format. - */ - String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale); - /** - * Furthermore, the DateTimeFormatter's timezone offset letter ('O') - * does not recognize UTC but recognizes GMT. According to - * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only - * differs from UTC by at most 1 second and so substitution will only - * introduce a trivial amount of error. - */ - String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); - TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, - ZonedDateTime::from, - LocalDateTime::from, - OffsetDateTime::from); - //Query for the ZoneID - if (result.query(TemporalQueries.zoneId()) == null) { - //If none, assumed GMT+0. - return ZonedDateTime.of(LocalDateTime.from(result), - ZoneId.of("GMT")).toEpochSecond(); - } else { - return Instant.from(result).getEpochSecond(); - } - } - - /** - * Reverses the order of the date time components. - * - * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990 - * - * @param dateTime - * @return - */ - private String reverseOrderOfDateTimeComponents(String dateTime) { - StringBuilder reversedDateTime = new StringBuilder(dateTime.length()); - String[] dateTimeComponents = dateTime.split(" "); - for (String component : dateTimeComponents) { - reversedDateTime.insert(0, " ").insert(0, component); - } - return reversedDateTime.toString().trim(); - } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYUtils.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYUtils.java new file mode 100755 index 0000000000..8264d73a82 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYUtils.java @@ -0,0 +1,151 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019-2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; +import org.sleuthkit.datamodel.CommunicationsUtils; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Common utility methods shared among all XRY parser implementations. + */ +public final class XRYUtils { + + // Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() + // function for more details. + private static final DateTimeFormatter DATE_TIME_PARSER + = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); + + private static final String DEVICE_LOCALE = "(device)"; + private static final String NETWORK_LOCALE = "(network)"; + + public static boolean isPhoneValid(String phoneNumber) { + try { + CommunicationsUtils.normalizePhoneNum(phoneNumber); + return true; + } catch (TskCoreException ex) { + return false; + } + } + + public static boolean isEmailValid(String email) { + try { + CommunicationsUtils.normalizeEmailAddress(email); + return true; + } catch (TskCoreException ex) { + return false; + } + } + + /** + * Parses the date time value and calculates seconds since epoch. + * + * @param dateTime + * @return + */ + public static long calculateSecondsSinceEpoch(String dateTime) { + String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim(); + /** + * The format of time in XRY reports is of the form: + * + * 1/3/1990 1:23:54 AM UTC+4 + * + * In our current version of Java (openjdk-1.8.0.222), there is a bug + * with having the timezone offset (UTC+4 or GMT-7, for example) at the + * end of the date time input. This is fixed in later versions of the + * JDK (9 and beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 + * Rather than update the JDK to accommodate this, the components of the + * date time string are reversed: + * + * UTC+4 AM 1:23:54 1/3/1990 + * + * The java time package will correctly parse this date time format. + */ + String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale); + /** + * Furthermore, the DateTimeFormatter's timezone offset letter ('O') + * does not recognize UTC but recognizes GMT. According to + * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only + * differs from UTC by at most 1 second and so substitution will only + * introduce a trivial amount of error. + */ + String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); + TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, + ZonedDateTime::from, + LocalDateTime::from, + OffsetDateTime::from); + //Query for the ZoneID + if (result.query(TemporalQueries.zoneId()) == null) { + //If none, assumed GMT+0. + return ZonedDateTime.of(LocalDateTime.from(result), + ZoneId.of("GMT")).toEpochSecond(); + } else { + return Instant.from(result).getEpochSecond(); + } + } + + /** + * Reverses the order of the date time components. + * + * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990 + * + * @param dateTime + * @return + */ + private static String reverseOrderOfDateTimeComponents(String dateTime) { + StringBuilder reversedDateTime = new StringBuilder(dateTime.length()); + String[] dateTimeComponents = dateTime.split(" "); + for (String component : dateTimeComponents) { + reversedDateTime.insert(0, " ").insert(0, component); + } + return reversedDateTime.toString().trim(); + } + + /** + * Removes the locale from the date time value. + * + * Locale in this case being (Device) or (Network). + * + * @param dateTime XRY datetime value to be sanitized. + * @return A purer date time value. + */ + private static String removeDateTimeLocale(String dateTime) { + String result = dateTime; + int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE); + if (deviceIndex != -1) { + result = result.substring(0, deviceIndex); + } + int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); + if (networkIndex != -1) { + result = result.substring(0, networkIndex); + } + return result; + } + + private XRYUtils() { + + } +}