mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 18:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updated the XRY parser implementations to do some phone and email validation using CommunicationUtils. Created an XRYUtils to contain all the shared methods

Browse Source
This commit is contained in:
U-BASIS\dsmyda 2020-03-26 17:33:54 -04:00
parent 57efb88a6c
commit 88a58a6083
4 changed files with 209 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java
View File

@ -18,15 +18,7 @@
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -50,14 +42,6 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName());
//Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
//function for more details.
private static final DateTimeFormatter DATE_TIME_PARSER
= DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
private static final String DEVICE_LOCALE = "(device)";
private static final String NETWORK_LOCALE = "(network)";
/**
* All of the known XRY keys for call reports and their corresponding
* blackboard attribute types, if any.
@ -202,6 +186,10 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
switch (xryKey) {
case TEL:
case NUMBER:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
// Apply namespace or direction
if (xryNamespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) {
callerId = pair.getValue();
@ -216,15 +204,23 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
// Although confusing, as these are also 'name spaces', it appears
// later versions of XRY just made these standardized lines.
case TO:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
calleeList.add(pair.getValue());
break;
case FROM:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
callerId = pair.getValue();
break;
case TIME:
try {
//Tranform value to seconds since epoch
long dateTimeSinceEpoch = calculateSecondsSinceEpoch(pair.getValue());
long dateTimeSinceEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue());
startTime = dateTimeSinceEpoch;
} catch (DateTimeParseException ex) {
logger.log(Level.WARNING, String.format("[XRY DSP] Assumption"
@ -322,89 +318,4 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
endTime, callType, otherAttributes);
}
}
/**
* Removes the locale from the date time value.
*
* Locale in this case being (Device) or (Network).
*
* @param dateTime XRY datetime value to be sanitized.
* @return A purer date time value.
*/
private String removeDateTimeLocale(String dateTime) {
String result = dateTime;
int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
if (deviceIndex != -1) {
result = result.substring(0, deviceIndex);
}
int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
if (networkIndex != -1) {
result = result.substring(0, networkIndex);
}
return result;
}
/**
* Parses the date time value and calculates seconds since epoch.
*
* @param dateTime
* @return
*/
private long calculateSecondsSinceEpoch(String dateTime) {
String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
/**
* The format of time in XRY Messages reports is of the form:
*
* 1/3/1990 1:23:54 AM UTC+4
*
* In our current version of Java (openjdk-1.8.0.222), there is a bug
* with having the timezone offset (UTC+4 or GMT-7) at the end of the
* date time input. This is fixed in later versions of the JDK (9 and
* beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than
* update the JDK to accommodate this, the components of the date time
* string are reversed:
*
* UTC+4 AM 1:23:54 1/3/1990
*
* The java time package will correctly parse this date time format.
*/
String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
/**
* Furthermore, the DateTimeFormatter's timezone offset letter ('O')
* does not recognize UTC but recognizes GMT. According to
* https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only
* differs from UTC by at most 1 second and so substitution will only
* introduce a trivial amount of error.
*/
String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
ZonedDateTime::from,
LocalDateTime::from,
OffsetDateTime::from);
//Query for the ZoneID
if (result.query(TemporalQueries.zoneId()) == null) {
//If none, assumed GMT+0.
return ZonedDateTime.of(LocalDateTime.from(result),
ZoneId.of("GMT")).toEpochSecond();
} else {
return Instant.from(result).getEpochSecond();
}
}
/**
* Reverses the order of the date time components.
*
* Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990
*
* @param dateTime
* @return
*/
private String reverseOrderOfDateTimeComponents(String dateTime) {
StringBuilder reversedDateTime = new StringBuilder(dateTime.length());
String[] dateTimeComponents = dateTime.split(" ");
for (String component : dateTimeComponents) {
reversedDateTime.insert(0, " ").insert(0, component);
}
return reversedDateTime.toString().trim();
}
}

28
Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java
View File

@ -74,6 +74,10 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
}
break;
case TEL:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
if(phoneNumber != null) {
additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, pair.getValue()));
} else {
@ -81,6 +85,10 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
}
break;
case MOBILE:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
if(mobilePhoneNumber != null) {
additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_MOBILE, PARSER_NAME, pair.getValue()));
} else {
@ -88,22 +96,32 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
}
break;
case HOME:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
if(homePhoneNumber != null) {
additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_HOME, PARSER_NAME, pair.getValue()));
} else {
homePhoneNumber = pair.getValue();
}
break;
case EMAIL_HOME:
if(!XRYUtils.isEmailValid(pair.getValue())) {
continue;
}
hasAnEmail = true;
additionalAttributes.add(new BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME,
PARSER_NAME, pair.getValue()));
break;
default:
//Otherwise, the XryKey enum contains the correct BlackboardAttribute
//type.
if (xryKey.getType() != null) {
additionalAttributes.add(new BlackboardAttribute(xryKey.getType(),
PARSER_NAME, pair.getValue()));
if(xryKey == XryKey.EMAIL_HOME) {
hasAnEmail = true;
}
}
logger.log(Level.INFO, String.format("[XRY DSP] Key value pair "
@ -139,7 +157,7 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
HOME("home", null),
RELATED_APPLICATION("related application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME),
ADDRESS_HOME("address home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LOCATION),
EMAIL_HOME("email home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME),
EMAIL_HOME("email home", null),
DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED),
//Ignoring or need more information to decide.
STORAGE("storage", null),

126
Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java
View File

@ -20,15 +20,7 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.io.IOException;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@ -40,7 +32,6 @@ import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.Blackboard.BlackboardException;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
@ -59,14 +50,6 @@ final class XRYMessagesFileParser implements XRYFileParser {
private static final String PARSER_NAME = "XRY DSP";
//Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
//function for more details.
private static final DateTimeFormatter DATE_TIME_PARSER
= DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
private static final String DEVICE_LOCALE = "(device)";
private static final String NETWORK_LOCALE = "(network)";
/**
* All of the known XRY keys for message reports and their corresponding
* blackboard attribute types, if any.
@ -78,7 +61,7 @@ final class XRYMessagesFileParser implements XRYFileParser {
NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON),
TEXT("text", null),
TIME("time", null),
SERVICE_CENTER("service center", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER),
SERVICE_CENTER("service center", null),
FROM("from", null),
TO("to", null),
//The following keys either need special processing or more time and data to find a type.
@ -314,6 +297,10 @@ final class XRYMessagesFileParser implements XRYFileParser {
switch (key) {
case TEL:
case NUMBER:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
// Apply namespace or direction
if(namespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) {
senderId = pair.getValue();
@ -330,15 +317,23 @@ final class XRYMessagesFileParser implements XRYFileParser {
// Although confusing, as these are also 'name spaces', it appears
// later versions of XRY just made these standardized lines.
case FROM:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
senderId = pair.getValue();
break;
case TO:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
recipientIdsList.add(pair.getValue());
break;
case TIME:
try {
//Tranform value to seconds since epoch
long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(pair.getValue());
long dateTimeSinceInEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue());
dateTime = dateTimeSinceInEpoch;
} catch (DateTimeParseException ex) {
logger.log(Level.WARNING, String.format("[%s] Assumption"
@ -404,6 +399,14 @@ final class XRYMessagesFileParser implements XRYFileParser {
break;
}
break;
case SERVICE_CENTER:
if(!XRYUtils.isPhoneValid(pair.getValue())) {
continue;
}
otherAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER,
PARSER_NAME, pair.getValue()));
break;
default:
//Otherwise, the XryKey enum contains the correct BlackboardAttribute
//type.
@ -663,89 +666,4 @@ final class XRYMessagesFileParser implements XRYFileParser {
return Optional.empty();
}
/**
* Removes the locale from the date time value.
*
* Locale in this case being (Device) or (Network).
*
* @param dateTime XRY datetime value to be sanitized.
* @return A purer date time value.
*/
private String removeDateTimeLocale(String dateTime) {
String result = dateTime;
int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
if (deviceIndex != -1) {
result = result.substring(0, deviceIndex);
}
int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
if (networkIndex != -1) {
result = result.substring(0, networkIndex);
}
return result;
}
/**
* Parses the date time value and calculates seconds since epoch.
*
* @param dateTime
* @return
*/
private long calculateSecondsSinceEpoch(String dateTime) {
String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
/**
* The format of time in XRY Messages reports is of the form:
*
* 1/3/1990 1:23:54 AM UTC+4
*
* In our current version of Java (openjdk-1.8.0.222), there is a bug
* with having the timezone offset (UTC+4 or GMT-7) at the end of the
* date time input. This is fixed in later versions of the JDK (9 and
* beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than
* update the JDK to accommodate this, the components of the date time
* string are reversed:
*
* UTC+4 AM 1:23:54 1/3/1990
*
* The java time package will correctly parse this date time format.
*/
String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
/**
* Furthermore, the DateTimeFormatter's timezone offset letter ('O')
* does not recognize UTC but recognizes GMT. According to
* https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only
* differs from UTC by at most 1 second and so substitution will only
* introduce a trivial amount of error.
*/
String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
ZonedDateTime::from,
LocalDateTime::from,
OffsetDateTime::from);
//Query for the ZoneID
if (result.query(TemporalQueries.zoneId()) == null) {
//If none, assumed GMT+0.
return ZonedDateTime.of(LocalDateTime.from(result),
ZoneId.of("GMT")).toEpochSecond();
} else {
return Instant.from(result).getEpochSecond();
}
}
/**
* Reverses the order of the date time components.
*
* Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990
*
* @param dateTime
* @return
*/
private String reverseOrderOfDateTimeComponents(String dateTime) {
StringBuilder reversedDateTime = new StringBuilder(dateTime.length());
String[] dateTimeComponents = dateTime.split(" ");
for (String component : dateTimeComponents) {
reversedDateTime.insert(0, " ").insert(0, component);
}
return reversedDateTime.toString().trim();
}
}

151
Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYUtils.java Executable file
View File

@ -0,0 +1,151 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019-2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.datasourceprocessors.xry;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
import org.sleuthkit.datamodel.CommunicationsUtils;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Common utility methods shared among all XRY parser implementations.
*/
public final class XRYUtils {
// Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
// function for more details.
private static final DateTimeFormatter DATE_TIME_PARSER
= DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
private static final String DEVICE_LOCALE = "(device)";
private static final String NETWORK_LOCALE = "(network)";
public static boolean isPhoneValid(String phoneNumber) {
try {
CommunicationsUtils.normalizePhoneNum(phoneNumber);
return true;
} catch (TskCoreException ex) {
return false;
}
}
public static boolean isEmailValid(String email) {
try {
CommunicationsUtils.normalizeEmailAddress(email);
return true;
} catch (TskCoreException ex) {
return false;
}
}
/**
* Parses the date time value and calculates seconds since epoch.
*
* @param dateTime
* @return
*/
public static long calculateSecondsSinceEpoch(String dateTime) {
String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
/**
* The format of time in XRY reports is of the form:
*
* 1/3/1990 1:23:54 AM UTC+4
*
* In our current version of Java (openjdk-1.8.0.222), there is a bug
* with having the timezone offset (UTC+4 or GMT-7, for example) at the
* end of the date time input. This is fixed in later versions of the
* JDK (9 and beyond). https://bugs.openjdk.java.net/browse/JDK-8154050
* Rather than update the JDK to accommodate this, the components of the
* date time string are reversed:
*
* UTC+4 AM 1:23:54 1/3/1990
*
* The java time package will correctly parse this date time format.
*/
String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
/**
* Furthermore, the DateTimeFormatter's timezone offset letter ('O')
* does not recognize UTC but recognizes GMT. According to
* https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only
* differs from UTC by at most 1 second and so substitution will only
* introduce a trivial amount of error.
*/
String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
ZonedDateTime::from,
LocalDateTime::from,
OffsetDateTime::from);
//Query for the ZoneID
if (result.query(TemporalQueries.zoneId()) == null) {
//If none, assumed GMT+0.
return ZonedDateTime.of(LocalDateTime.from(result),
ZoneId.of("GMT")).toEpochSecond();
} else {
return Instant.from(result).getEpochSecond();
}
}
/**
* Reverses the order of the date time components.
*
* Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990
*
* @param dateTime
* @return
*/
private static String reverseOrderOfDateTimeComponents(String dateTime) {
StringBuilder reversedDateTime = new StringBuilder(dateTime.length());
String[] dateTimeComponents = dateTime.split(" ");
for (String component : dateTimeComponents) {
reversedDateTime.insert(0, " ").insert(0, component);
}
return reversedDateTime.toString().trim();
}
/**
* Removes the locale from the date time value.
*
* Locale in this case being (Device) or (Network).
*
* @param dateTime XRY datetime value to be sanitized.
* @return A purer date time value.
*/
private static String removeDateTimeLocale(String dateTime) {
String result = dateTime;
int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
if (deviceIndex != -1) {
result = result.substring(0, deviceIndex);
}
int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
if (networkIndex != -1) {
result = result.substring(0, networkIndex);
}
return result;
}
private XRYUtils() {
}
}