Updated the XRY parser implementations to do some phone and email validation using CommunicationUtils. Created an XRYUtils to contain all the shared methods

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java

@@ -18,15 +18,7 @@
  */
 package org.sleuthkit.autopsy.datasourceprocessors.xry;
 
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.OffsetDateTime;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
-import java.time.temporal.TemporalAccessor;
-import java.time.temporal.TemporalQueries;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -50,14 +42,6 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
 
     private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName());
 
-    //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
-    //function for more details.
-    private static final DateTimeFormatter DATE_TIME_PARSER
-            = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
-
-    private static final String DEVICE_LOCALE = "(device)";
-    private static final String NETWORK_LOCALE = "(network)";
-
     /**
      * All of the known XRY keys for call reports and their corresponding
      * blackboard attribute types, if any.
@@ -202,6 +186,10 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
             switch (xryKey) {
                 case TEL:
                 case NUMBER:
+                    if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                        continue;
+                    }
+                    
                     // Apply namespace or direction
                     if (xryNamespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) {
                         callerId = pair.getValue();
@@ -216,15 +204,23 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
                 // Although confusing, as these are also 'name spaces', it appears
                 // later versions of XRY just made these standardized lines.
                 case TO:
+                    if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                        continue;
+                    }
+                    
                     calleeList.add(pair.getValue());
                     break;
                 case FROM:
+                    if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                        continue;
+                    }
+                    
                     callerId = pair.getValue();
                     break;
                 case TIME:
                     try {
                         //Tranform value to seconds since epoch
-                        long dateTimeSinceEpoch = calculateSecondsSinceEpoch(pair.getValue());
+                        long dateTimeSinceEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue());
                         startTime = dateTimeSinceEpoch;
                     } catch (DateTimeParseException ex) {
                         logger.log(Level.WARNING, String.format("[XRY DSP] Assumption"
@@ -322,89 +318,4 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser {
                     endTime, callType, otherAttributes);
         }
     }
-
-    /**
-     * Removes the locale from the date time value.
-     *
-     * Locale in this case being (Device) or (Network).
-     *
-     * @param dateTime XRY datetime value to be sanitized.
-     * @return A purer date time value.
-     */
-    private String removeDateTimeLocale(String dateTime) {
-        String result = dateTime;
-        int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
-        if (deviceIndex != -1) {
-            result = result.substring(0, deviceIndex);
-        }
-        int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
-        if (networkIndex != -1) {
-            result = result.substring(0, networkIndex);
-        }
-        return result;
-    }
-
-    /**
-     * Parses the date time value and calculates seconds since epoch.
-     *
-     * @param dateTime
-     * @return
-     */
-    private long calculateSecondsSinceEpoch(String dateTime) {
-        String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
-        /**
-         * The format of time in XRY Messages reports is of the form:
-         *
-         * 1/3/1990 1:23:54 AM UTC+4
-         *
-         * In our current version of Java (openjdk-1.8.0.222), there is a bug
-         * with having the timezone offset (UTC+4 or GMT-7) at the end of the
-         * date time input. This is fixed in later versions of the JDK (9 and
-         * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than
-         * update the JDK to accommodate this, the components of the date time
-         * string are reversed:
-         *
-         * UTC+4 AM 1:23:54 1/3/1990
-         *
-         * The java time package will correctly parse this date time format.
-         */
-        String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
-        /**
-         * Furthermore, the DateTimeFormatter's timezone offset letter ('O')
-         * does not recognize UTC but recognizes GMT. According to
-         * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only
-         * differs from UTC by at most 1 second and so substitution will only
-         * introduce a trivial amount of error.
-         */
-        String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
-        TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
-                ZonedDateTime::from,
-                LocalDateTime::from,
-                OffsetDateTime::from);
-        //Query for the ZoneID
-        if (result.query(TemporalQueries.zoneId()) == null) {
-            //If none, assumed GMT+0.
-            return ZonedDateTime.of(LocalDateTime.from(result),
-                    ZoneId.of("GMT")).toEpochSecond();
-        } else {
-            return Instant.from(result).getEpochSecond();
-        }
-    }
-
-    /**
-     * Reverses the order of the date time components.
-     *
-     * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990
-     *
-     * @param dateTime
-     * @return
-     */
-    private String reverseOrderOfDateTimeComponents(String dateTime) {
-        StringBuilder reversedDateTime = new StringBuilder(dateTime.length());
-        String[] dateTimeComponents = dateTime.split(" ");
-        for (String component : dateTimeComponents) {
-            reversedDateTime.insert(0, " ").insert(0, component);
-        }
-        return reversedDateTime.toString().trim();
-    }
 }

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java

@@ -74,6 +74,10 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
                     }
                     break;
                 case TEL:
+                    if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                        continue;
+                    }
+                    
                     if(phoneNumber != null) {
                         additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, pair.getValue()));
                     } else {
@@ -81,6 +85,10 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
                     }
                     break;
                 case MOBILE:
+                    if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                        continue;
+                    }
+                    
                     if(mobilePhoneNumber != null) {
                         additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_MOBILE, PARSER_NAME, pair.getValue()));
                     } else {
@@ -88,22 +96,32 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
                     }
                     break;
                 case HOME:
+                    if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                        continue;
+                    }
+                    
                     if(homePhoneNumber != null) {
                         additionalAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_HOME, PARSER_NAME, pair.getValue()));
                     } else {
                         homePhoneNumber = pair.getValue();
                     }
                     break;
+                case EMAIL_HOME:
+                    if(!XRYUtils.isEmailValid(pair.getValue())) {
+                        continue;
+                    }
+                    
+                    hasAnEmail = true;
+                    additionalAttributes.add(new BlackboardAttribute(
+                            BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME,
+                            PARSER_NAME, pair.getValue()));
+                    break;
                 default:
                     //Otherwise, the XryKey enum contains the correct BlackboardAttribute
                     //type.
                     if (xryKey.getType() != null) {
                         additionalAttributes.add(new BlackboardAttribute(xryKey.getType(),
                                 PARSER_NAME, pair.getValue()));
-                                           
-                        if(xryKey == XryKey.EMAIL_HOME) {
-                            hasAnEmail = true;
-                        }
                     }
 
                     logger.log(Level.INFO, String.format("[XRY DSP] Key value pair "
@@ -139,7 +157,7 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser {
         HOME("home", null),
         RELATED_APPLICATION("related application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME),
         ADDRESS_HOME("address home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LOCATION),
-        EMAIL_HOME("email home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME),
+        EMAIL_HOME("email home", null),
         DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED),
         //Ignoring or need more information to decide.
         STORAGE("storage", null),

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java

@@ -20,15 +20,7 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry;
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.OffsetDateTime;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
-import java.time.temporal.TemporalAccessor;
-import java.time.temporal.TemporalQueries;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -40,7 +32,6 @@ import java.util.logging.Level;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.datamodel.Account;
 import org.sleuthkit.datamodel.Blackboard.BlackboardException;
-import org.sleuthkit.datamodel.BlackboardArtifact;
 import org.sleuthkit.datamodel.BlackboardAttribute;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.SleuthkitCase;
@@ -59,14 +50,6 @@ final class XRYMessagesFileParser implements XRYFileParser {
 
     private static final String PARSER_NAME = "XRY DSP";
 
-    //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
-    //function for more details.
-    private static final DateTimeFormatter DATE_TIME_PARSER
-            = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
-
-    private static final String DEVICE_LOCALE = "(device)";
-    private static final String NETWORK_LOCALE = "(network)";
-
     /**
      * All of the known XRY keys for message reports and their corresponding
      * blackboard attribute types, if any.
@@ -78,7 +61,7 @@ final class XRYMessagesFileParser implements XRYFileParser {
         NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON),
         TEXT("text", null),
         TIME("time", null),
-        SERVICE_CENTER("service center", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER),
+        SERVICE_CENTER("service center", null),
         FROM("from", null),
         TO("to", null),
         //The following keys either need special processing or more time and data to find a type.
@@ -314,6 +297,10 @@ final class XRYMessagesFileParser implements XRYFileParser {
                 switch (key) {
                     case TEL:
                     case NUMBER:
+                        if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                            continue;
+                        }
+                        
                         // Apply namespace or direction
                         if(namespace == XryNamespace.FROM || direction == CommunicationDirection.INCOMING) {
                             senderId = pair.getValue();
@@ -330,15 +317,23 @@ final class XRYMessagesFileParser implements XRYFileParser {
                     // Although confusing, as these are also 'name spaces', it appears
                     // later versions of XRY just made these standardized lines.
                     case FROM:
+                        if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                            continue;
+                        }
+                        
                         senderId = pair.getValue();
                         break;
                     case TO:
+                        if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                            continue;
+                        }
+                        
                         recipientIdsList.add(pair.getValue());
                         break;
                     case TIME:
                         try {
                             //Tranform value to seconds since epoch
-                            long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(pair.getValue());
+                            long dateTimeSinceInEpoch = XRYUtils.calculateSecondsSinceEpoch(pair.getValue());
                             dateTime = dateTimeSinceInEpoch;
                         } catch (DateTimeParseException ex) {
                             logger.log(Level.WARNING, String.format("[%s] Assumption"
@@ -404,6 +399,14 @@ final class XRYMessagesFileParser implements XRYFileParser {
                                 break;
                         }
                         break;
+                    case SERVICE_CENTER:
+                        if(!XRYUtils.isPhoneValid(pair.getValue())) {
+                            continue;
+                        }
+                        
+                        otherAttributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, 
+                                PARSER_NAME, pair.getValue()));
+                        break;
                     default:
                         //Otherwise, the XryKey enum contains the correct BlackboardAttribute
                         //type.
@@ -663,89 +666,4 @@ final class XRYMessagesFileParser implements XRYFileParser {
 
         return Optional.empty();
     }
-    
-    /**
-     * Removes the locale from the date time value.
-     *
-     * Locale in this case being (Device) or (Network).
-     *
-     * @param dateTime XRY datetime value to be sanitized.
-     * @return A purer date time value.
-     */
-    private String removeDateTimeLocale(String dateTime) {
-        String result = dateTime;
-        int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
-        if (deviceIndex != -1) {
-            result = result.substring(0, deviceIndex);
-        }
-        int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
-        if (networkIndex != -1) {
-            result = result.substring(0, networkIndex);
-        }
-        return result;
-    }
-
-    /**
-     * Parses the date time value and calculates seconds since epoch.
-     *
-     * @param dateTime
-     * @return
-     */
-    private long calculateSecondsSinceEpoch(String dateTime) {
-        String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
-        /**
-         * The format of time in XRY Messages reports is of the form:
-         *
-         * 1/3/1990 1:23:54 AM UTC+4
-         *
-         * In our current version of Java (openjdk-1.8.0.222), there is a bug
-         * with having the timezone offset (UTC+4 or GMT-7) at the end of the
-         * date time input. This is fixed in later versions of the JDK (9 and
-         * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than
-         * update the JDK to accommodate this, the components of the date time
-         * string are reversed:
-         *
-         * UTC+4 AM 1:23:54 1/3/1990
-         *
-         * The java time package will correctly parse this date time format.
-         */
-        String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
-        /**
-         * Furthermore, the DateTimeFormatter's timezone offset letter ('O')
-         * does not recognize UTC but recognizes GMT. According to
-         * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only
-         * differs from UTC by at most 1 second and so substitution will only
-         * introduce a trivial amount of error.
-         */
-        String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
-        TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
-                ZonedDateTime::from,
-                LocalDateTime::from,
-                OffsetDateTime::from);
-        //Query for the ZoneID
-        if (result.query(TemporalQueries.zoneId()) == null) {
-            //If none, assumed GMT+0.
-            return ZonedDateTime.of(LocalDateTime.from(result),
-                    ZoneId.of("GMT")).toEpochSecond();
-        } else {
-            return Instant.from(result).getEpochSecond();
-        }
-    }
-
-    /**
-     * Reverses the order of the date time components.
-     *
-     * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990
-     *
-     * @param dateTime
-     * @return
-     */
-    private String reverseOrderOfDateTimeComponents(String dateTime) {
-        StringBuilder reversedDateTime = new StringBuilder(dateTime.length());
-        String[] dateTimeComponents = dateTime.split(" ");
-        for (String component : dateTimeComponents) {
-            reversedDateTime.insert(0, " ").insert(0, component);
-        }
-        return reversedDateTime.toString().trim();
-    }
 }

Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYUtils.java

@@ -0,0 +1,151 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2019-2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.datasourceprocessors.xry;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalQueries;
+import org.sleuthkit.datamodel.CommunicationsUtils;
+import org.sleuthkit.datamodel.TskCoreException;
+
+/**
+ * Common utility methods shared among all XRY parser implementations.
+ */
+public final class XRYUtils {
+
+    // Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch()
+    // function for more details.
+    private static final DateTimeFormatter DATE_TIME_PARSER
+            = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y");
+
+    private static final String DEVICE_LOCALE = "(device)";
+    private static final String NETWORK_LOCALE = "(network)";
+
+    public static boolean isPhoneValid(String phoneNumber) {
+        try {
+            CommunicationsUtils.normalizePhoneNum(phoneNumber);
+            return true;
+        } catch (TskCoreException ex) {
+            return false;
+        }
+    }
+
+    public static boolean isEmailValid(String email) {
+        try {
+            CommunicationsUtils.normalizeEmailAddress(email);
+            return true;
+        } catch (TskCoreException ex) {
+            return false;
+        }
+    }
+
+    /**
+     * Parses the date time value and calculates seconds since epoch.
+     *
+     * @param dateTime
+     * @return
+     */
+    public static long calculateSecondsSinceEpoch(String dateTime) {
+        String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim();
+        /**
+         * The format of time in XRY reports is of the form:
+         *
+         * 1/3/1990 1:23:54 AM UTC+4
+         *
+         * In our current version of Java (openjdk-1.8.0.222), there is a bug
+         * with having the timezone offset (UTC+4 or GMT-7, for example) at the
+         * end of the date time input. This is fixed in later versions of the
+         * JDK (9 and beyond). https://bugs.openjdk.java.net/browse/JDK-8154050
+         * Rather than update the JDK to accommodate this, the components of the
+         * date time string are reversed:
+         *
+         * UTC+4 AM 1:23:54 1/3/1990
+         *
+         * The java time package will correctly parse this date time format.
+         */
+        String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale);
+        /**
+         * Furthermore, the DateTimeFormatter's timezone offset letter ('O')
+         * does not recognize UTC but recognizes GMT. According to
+         * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only
+         * differs from UTC by at most 1 second and so substitution will only
+         * introduce a trivial amount of error.
+         */
+        String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT");
+        TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT,
+                ZonedDateTime::from,
+                LocalDateTime::from,
+                OffsetDateTime::from);
+        //Query for the ZoneID
+        if (result.query(TemporalQueries.zoneId()) == null) {
+            //If none, assumed GMT+0.
+            return ZonedDateTime.of(LocalDateTime.from(result),
+                    ZoneId.of("GMT")).toEpochSecond();
+        } else {
+            return Instant.from(result).getEpochSecond();
+        }
+    }
+
+    /**
+     * Reverses the order of the date time components.
+     *
+     * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990
+     *
+     * @param dateTime
+     * @return
+     */
+    private static String reverseOrderOfDateTimeComponents(String dateTime) {
+        StringBuilder reversedDateTime = new StringBuilder(dateTime.length());
+        String[] dateTimeComponents = dateTime.split(" ");
+        for (String component : dateTimeComponents) {
+            reversedDateTime.insert(0, " ").insert(0, component);
+        }
+        return reversedDateTime.toString().trim();
+    }
+
+    /**
+     * Removes the locale from the date time value.
+     *
+     * Locale in this case being (Device) or (Network).
+     *
+     * @param dateTime XRY datetime value to be sanitized.
+     * @return A purer date time value.
+     */
+    private static String removeDateTimeLocale(String dateTime) {
+        String result = dateTime;
+        int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE);
+        if (deviceIndex != -1) {
+            result = result.substring(0, deviceIndex);
+        }
+        int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE);
+        if (networkIndex != -1) {
+            result = result.substring(0, networkIndex);
+        }
+        return result;
+    }
+
+    private XRYUtils() {
+
+    }
+}