6172: Validation of correlation attribute value not happening correctly

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java

@@ -23,6 +23,8 @@ import java.util.List;
 import java.util.Optional;
 import org.apache.commons.validator.routines.DomainValidator;
 import org.apache.commons.validator.routines.EmailValidator;
+import org.sleuthkit.datamodel.CommunicationsUtils;
+import org.sleuthkit.datamodel.TskCoreException;
 
 /**
  * Provides functions for normalizing data by attribute type before insertion or
@@ -152,26 +154,25 @@ final public class CorrelationAttributeNormalizer {
     }
 
     /**
-     * Verify that there is an '@' and no invalid characters. Should normalize
+     * Verify and normalize email address.
-     * to lower case.
      */
     private static String normalizeEmail(String data) throws CorrelationAttributeNormalizationException {
-        EmailValidator validator = EmailValidator.getInstance(true, true);
+       try {
-        if (validator.isValid(data)) {
+           return CommunicationsUtils.normalizeEmailAddress(data);
-            return data.toLowerCase();
+       }
-        } else {
+       catch(TskCoreException ex) {
-            throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid email address: %s", data));
+           throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid email address: %s", data), ex);
-        }
+       }
     }
 
     /**
-     * Verify it is only numbers and '+'. Strip spaces, dashes, and parentheses.
+     * Verify and normalize phone number.
      */
     private static String normalizePhone(String data) throws CorrelationAttributeNormalizationException {
-        if (data.matches("\\+?[0-9()\\-\\s]+")) {
+        try {
-            String phoneNumber = data.replaceAll("[^0-9\\+]", "");
+             return CommunicationsUtils.normalizePhoneNum(data);
-            return phoneNumber;
+        }
-        } else {
+        catch(TskCoreException ex) {
             throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid phone number: %s", data));
         }
     }

Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java

@@ -33,6 +33,7 @@ import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment;
 import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments;
+import org.sleuthkit.datamodel.CommunicationsUtils;
 
 /**
  * 
@@ -97,20 +98,27 @@ class AccountSummary {
                             
                             boolean isReference = false;
                             
-                            for (BlackboardAttribute attribute: attributes) {
+                            for (BlackboardAttribute attribute : attributes) {
+
                                 String attributeTypeName = attribute.getAttributeType().getTypeName();
                                 String attributeValue = attribute.getValueString();
+                                try {
+                                    if (attributeTypeName.contains("PHONE")) {
+                                        attributeValue = CommunicationsUtils.normalizePhoneNum(attributeValue);
+                                    } else if (attributeTypeName.contains("EMAIL")) {
+                                        attributeValue = CommunicationsUtils.normalizeEmailAddress(attributeValue);
+                                    }
                                     
-                                if (attributeTypeName.contains("PHONE")) {
+                                    if (typeSpecificID.equals(attributeValue)) {
-                                    attributeValue = RelationshipsNodeUtilities.normalizePhoneNum(attributeValue);
+                                        isReference = true;
-                                } else if (attributeTypeName.contains("EMAIL")) {
+                                        break;
-                                    attributeValue = RelationshipsNodeUtilities.normalizeEmailAddress(attributeValue);
+                                    }
+                                } catch (TskCoreException ex) {
+                                    logger.log(Level.WARNING, String.format("Exception thrown "
+                                            + "in trying to normalize attribute value: %s",
+                                            attributeValue), ex); //NON-NLS
                                 }
 
-                                if ( typeSpecificID.equals(attributeValue) ) {
-                                    isReference = true;
-                                    break;
-                                }
                             }
                             if (isReference) {
                                 referenceCnt++;

Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java

@@ -69,42 +69,4 @@ final class RelationshipsNodeUtilities {
         }
     }
     
-    	/**
-	 * Normalize the phone number by removing all non numeric characters, except
-	 * for leading +.
-         * 
-         * This function copied from CommunicationManager.
-	 *
-	 * @param phoneNum The phone number to normalize
-	 *
-	 * @return The normalized phone number.
-	 */
-    static String normalizePhoneNum(String phoneNum) {
-        String normailzedPhoneNum = phoneNum.replaceAll("\\D", "");
-
-        if (phoneNum.startsWith("+")) {
-            normailzedPhoneNum = "+" + normailzedPhoneNum;
-        }
-
-        if (normailzedPhoneNum.isEmpty()) {
-            normailzedPhoneNum = phoneNum;
-        }
-
-        return normailzedPhoneNum;
-    }
-
-    /**
-     * Normalize the given email address by converting it to lowercase.
-     * 
-     * This function copied from CommunicationManager.
-     *
-     * @param emailAddress The email address tot normalize
-     *
-     * @return The normalized email address.
-     */
-    static String normalizeEmailAddress(String emailAddress) {
-        String normailzedEmailAddr = emailAddress.toLowerCase();
-
-        return normailzedEmailAddr;
-    }
 }

InternalPythonModules/android/calllog.py

@@ -105,6 +105,9 @@ class CallLogAnalyzer(general.AndroidComponentAnalyzer):
                                         timeStamp = resultSet.getLong("date") / 1000
                                          
                                         number = resultSet.getString("number")
+                                        if not general.isValidPhoneNumer(number):
+                                            number = None
+                                            
                                         duration = resultSet.getLong("duration") # duration of call is in seconds
                                         name = resultSet.getString("name") # name of person dialed or called. None if unregistered
 

InternalPythonModules/android/general.py

@@ -1,7 +1,7 @@
 """
 Autopsy Forensic Browser
 
-Copyright 2016 Basis Technology Corp.
+Copyright 2016-2020 Basis Technology Corp.
 Contact: carrier <at> sleuthkit <dot> org
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,8 +15,11 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
 """
 
+import re
+
 MODULE_NAME = "Android Analyzer"
 
 """
@@ -37,3 +40,15 @@ def appendAttachmentList(msgBody, attachmentsList):
         body = body + "\n".join(list(filter(None, attachmentsList)))
 
     return body
+
+"""
+Checks if the given string might be a phone number.
+"""
+def isValidPhoneNumer(data):
+    return bool(re.match(r"^\+?[0-9()\-\s]+$", data))
+
+"""
+Checks if the given string is a valid email address.
+"""
+def isValidEmailAddress(data):
+    return bool(re.match(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", data))

InternalPythonModules/android/textnow.py

@@ -109,13 +109,21 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer):
         try:
             contacts_parser = TextNowContactsParser(textnow_db)
             while contacts_parser.next():
-                helper.addContact( 
+                name = contacts_parser.get_contact_name()
-                    contacts_parser.get_contact_name(), 
+                phone = contacts_parser.get_phone()
-                    contacts_parser.get_phone(),
+                home_phone = contacts_parser.get_home_phone()
-                    contacts_parser.get_home_phone(),
+                mobile_phone = contacts_parser.get_mobile_phone()
-                    contacts_parser.get_mobile_phone(),
+                email = contacts_parser.get_email()
-                    contacts_parser.get_email()
+
-                )
+                # add contact if we have at least one valid phone/email
+                if phone or home_phone or mobile_phone or email:
+                    helper.addContact( 
+                        name, 
+                        phone,
+                        home_phone,
+                        mobile_phone,
+                        email
+                    )
             contacts_parser.close()
         except SQLException as ex:
             #Error parsing TextNow db
@@ -277,7 +285,13 @@ class TextNowContactsParser(TskContactsParser):
         return self.result_set.getString("name")
     
     def get_phone(self):
-        return self.result_set.getString("number")
+        number = self.result_set.getString("number")
+        return (number if general.isValidPhoneNumer(number) else None)
+        
+    def get_email(self):
+        # occasionally the 'number' column may have an email address instead
+        value = self.result_set.getString("number")
+        return (value if general.isValidEmailAddress(value) else None)
 
 class TextNowMessagesParser(TskMessagesParser):
     """

InternalPythonModules/android/whatsapp.py

@@ -172,14 +172,22 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer):
         try:
             contacts_parser = WhatsAppContactsParser(contacts_db, self._PARSER_NAME)
             while contacts_parser.next():
-                helper.addContact( 
+                name = contacts_parser.get_contact_name()
-                    contacts_parser.get_contact_name(), 
+                phone = contacts_parser.get_phone()
-                    contacts_parser.get_phone(),
+                home_phone = contacts_parser.get_home_phone()
-                    contacts_parser.get_home_phone(),
+                mobile_phone = contacts_parser.get_mobile_phone()
-                    contacts_parser.get_mobile_phone(),
+                email = contacts_parser.get_email()
-                    contacts_parser.get_email(),
+
-                    contacts_parser.get_other_attributes()
+                # add contact if we have at least one valid phone/email
-                )
+                if phone or home_phone or mobile_phone or email:
+                    helper.addContact( 
+                        name, 
+                        phone,
+                        home_phone,
+                        mobile_phone,
+                        email,
+                        contacts_parser.get_other_attributes()
+                    )
             contacts_parser.close()
         except SQLException as ex:
             self._logger.log(Level.WARNING, "Error querying the whatsapp database for contacts.", ex)
@@ -426,7 +434,13 @@ class WhatsAppContactsParser(TskContactsParser):
         return self.result_set.getString("name")
 
     def get_phone(self):
-        return self.result_set.getString("number")
+        number = self.result_set.getString("number")
+        return (number if general.isValidPhoneNumer(number) else None)
+
+    def get_email(self):
+        # occasionally the 'number' column may have an email address instead
+        value = self.result_set.getString("number")
+        return (value if general.isValidEmailAddress(value) else None)
         
     def get_other_attributes(self):
         return [BlackboardAttribute(