diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index f68dab484a..d762e74945 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Optional; import org.apache.commons.validator.routines.DomainValidator; import org.apache.commons.validator.routines.EmailValidator; +import org.sleuthkit.datamodel.CommunicationsUtils; +import org.sleuthkit.datamodel.TskCoreException; /** * Provides functions for normalizing data by attribute type before insertion or @@ -152,26 +154,25 @@ final public class CorrelationAttributeNormalizer { } /** - * Verify that there is an '@' and no invalid characters. Should normalize - * to lower case. + * Verify and normalize email address. */ private static String normalizeEmail(String data) throws CorrelationAttributeNormalizationException { - EmailValidator validator = EmailValidator.getInstance(true, true); - if (validator.isValid(data)) { - return data.toLowerCase(); - } else { - throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid email address: %s", data)); - } + try { + return CommunicationsUtils.normalizeEmailAddress(data); + } + catch(TskCoreException ex) { + throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid email address: %s", data), ex); + } } /** - * Verify it is only numbers and '+'. Strip spaces, dashes, and parentheses. + * Verify and normalize phone number. */ private static String normalizePhone(String data) throws CorrelationAttributeNormalizationException { - if (data.matches("\\+?[0-9()\\-\\s]+")) { - String phoneNumber = data.replaceAll("[^0-9\\+]", ""); - return phoneNumber; - } else { + try { + return CommunicationsUtils.normalizePhoneNum(data); + } + catch(TskCoreException ex) { throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid phone number: %s", data)); } } diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java index 0888a105a3..5401f1a839 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/AccountSummary.java @@ -33,6 +33,7 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.FileAttachment; import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments; +import org.sleuthkit.datamodel.CommunicationsUtils; /** * @@ -97,20 +98,27 @@ class AccountSummary { boolean isReference = false; - for (BlackboardAttribute attribute: attributes) { + for (BlackboardAttribute attribute : attributes) { + String attributeTypeName = attribute.getAttributeType().getTypeName(); String attributeValue = attribute.getValueString(); - - if (attributeTypeName.contains("PHONE")) { - attributeValue = RelationshipsNodeUtilities.normalizePhoneNum(attributeValue); - } else if (attributeTypeName.contains("EMAIL")) { - attributeValue = RelationshipsNodeUtilities.normalizeEmailAddress(attributeValue); - } - - if ( typeSpecificID.equals(attributeValue) ) { - isReference = true; - break; + try { + if (attributeTypeName.contains("PHONE")) { + attributeValue = CommunicationsUtils.normalizePhoneNum(attributeValue); + } else if (attributeTypeName.contains("EMAIL")) { + attributeValue = CommunicationsUtils.normalizeEmailAddress(attributeValue); + } + + if (typeSpecificID.equals(attributeValue)) { + isReference = true; + break; + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Exception thrown " + + "in trying to normalize attribute value: %s", + attributeValue), ex); //NON-NLS } + } if (isReference) { referenceCnt++; diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java index 252b4ee1ff..be150dd590 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipsNodeUtilities.java @@ -69,42 +69,4 @@ final class RelationshipsNodeUtilities { } } - /** - * Normalize the phone number by removing all non numeric characters, except - * for leading +. - * - * This function copied from CommunicationManager. - * - * @param phoneNum The phone number to normalize - * - * @return The normalized phone number. - */ - static String normalizePhoneNum(String phoneNum) { - String normailzedPhoneNum = phoneNum.replaceAll("\\D", ""); - - if (phoneNum.startsWith("+")) { - normailzedPhoneNum = "+" + normailzedPhoneNum; - } - - if (normailzedPhoneNum.isEmpty()) { - normailzedPhoneNum = phoneNum; - } - - return normailzedPhoneNum; - } - - /** - * Normalize the given email address by converting it to lowercase. - * - * This function copied from CommunicationManager. - * - * @param emailAddress The email address tot normalize - * - * @return The normalized email address. - */ - static String normalizeEmailAddress(String emailAddress) { - String normailzedEmailAddr = emailAddress.toLowerCase(); - - return normailzedEmailAddr; - } } diff --git a/InternalPythonModules/android/calllog.py b/InternalPythonModules/android/calllog.py index 71c5f112ad..5b3faf0697 100644 --- a/InternalPythonModules/android/calllog.py +++ b/InternalPythonModules/android/calllog.py @@ -105,6 +105,9 @@ class CallLogAnalyzer(general.AndroidComponentAnalyzer): timeStamp = resultSet.getLong("date") / 1000 number = resultSet.getString("number") + if not general.isValidPhoneNumer(number): + number = None + duration = resultSet.getLong("duration") # duration of call is in seconds name = resultSet.getString("name") # name of person dialed or called. None if unregistered diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index e8cb199b8d..f0dfb05dfd 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -1,7 +1,7 @@ """ Autopsy Forensic Browser -Copyright 2016 Basis Technology Corp. +Copyright 2016-2020 Basis Technology Corp. Contact: carrier sleuthkit org Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,11 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + """ +import re + MODULE_NAME = "Android Analyzer" """ @@ -37,3 +40,15 @@ def appendAttachmentList(msgBody, attachmentsList): body = body + "\n".join(list(filter(None, attachmentsList))) return body + +""" +Checks if the given string might be a phone number. +""" +def isValidPhoneNumer(data): + return bool(re.match(r"^\+?[0-9()\-\s]+$", data)) + +""" +Checks if the given string is a valid email address. +""" +def isValidEmailAddress(data): + return bool(re.match(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", data)) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index 49ab3325c2..e05763cae9 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -109,13 +109,21 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): try: contacts_parser = TextNowContactsParser(textnow_db) while contacts_parser.next(): - helper.addContact( - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) + name = contacts_parser.get_contact_name() + phone = contacts_parser.get_phone() + home_phone = contacts_parser.get_home_phone() + mobile_phone = contacts_parser.get_mobile_phone() + email = contacts_parser.get_email() + + # add contact if we have at least one valid phone/email + if phone or home_phone or mobile_phone or email: + helper.addContact( + name, + phone, + home_phone, + mobile_phone, + email + ) contacts_parser.close() except SQLException as ex: #Error parsing TextNow db @@ -277,7 +285,13 @@ class TextNowContactsParser(TskContactsParser): return self.result_set.getString("name") def get_phone(self): - return self.result_set.getString("number") + number = self.result_set.getString("number") + return (number if general.isValidPhoneNumer(number) else None) + + def get_email(self): + # occasionally the 'number' column may have an email address instead + value = self.result_set.getString("number") + return (value if general.isValidEmailAddress(value) else None) class TextNowMessagesParser(TskMessagesParser): """ diff --git a/InternalPythonModules/android/whatsapp.py b/InternalPythonModules/android/whatsapp.py index 85d71983d2..5346545450 100644 --- a/InternalPythonModules/android/whatsapp.py +++ b/InternalPythonModules/android/whatsapp.py @@ -172,14 +172,22 @@ class WhatsAppAnalyzer(general.AndroidComponentAnalyzer): try: contacts_parser = WhatsAppContactsParser(contacts_db, self._PARSER_NAME) while contacts_parser.next(): - helper.addContact( - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email(), - contacts_parser.get_other_attributes() - ) + name = contacts_parser.get_contact_name() + phone = contacts_parser.get_phone() + home_phone = contacts_parser.get_home_phone() + mobile_phone = contacts_parser.get_mobile_phone() + email = contacts_parser.get_email() + + # add contact if we have at least one valid phone/email + if phone or home_phone or mobile_phone or email: + helper.addContact( + name, + phone, + home_phone, + mobile_phone, + email, + contacts_parser.get_other_attributes() + ) contacts_parser.close() except SQLException as ex: self._logger.log(Level.WARNING, "Error querying the whatsapp database for contacts.", ex) @@ -426,8 +434,14 @@ class WhatsAppContactsParser(TskContactsParser): return self.result_set.getString("name") def get_phone(self): - return self.result_set.getString("number") + number = self.result_set.getString("number") + return (number if general.isValidPhoneNumer(number) else None) + def get_email(self): + # occasionally the 'number' column may have an email address instead + value = self.result_set.getString("number") + return (value if general.isValidEmailAddress(value) else None) + def get_other_attributes(self): return [BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ID,