From 0213e40d3ef2321354a47c8982e3661704388844 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 16:48:21 -0400 Subject: [PATCH 01/15] Initial infra commit --- .../android/ResultSetIterator.py | 35 +++++++++ .../android/TskCallLogsParser.py | 63 ++++++++++++++++ .../android/TskContactsParser.py | 49 +++++++++++++ .../android/TskMessagesParser.py | 72 +++++++++++++++++++ 4 files changed, 219 insertions(+) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..3bc0e0141b --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,63 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self._DEFAULT_STRING = "" + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = Account.Address("","") + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + + def get_call_direction(self): + return self._DEFAULT_DIRECTION + + def get_phone_number_from(self): + return self._DEFAULT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ADDRESS + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..69d05cd6fe --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,72 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_COMMUNICATION_DIRECTION + + def get_phone_number_from(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_phone_number_to(self): + return self._DEFAULT_ACCOUNT_ADDRESS + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_MSG_READ_STATUS + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT From 7c15ebc260529d5307b027a9deaed329abd56343 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 11 Sep 2019 16:58:19 -0400 Subject: [PATCH 02/15] Initial template commit --- InternalPythonModules/android/textnow.py | 195 +++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 InternalPythonModules/android/textnow.py diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py new file mode 100644 index 0000000000..a84d1a3931 --- /dev/null +++ b/InternalPythonModules/android/textnow.py @@ -0,0 +1,195 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskMessagesParser import TskMessagesParser +from TskContactsParser import TskContactsParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class TextNowAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the TextNow App databases for TSK contacts, message + and calllog artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._TEXTNOW_PACKAGE_NAME = "com.enflick.android.TextNow" + self._PARSER_NAME = "TextNow Parser" + + def analyze(self, dataSource, fileManager, context): + """ + Extract, Transform and Load all messages, contacts and + calllogs from the TextNow databases. + """ + + try: + textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "viber_data", True, self._TEXTNOW_PACKAGE_NAME) + + #Extract TSK_CONTACT and TSK_CALLLOG information + for textnow_db in textnow_dbs: + helper = AppDBParserHelper(self._PARSER_NAME, + textnow_db.getDBFile(), Account.Type.TEXTNOW) + + contacts_parser = TextNowContactsParser(textnow_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + + calllog_parser = TextNowCallLogsParser(textnow_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + + messages_parser = TextNowMessagesParser(textnow_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + + textnow_db.close() + except (SQLException, TskCoreException) as ex: + #Error parsing TextNow db + self._logger.log(Level.WARNING, "Error parsing TextNow Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exec()) + +class TextNowCallLogsParser(TskCallLogsParser): + """ + Extracts TSK_CALLLOG information from the TextNow database. + TSK_CALLLOG fields that are not in the TextNow database are given + a default value inherited from the super class. + """ + + def __init__(self, calllog_db): + super(TextNowCallLogsParser, self).__init__(calllog_db.runQuery( + """ + """ + ) + ) + + def get_phone_number_from(self): + + def get_phone_number_to(self): + + def get_call_direction(self): + + def get_call_start_date_time(self): + + def get_call_end_date_time(self): + + def get_call_type(self): + +class TextNowContactsParser(TskContactsParser): + """ + Extracts TSK_CONTACT information from the TextNow database. + TSK_CONTACT fields that are not in the TextNow database are given + a default value inherited from the super class. + """ + + def __init__(self, contact_db): + super(TextNowContactsParser, self).__init__(contact_db.runQuery( + """ + """ + ) + ) + + def get_account_name(self): + + def get_contact_name(self): + + def get_phone(self): + +class TextNowMessagesParser(TskMessagesParser): + """ + Extract TSK_MESSAGE information from the TextNow database. + TSK_CONTACT fields that are not in the TextNow database are given + a default value inherited from the super class. + """ + + def __init__(self, message_db): + super(TextNowMessagesParser, self).__init__(message_db.runQuery( + """ + """ + ) + ) + self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" + + def get_message_type(self): + return self._TEXTNOW_MESSAGE_TYPE + + def get_phone_number_from(self): + + def get_message_direction(self): + + def get_phone_number_to(self): + + def get_message_date_time(self): + + def get_message_read_status(self): + + def get_message_text(self): + + def get_thread_id(self): From 4f171c7a51361c75918fa0f0b9a0b0ae53612c84 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Sep 2019 17:19:14 -0400 Subject: [PATCH 03/15] Updated parser templates with correct default values --- InternalPythonModules/android/TskCallLogsParser.py | 2 +- InternalPythonModules/android/TskMessagesParser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 3bc0e0141b..763ba3c15f 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -36,7 +36,7 @@ class TskCallLogsParser(ResultSetIterator): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN - self._DEFAULT_ADDRESS = Account.Address("","") + self._DEFAULT_ADDRESS = None self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 69d05cd6fe..15c4166db7 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -36,7 +36,7 @@ class TskMessagesParser(ResultSetIterator): self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN - self._DEFAULT_ACCOUNT_ADDRESS = Account.Address("","") + self._DEFAULT_ACCOUNT_ADDRESS = None self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING From e90a2ea15c0e81538fe188e9d8a1dcc952cf01ea Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Sep 2019 17:46:53 -0400 Subject: [PATCH 04/15] More infra additions --- InternalPythonModules/android/general.py | 11 +++++++++++ InternalPythonModules/android/module.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..f3dce7b96a 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import textnow class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), textnow.TextNowAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) From 8f3eab11a421bf01c0483424520d0ae76ea6047d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Sep 2019 17:47:14 -0400 Subject: [PATCH 05/15] Fully implemented text now parser --- InternalPythonModules/android/textnow.py | 136 ++++++++++++++++++++++- 1 file changed, 131 insertions(+), 5 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index a84d1a3931..6ab7d14801 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -41,6 +41,7 @@ from org.sleuthkit.datamodel import Account from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser +from general import appendAttachmentList import traceback import general @@ -55,6 +56,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) self._TEXTNOW_PACKAGE_NAME = "com.enflick.android.TextNow" self._PARSER_NAME = "TextNow Parser" + self._VERSION = "6.41.0.2" def analyze(self, dataSource, fileManager, context): """ @@ -64,13 +66,13 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): try: textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "viber_data", True, self._TEXTNOW_PACKAGE_NAME) + "textnow_data.db", True, self._TEXTNOW_PACKAGE_NAME) - #Extract TSK_CONTACT and TSK_CALLLOG information for textnow_db in textnow_dbs: helper = AppDBParserHelper(self._PARSER_NAME, textnow_db.getDBFile(), Account.Type.TEXTNOW) + #Extract TSK_CONTACT information contacts_parser = TextNowContactsParser(textnow_db) while contacts_parser.next(): helper.addContact( @@ -83,6 +85,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): ) contacts_parser.close() + #Extract TSK_CALLLOG information calllog_parser = TextNowCallLogsParser(textnow_db) while calllog_parser.next(): helper.addCalllog( @@ -95,6 +98,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): ) calllog_parser.close() + #Extract TSK_MESSAGES information messages_parser = TextNowMessagesParser(textnow_db) while messages_parser.next(): helper.addMessage( @@ -109,7 +113,7 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - + textnow_db.close() except (SQLException, TskCoreException) as ex: #Error parsing TextNow db @@ -126,21 +130,47 @@ class TextNowCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(TextNowCallLogsParser, self).__init__(calllog_db.runQuery( """ + SELECT contact_value AS num, + message_direction AS direction, + message_text AS duration, + date AS datetime + FROM messages AS M + WHERE message_type IN ( 100, 102 ) """ ) ) + self._INCOMING_CALL_TYPE = 1 + self._OUTGOING_CALL_TYPE = 2 + self._has_errors = False def get_phone_number_from(self): + if self.get_call_direction() == self.OUTGOING_CALL: + return super(TextNowCallLogsParser, self).get_phone_number_from() + return Account.Address(self.result_set.getString("num"), + self.result_set.getString("num")) def get_phone_number_to(self): + if self.get_call_direction() == self.INCOMING_CALL: + return super(TextNowCallLogsParser, self).get_phone_number_to() + return Account.Address(self.result_set.getString("num"), + self.result_set.getString("num")) def get_call_direction(self): + if self.result_set.getInt("direction") == self._INCOMING_CALL_TYPE: + return self.INCOMING_CALL + return self.OUTGOING_CALL def get_call_start_date_time(self): + return self.result_set.getLong("datetime") / 1000 def get_call_end_date_time(self): - - def get_call_type(self): + start = self.get_call_start_date_time() + duration = self.result_set.getString("duration") + try: + return start + long(duration) + except ValueError as ve: + self._has_errors = True + return super(TextNowCallLogsParser, self).get_call_end_date_time() class TextNowContactsParser(TskContactsParser): """ @@ -152,15 +182,25 @@ class TextNowContactsParser(TskContactsParser): def __init__(self, contact_db): super(TextNowContactsParser, self).__init__(contact_db.runQuery( """ + SELECT C.contact_value AS number, + CASE + WHEN contact_name IS NULL THEN contact_value + WHEN contact_name == "" THEN contact_value + ELSE contact_name + END name + FROM contacts AS C """ ) ) def get_account_name(self): + return self.result_set.getString("number") def get_contact_name(self): + return self.result_set.getString("name") def get_phone(self): + return self.result_set.getString("number") class TextNowMessagesParser(TskMessagesParser): """ @@ -170,26 +210,112 @@ class TextNowMessagesParser(TskMessagesParser): """ def __init__(self, message_db): + """ + The TextNow database in v6.41.0.2 is structured as follows: + - A messages table, which stores messages from/to a number + - A contacts table, which stores phone numbers + - A groups table, which stores each group the device owner is a part of + - A group_members table, which stores who is in each group + + The messages table contains both call logs and messages, with a type + column differentiating the two. + + The query below does the following: + - The group_info inner query creates a comma seperated list of group recipients + for each group. This result is then joined on the groups table to get the thread id. + - The contacts table is unioned with this result so we have a complete map + of "from" phone_numbers -> recipients (group or single). This is the + 'to_from_map' inner query. + - Finally, the to_from_map results are joined with the messages table to get all + of the communication details. + """ super(TextNowMessagesParser, self).__init__(message_db.runQuery( """ + SELECT CASE + WHEN message_direction == 2 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE contact_name + end from_address, + CASE + WHEN message_direction == 1 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE to_addresses + end to_address, + message_direction, + message_text, + M.READ, + M.date, + M.attach, + thread_id + FROM (SELECT group_info.contact_value, + group_info.to_addresses, + G._id AS thread_id + FROM (SELECT GM.contact_value, + Group_concat(GM.member_contact_value) AS to_addresses + FROM group_members AS GM + GROUP BY GM.contact_value) AS group_info + JOIN groups AS G + ON G.contact_value = group_info.contact_value + UNION + SELECT c.contact_value, + NULL, + -1 + FROM contacts AS c) AS to_from_map + JOIN messages AS M + ON M.contact_value = to_from_map.contact_value + WHERE message_type NOT IN ( 102, 100 ) """ ) ) self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" + self._INCOMING_MESSAGE_TYPE = 1 + self._OUTGOING_MESSAGE_TYPE = 2 + self._UNKNOWN_THREAD_ID = -1 def get_message_type(self): return self._TEXTNOW_MESSAGE_TYPE def get_phone_number_from(self): + if self.result_set.getString("from_address") == "": + return super(TextNowMessagesParser, self).get_phone_number_from() + return Account.Address(self.result_set.getString("from_address"), + self.result_set.getString("from_address")) def get_message_direction(self): + direction = self.result_set.getInt("message_direction") + if direction == self._INCOMING_MESSAGE_TYPE: + return self.INCOMING + return self.OUTGOING def get_phone_number_to(self): + if self.result_set.getString("to_address") == "": + return super(TextNowMessagesParser, self).get_phone_number_to() + return Account.Address(self.result_set.getString("to_address"), + self.result_set.getString("to_address")) def get_message_date_time(self): + #convert ms to s + return self.result_set.getLong("date") / 1000; def get_message_read_status(self): + read = self.result_set.getBoolean("read") + if self.get_message_direction() == self.INCOMING: + if read == True: + return self.READ + return self.UNREAD + + #read status for outgoing messages cannot be determined, give default + return super(TextNowMessagesParser, self).get_message_read_status() def get_message_text(self): + text = self.result_set.getString("message_text") + attachment = self.result_set.getString("attach") + if attachment != "": + text = appendAttachmentList(text, [attachment]) + return text def get_thread_id(self): + thread_id = self.result_set.getInt("thread_id") + if thread_id == self._UNKNOWN_THREAD_ID: + return super(TextNowMessagesParser, self).get_thread_id() + return str(thread_id) From 65827284f1ec75fa9fb8e640ed09c354143c7c37 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 18 Sep 2019 16:05:33 -0400 Subject: [PATCH 06/15] Implement change --- .../dsp/AddLogicalImageTask.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 6c0bc5a0d6..50f6819264 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -63,6 +63,8 @@ final class AddLogicalImageTask implements Runnable { private final static String MODULE_NAME = "Logical Imager"; //NON-NLS private final static String ROOT_STR = "root"; // NON-NLS private final static String VHD_EXTENSION = ".vhd"; // NON-NLS + private final static int REPORT_PROGRESS_INTERVAL = 100; + private final static int POST_ARTIFACT_INTERVAL = 1000; private final String deviceId; private final String timeZone; private final File src; @@ -362,9 +364,12 @@ final class AddLogicalImageTask implements Runnable { String filename = fields[7]; String parentPath = fields[8]; - if (lineNumber % 100 == 0) { + if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); } + if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { + postArtifacts(artifacts); + } String query = makeQuery(createVHD, vhdFilename, fileMetaAddressStr, parentPath, filename); // TODO - findAllFilesWhere should SQL-escape the query @@ -375,15 +380,19 @@ final class AddLogicalImageTask implements Runnable { lineNumber++; } // end reading file - try { - // index the artifact for keyword search - blackboard.postArtifacts(artifacts, MODULE_NAME); - } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS - } + postArtifacts(artifacts); } } + private void postArtifacts(List artifacts) { + try { + // index the artifact for keyword search + blackboard.postArtifacts(artifacts, MODULE_NAME); + } catch (Blackboard.BlackboardException ex) { + LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS + } + } + private void addInterestingFileToArtifacts(AbstractFile file, String ruleSetName, String ruleName, List artifacts) throws TskCoreException { Collection attributes = new ArrayList<>(); BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, ruleSetName); @@ -440,7 +449,7 @@ final class AddLogicalImageTask implements Runnable { String ctime = fields[13]; parentPath = ROOT_STR + "/" + vhdFilename + "/" + parentPath; - if (lineNumber % 100 == 0) { + if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingExtractedFile(lineNumber, totalFiles)); } From a51d718645d4fbc94c4680623b0cd6c50a66f517 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 18 Sep 2019 16:08:44 -0400 Subject: [PATCH 07/15] clear artifacts after posting --- .../sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 50f6819264..64000e8f36 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -369,6 +369,7 @@ final class AddLogicalImageTask implements Runnable { } if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { postArtifacts(artifacts); + artifacts.clear(); } String query = makeQuery(createVHD, vhdFilename, fileMetaAddressStr, parentPath, filename); From e7b6e1a165896093646dd11b5fdf3a7aa932ee9e Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 19 Sep 2019 12:46:54 -0400 Subject: [PATCH 08/15] fix merge error --- .../dsp/AddLogicalImageTask.java | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index a569f93b75..b6ad885c4a 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -367,21 +367,19 @@ final class AddLogicalImageTask implements Runnable { List fileIds = entry.getValue(); for (Long fileId: fileIds) { - if (lineNumber % 100 == 0) { + if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); } + if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { + postArtifacts(artifacts); + artifacts.clear(); + } addInterestingFileToArtifacts(fileId, ruleSetName, ruleName, artifacts); lineNumber++; } iterator.remove(); } - - try { - // index the artifact for keyword search - blackboard.postArtifacts(artifacts, MODULE_NAME); - } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS - } + postArtifacts(artifacts); } private void addInterestingFileToArtifacts(long fileId, String ruleSetName, String ruleName, List artifacts) throws TskCoreException { @@ -429,11 +427,7 @@ final class AddLogicalImageTask implements Runnable { String parentPath = fields[8]; if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); - } - if (lineNumber % POST_ARTIFACT_INTERVAL == 0) { - postArtifacts(artifacts); - artifacts.clear(); + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_searchingInterestingFile(lineNumber, totalFiles)); } String query = makeQuery(vhdFilename, fileMetaAddressStr, parentPath, filename); @@ -450,8 +444,6 @@ final class AddLogicalImageTask implements Runnable { } lineNumber++; } // end reading file - - postArtifacts(artifacts); } return interestingFileMap; } From e979034b519cc523d908cac598b8d9d2598e9d0d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 14:33:02 -0400 Subject: [PATCH 09/15] Update base parser classes with new infra changes --- .../android/TskCallLogsParser.py | 16 +++++++++------- .../android/TskMessagesParser.py | 17 +++++++++-------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 763ba3c15f..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,14 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT From b052e9274df92b968f841b288b5fa9ec941de237 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 19 Sep 2019 15:21:00 -0400 Subject: [PATCH 10/15] Check for cancel --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index b6ad885c4a..f5d728b633 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -367,6 +367,10 @@ final class AddLogicalImageTask implements Runnable { List fileIds = entry.getValue(); for (Long fileId: fileIds) { + if (cancelled) { + postArtifacts(artifacts); + return; + } if (lineNumber % REPORT_PROGRESS_INTERVAL == 0) { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFile(lineNumber, totalFiles)); } From 424dad7e148466286bc06b0ccd449b048cf37e92 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 15:53:04 -0400 Subject: [PATCH 11/15] made changes to textnow with updated infra --- InternalPythonModules/android/textnow.py | 198 +++++++++++++++-------- 1 file changed, 131 insertions(+), 67 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index 6ab7d14801..1471f720a5 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -26,22 +26,30 @@ from java.sql import ResultSet from java.sql import SQLException from java.sql import Statement from java.util.logging import Level +from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper + +from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + from TskMessagesParser import TskMessagesParser from TskContactsParser import TskContactsParser from TskCallLogsParser import TskCallLogsParser -from general import appendAttachmentList import traceback import general @@ -50,8 +58,17 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): """ Parses the TextNow App databases for TSK contacts, message and calllog artifacts. - """ - + + The TextNow database in v6.41.0.2 is structured as follows: + - A messages table, which stores messages from/to a number + - A contacts table, which stores phone numbers + - A groups table, which stores each group the device owner is a part of + - A group_members table, which stores who is in each group + + The messages table contains both call logs and messages, with a type + column differentiating the two. + """ + def __init__(self): self._logger = Logger.getLogger(self.__class__.__name__) self._TEXTNOW_PACKAGE_NAME = "com.enflick.android.TextNow" @@ -64,61 +81,116 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): calllogs from the TextNow databases. """ - try: - textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, + textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, "textnow_data.db", True, self._TEXTNOW_PACKAGE_NAME) - for textnow_db in textnow_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, - textnow_db.getDBFile(), Account.Type.TEXTNOW) - - #Extract TSK_CONTACT information - contacts_parser = TextNowContactsParser(textnow_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() - - #Extract TSK_CALLLOG information - calllog_parser = TextNowCallLogsParser(textnow_db) - while calllog_parser.next(): - helper.addCalllog( - calllog_parser.get_call_direction(), - calllog_parser.get_phone_number_from(), - calllog_parser.get_phone_number_to(), - calllog_parser.get_call_start_date_time(), - calllog_parser.get_call_end_date_time(), - calllog_parser.get_call_type() - ) - calllog_parser.close() - - #Extract TSK_MESSAGES information - messages_parser = TextNowMessagesParser(textnow_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() - + for textnow_db in textnow_dbs: + try: + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + textnow_db.getDBFile(), Account.Type.TEXTNOW + ) + self.parse_contacts(textnow_db, helper) + self.parse_calllogs(textnow_db, helper) + self.parse_messages(textnow_db, helper) + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + finally: textnow_db.close() - except (SQLException, TskCoreException) as ex: + + def parse_contacts(self, textnow_db, helper): + #Query for contacts and iterate row by row adding + #each contact artifact + try: + contacts_parser = TextNowContactsParser(textnow_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: #Error parsing TextNow db - self._logger.log(Level.WARNING, "Error parsing TextNow Databases", ex) - self._logger.log(Level.WARNING, traceback.format_exec()) + self._logger.log(Level.WARNING, "Error parsing TextNow databases for contacts", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifacts to the case database.. case database is not complete. + self._logger.log(Level.SEVERE, + "Error adding TextNow contacts artifacts to the case database", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard... + self._logger.log(Level.WARNING, + "Error posting TextNow contacts artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_calllogs(self, textnow_db, helper): + #Query for call logs and iterate row by row adding + #each call log artifact + try: + calllog_parser = TextNowCallLogsParser(textnow_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing TextNow databases for calllogs", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifacts to the case database.. case database is not complete. + self._logger.log(Level.SEVERE, + "Error adding TextNow call log artifacts to the case database", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard... + self._logger.log(Level.WARNING, + "Error posting TextNow call log artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_messages(self, textnow_db, helper): + #Query for messages and iterate row by row adding + #each message artifact + try: + messages_parser = TextNowMessagesParser(textnow_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + #Error parsing TextNow db + self._logger.log(Level.WARNING, "Error parsing TextNow databases for messages.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifacts to the case database.. case database is not complete. + self._logger.log(Level.SEVERE, + "Error adding TextNow messages artifacts to the case database", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard... + self._logger.log(Level.WARNING, + "Error posting TextNow messages artifact to the blackboard", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) class TextNowCallLogsParser(TskCallLogsParser): """ @@ -128,6 +200,9 @@ class TextNowCallLogsParser(TskCallLogsParser): """ def __init__(self, calllog_db): + """ + message_type of 100 or 102 are for calls (audio, video) + """ super(TextNowCallLogsParser, self).__init__(calllog_db.runQuery( """ SELECT contact_value AS num, @@ -141,7 +216,6 @@ class TextNowCallLogsParser(TskCallLogsParser): ) self._INCOMING_CALL_TYPE = 1 self._OUTGOING_CALL_TYPE = 2 - self._has_errors = False def get_phone_number_from(self): if self.get_call_direction() == self.OUTGOING_CALL: @@ -169,7 +243,6 @@ class TextNowCallLogsParser(TskCallLogsParser): try: return start + long(duration) except ValueError as ve: - self._has_errors = True return super(TextNowCallLogsParser, self).get_call_end_date_time() class TextNowContactsParser(TskContactsParser): @@ -211,15 +284,6 @@ class TextNowMessagesParser(TskMessagesParser): def __init__(self, message_db): """ - The TextNow database in v6.41.0.2 is structured as follows: - - A messages table, which stores messages from/to a number - - A contacts table, which stores phone numbers - - A groups table, which stores each group the device owner is a part of - - A group_members table, which stores who is in each group - - The messages table contains both call logs and messages, with a type - column differentiating the two. - The query below does the following: - The group_info inner query creates a comma seperated list of group recipients for each group. This result is then joined on the groups table to get the thread id. @@ -311,7 +375,7 @@ class TextNowMessagesParser(TskMessagesParser): text = self.result_set.getString("message_text") attachment = self.result_set.getString("attach") if attachment != "": - text = appendAttachmentList(text, [attachment]) + text = general.appendAttachmentList(text, [attachment]) return text def get_thread_id(self): From 97a2d081238227a775f37546cfcbc12f48f83a96 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 19 Sep 2019 16:21:00 -0400 Subject: [PATCH 12/15] Address review comments. --- InternalPythonModules/android/imo.py | 38 ++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 5a653ff9d5..36b90bc3be 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -56,10 +56,14 @@ and adds artifacts to the case. class IMOAnalyzer(general.AndroidComponentAnalyzer): def __init__(self): self._logger = Logger.getLogger(self.__class__.__name__) + self._PACKAGE_NAME = "com.imo.android.imous" + self._PARSER_NAME = "IMO Parser" + self._MESSAGE_TYPE = "IMO Message" + self._VERSION = "9.8.0" def analyze(self, dataSource, fileManager, context): selfAccountAddress = None - accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", True, "com.imo.android.imous") + accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", True, self._PACKAGE_NAME) for accountDb in accountDbs: try: accountResultSet = accountDb.runQuery("SELECT uid, name FROM account") @@ -71,16 +75,26 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): selfAccountAddress = Account.Address(accountResultSet.getString("uid"), accountResultSet.getString("name")) except SQLException as ex: - self._logger.log(Level.SEVERE, "Error processing query result for account", ex) + self._logger.log(Level.WARNING, "Error processing query result for account", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) finally: accountDb.close() - friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", True, "com.imo.android.imous") + friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", True, self._PACKAGE_NAME) for friendsDb in friendsDbs: try: - friendsDBHelper = CommunicationArtifactsHelper(Case.getCurrentCase().getSleuthkitCase(), - "IMO Parser", friendsDb.getDBFile(), + current_case = Case.getCurrentCaseThrows() + if selfAccountAddress is not None: + friendsDBHelper = CommunicationArtifactsHelper(current_case.getSleuthkitCase(), + self._PARSER_NAME, + friendsDb.getDBFile(), Account.Type.IMO, Account.Type.IMO, selfAccountAddress ) + else: + friendsDBHelper = CommunicationArtifactsHelper(current_case.getSleuthkitCase(), + self._PARSER_NAME, + friendsDb.getDBFile(), + Account.Type.IMO + ) contactsResultSet = friendsDb.runQuery("SELECT buid, name FROM friends") if contactsResultSet is not None: while contactsResultSet.next(): @@ -121,7 +135,7 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): messageArtifact = friendsDBHelper.addMessage( - "IMO Message", + self._MESSAGE_TYPE, direction, fromAddress, toAddress, @@ -137,8 +151,16 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): except SQLException as ex: self._logger.log(Level.WARNING, "Error processing query result for IMO friends", ex) - except (TskCoreException, BlackboardException) as ex: - self._logger.log(Level.WARNING, "Failed to message artifacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, "Failed to add IMO message artifacts.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + self._logger.log(Level.WARNING, "Failed to post artifacts.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) finally: friendsDb.close() From a71fb3b283d6a7168f081953baa8a695c2c0b725 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 17:24:37 -0400 Subject: [PATCH 13/15] Changed where the bail out happens for nocurrentcase.. also closed all databases after parsing rather than during --- InternalPythonModules/android/textnow.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index 1471f720a5..5ca6d4e909 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -83,9 +83,9 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): textnow_dbs = AppSQLiteDB.findAppDatabases(dataSource, "textnow_data.db", True, self._TEXTNOW_PACKAGE_NAME) - - for textnow_db in textnow_dbs: - try: + + try: + for textnow_db in textnow_dbs: current_case = Case.getCurrentCaseThrows() helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, @@ -94,11 +94,12 @@ class TextNowAnalyzer(general.AndroidComponentAnalyzer): self.parse_contacts(textnow_db, helper) self.parse_calllogs(textnow_db, helper) self.parse_messages(textnow_db, helper) - except NoCurrentCaseException as ex: - self._logger.log(Level.WARNING, "No case currently open.", ex) - self._logger.log(Level.WARNING, traceback.format_exc()) - finally: - textnow_db.close() + except NoCurrentCaseException as ex: + self._logger.log(Level.WARNING, "No case currently open.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + for textnow_db in textnow_dbs: + textnow_db.close() def parse_contacts(self, textnow_db, helper): #Query for contacts and iterate row by row adding From d6c07d5055caf99746b89eb0ec3609fbdef51c39 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 17:48:23 -0400 Subject: [PATCH 14/15] Bug fixes --- InternalPythonModules/android/textnow.py | 82 +++++++++++++----------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/InternalPythonModules/android/textnow.py b/InternalPythonModules/android/textnow.py index 5ca6d4e909..216864f2e1 100644 --- a/InternalPythonModules/android/textnow.py +++ b/InternalPythonModules/android/textnow.py @@ -296,46 +296,47 @@ class TextNowMessagesParser(TskMessagesParser): """ super(TextNowMessagesParser, self).__init__(message_db.runQuery( """ - SELECT CASE - WHEN message_direction == 2 THEN "" - WHEN to_addresses IS NULL THEN M.contact_value - ELSE contact_name - end from_address, - CASE - WHEN message_direction == 1 THEN "" - WHEN to_addresses IS NULL THEN M.contact_value - ELSE to_addresses - end to_address, - message_direction, - message_text, - M.READ, - M.date, - M.attach, - thread_id - FROM (SELECT group_info.contact_value, - group_info.to_addresses, - G._id AS thread_id - FROM (SELECT GM.contact_value, - Group_concat(GM.member_contact_value) AS to_addresses - FROM group_members AS GM - GROUP BY GM.contact_value) AS group_info - JOIN groups AS G - ON G.contact_value = group_info.contact_value - UNION - SELECT c.contact_value, - NULL, - -1 - FROM contacts AS c) AS to_from_map - JOIN messages AS M - ON M.contact_value = to_from_map.contact_value - WHERE message_type NOT IN ( 102, 100 ) + + SELECT CASE + WHEN message_direction == 2 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE contact_name + end from_address, + CASE + WHEN message_direction == 1 THEN "" + WHEN to_addresses IS NULL THEN M.contact_value + ELSE to_addresses + end to_address, + message_direction, + message_text, + M.READ, + M.date, + M.attach, + thread_id + FROM (SELECT group_info.contact_value, + group_info.to_addresses, + G.contact_value AS thread_id + FROM (SELECT GM.contact_value, + Group_concat(GM.member_contact_value) AS to_addresses + FROM group_members AS GM + GROUP BY GM.contact_value) AS group_info + JOIN groups AS G + ON G.contact_value = group_info.contact_value + UNION + SELECT c.contact_value, + NULL, + "-1" + FROM contacts AS c) AS to_from_map + JOIN messages AS M + ON M.contact_value = to_from_map.contact_value + WHERE message_type NOT IN ( 102, 100 ) """ ) ) self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" self._INCOMING_MESSAGE_TYPE = 1 self._OUTGOING_MESSAGE_TYPE = 2 - self._UNKNOWN_THREAD_ID = -1 + self._UNKNOWN_THREAD_ID = "-1" def get_message_type(self): return self._TEXTNOW_MESSAGE_TYPE @@ -355,8 +356,13 @@ class TextNowMessagesParser(TskMessagesParser): def get_phone_number_to(self): if self.result_set.getString("to_address") == "": return super(TextNowMessagesParser, self).get_phone_number_to() - return Account.Address(self.result_set.getString("to_address"), - self.result_set.getString("to_address")) + recipients = self.result_set.getString("to_address").split(",") + + recipient_accounts = [] + for recipient in recipients: + recipient_accounts.append(Account.Address(recipient, recipient)) + + return recipient_accounts def get_message_date_time(self): #convert ms to s @@ -380,7 +386,7 @@ class TextNowMessagesParser(TskMessagesParser): return text def get_thread_id(self): - thread_id = self.result_set.getInt("thread_id") + thread_id = self.result_set.getString("thread_id") if thread_id == self._UNKNOWN_THREAD_ID: return super(TextNowMessagesParser, self).get_thread_id() - return str(thread_id) + return thread_id From a8827a304ddb2de33a09928f70f45a8c414812df Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 19 Sep 2019 19:53:45 -0400 Subject: [PATCH 15/15] Fixed missing import. --- InternalPythonModules/android/imo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 36b90bc3be..714c029445 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -29,6 +29,7 @@ from java.util.logging import Level from java.util import ArrayList from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB