From 422ff6330ec5b5c30d634d70d960c20b6af71b0e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 9 Sep 2019 13:48:09 -0400 Subject: [PATCH 1/8] Moved work to use this branch as base --- .../android/ResultSetIterator.py | 35 +++ .../android/TskCallLogsParser.py | 59 +++++ .../android/TskContactsParser.py | 49 ++++ .../android/TskMessagesParser.py | 68 ++++++ InternalPythonModules/android/line.py | 225 ++++++++++++++++++ InternalPythonModules/android/module.py | 3 +- 6 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 InternalPythonModules/android/ResultSetIterator.py create mode 100644 InternalPythonModules/android/TskCallLogsParser.py create mode 100644 InternalPythonModules/android/TskContactsParser.py create mode 100644 InternalPythonModules/android/TskMessagesParser.py create mode 100644 InternalPythonModules/android/line.py diff --git a/InternalPythonModules/android/ResultSetIterator.py b/InternalPythonModules/android/ResultSetIterator.py new file mode 100644 index 0000000000..4abd4438df --- /dev/null +++ b/InternalPythonModules/android/ResultSetIterator.py @@ -0,0 +1,35 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +class ResultSetIterator(object): + """ + Generic base class for iterating through database recordms + """ + + def __init__(self, result_set): + self.result_set = result_set + + def next(self): + if self.result_set is None: + return False + return self.result_set.next() + + def close(self): + if self.result_set is not None: + self.result_set.close() diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py new file mode 100644 index 0000000000..77c7aa12da --- /dev/null +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -0,0 +1,59 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskCallLogsParser(ResultSetIterator): + """ + Generic TSK_CALLLOG artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CALLLOG + format. + + A simple example of data transformation would be computing + the end time of a call when the database only supplies the start + time and duration. + """ + + def __init__(self, result_set): + super(TskCallLogsParser, self).__init__(result_set) + self.INCOMING_MSG_STRING = "Incoming" + self.OUTGOING_MSG_STRING = "Outgoing" + self._DEFAULT_STRING = "" + self._DEFAULT_LONG = -1L + + def get_account_name(self): + return self._DEFAULT_STRING + + def get_call_direction(self): + return self._DEFAULT_STRING + + def get_phone_number_from(self): + return self._DEFAULT_STRING + + def get_phone_number_to(self): + return self._DEFAULT_STRING + + def get_call_start_date_time(self): + return self._DEFAULT_LONG + + def get_call_end_date_time(self): + return self._DEFAULT_LONG + + def get_contact_name(self): + return self._DEFAULT_STRING diff --git a/InternalPythonModules/android/TskContactsParser.py b/InternalPythonModules/android/TskContactsParser.py new file mode 100644 index 0000000000..122e6a9445 --- /dev/null +++ b/InternalPythonModules/android/TskContactsParser.py @@ -0,0 +1,49 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskContactsParser(ResultSetIterator): + """ + Generic TSK_CONTACT artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_CONTACT + format. + """ + + def __init__(self, result_set): + super(TskContactsParser, self).__init__(result_set) + self._DEFAULT_VALUE = "" + + def get_account_name(self): + return self._DEFAULT_VALUE + + def get_contact_name(self): + return self._DEFAULT_VALUE + + def get_phone(self): + return self._DEFAULT_VALUE + + def get_home_phone(self): + return self._DEFAULT_VALUE + + def get_mobile_phone(self): + return self._DEFAULT_VALUE + + def get_email(self): + return self._DEFAULT_VALUE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py new file mode 100644 index 0000000000..0346a203e7 --- /dev/null +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -0,0 +1,68 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ResultSetIterator import ResultSetIterator + +class TskMessagesParser(ResultSetIterator): + """ + Generic TSK_MESSAGE artifact template. Each of these methods + will contain the extraction and transformation logic for + converting raw database records to the expected TSK_MESSAGE + format. + + An easy example of such a transformation would be converting + message date time from milliseconds to seconds. + """ + + def __init__(self, result_set): + super(TskMessagesParser, self).__init__(result_set) + self.INCOMING_MSG_STRING = "Incoming" + self.OUTGOING_MSG_STRING = "Outgoing" + self._DEFAULT_TEXT = "" + self._DEFAULT_LONG = -1L + self._DEFAULT_INT = -1 + + def get_account_id(self): + return self._DEFAULT_TEXT + + def get_message_type(self): + return self._DEFAULT_TEXT + + def get_message_direction(self): + return self._DEFAULT_TEXT + + def get_phone_number_from(self): + return self._DEFAULT_TEXT + + def get_phone_number_to(self): + return self._DEFAULT_TEXT + + def get_message_date_time(self): + return self._DEFAULT_LONG + + def get_message_read_status(self): + return self._DEFAULT_INT + + def get_message_subject(self): + return self._DEFAULT_TEXT + + def get_message_text(self): + return self._DEFAULT_TEXT + + def get_thread_id(self): + return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py new file mode 100644 index 0000000000..67589fcbea --- /dev/null +++ b/InternalPythonModules/android/line.py @@ -0,0 +1,225 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from java.util import ArrayList +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.datamodel import ContentUtils +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account +from TskContactsParser import TskContactsParser +from TskMessagesParser import TskMessagesParser +from TskCallLogsParser import TskCallLogsParser + +import traceback +import general + +class LineAnalyzer(general.AndroidComponentAnalyzer): + """ + Parses the Line App databases for TSK contacts & message artifacts. + """ + + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + self._LINE_PACKAGE_NAME = "jp.naver.line.android" + self._PARSER_NAME = "Line Parser" + + def analyze(self, dataSource, fileManager, context): + try: + contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line", self._LINE_PACKAGE_NAME) + calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", self._LINE_PACKAGE_NAME) + + for contact_and_message_db in contact_and_message_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) + + contacts_parser = LineContactsParser(contact_and_message_db) + while contacts_parser.next(): + blackboard_util.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + """ + messages_parser = LineMessagesParser(line_db) + while messages_parser.next(): + blackboard_util.addMessage( + messages_parser.get_account_id(), + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + """ + contact_and_message_db.close() + + for calllog_db in calllog_dbs: + blackboard_util = BlackboardUtil(self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) + + calllog_parser = LineCallLogsParser(calllog_db) + while calllog_parser.next(): + blackboard_util.addCalllog( + calllog_parser.get_account_name(), + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_contact_name() + ) + + calllog_parser.close() + calllog_db.close() + + except (SQLException, TskCoreException) as ex: + # Error parsing Line databases. + self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + +class LineCallLogsParser(TskCallLogsParser): + """ + Parses out TSK_CALLLOG information from the Line database. + TSK_CALLLOG fields that are not in the line database are given + a default value inherited from the super class. + """ + + def __init__(self, calllog_db): + super(LineCallLogsParser, self).__init__(calllog_db.runQuery( + """ + SELECT C.caller_mid AS mid, + substr(C.call_type, -1) AS direction, + C.start_time AS start_time, + C.end_time AS end_time + FROM call_history AS C + """ + ) + ) + self._OUTGOING_CALL = "O" + self._INCOMING_CALL = "I" + self._had_error = False + + def get_call_direction(self): + direction = self.result_set.getString("direction") + if direction == self._OUTGOING_CALL: + return self.OUTGOING_MSG_STRING + return self.INCOMING_MSG_STRING + + def get_call_start_date_time(self): + start_time = self.result_set.getString("start_time") + try: + return long(start_time) / 1000 + except ValueError as ve: + self._had_error = True + + def get_call_end_date_time(self): + end_time = self.result_set.getString("end_time") + try: + return long(end_time) / 1000 + except ValueError as ve: + self._had_error = True + +class LineContactsParser(TskContactsParser): + """ + Parses out TSK_CONTACT information from the Line database. + TSK_CONTACT fields that are not in the line database are given + a default value inherited from the super class. + """ + + def __init__(self, contact_db): + super(LineContactsParser, self).__init__(contact_db.runQuery( + """ + SELECT name, + server_name + FROM contacts + """ + ) + ) + def get_account_name(self): + return self.result_set.getString("server_name") + + def get_contact_name(self): + return self.result_set.getString("name") + +class LineMessagesParser(TskMessagesParser): + """ + Parse out TSK_MESSAGE information from the Line database. + TSK_MESSAGE fields that are not in the line database are given + a default value inherited from the super class. + """ + + def __init__(self, message_db): + super().__init__(message_db.runQuery( + """SELECT created_time, content, contacts.server_name AS server_name, read_count + FROM chat_history + JOIN contacts + ON chat_history.from_mid = contacts.m_id""" + )) + self._LINE_MESSAGE_TYPE = "Line Message" + self._had_error = False + + def get_account_id(self): + return self.result_set.getString("server_name") + + def get_message_type(self): + return self.LINE_MESSAGE_TYPE + + def get_phone_number_from(self): + return self.result_set("server_name") + + def get_message_date_time(self): + created_time = self.result_set.getString("created_time") + try: + #Get time in seconds (created_time is stored in ms from epoch) + return long(created_time) / 1000 + except ValueError as ve: + self._had_error = True + return super(LineMessagesParser, self).get_message_date_time() + + def get_message_text(self): + content = self.result_set.getString("content") + if not LineContentUtil.is_text_message(content): + return "" + return content diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 6430ec82be..9893df2b74 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -47,6 +47,7 @@ import tangomessage import textmessage import wwfmessage import imo +import line class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -91,7 +92,7 @@ class AndroidIngestModule(DataSourceIngestModule): analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), - cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer(), line.LineAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) From b773f8a03db96dbfe7e0dc1e7d06e748031d43f4 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 9 Sep 2019 16:37:21 -0400 Subject: [PATCH 2/8] debugging line queries --- InternalPythonModules/android/line.py | 36 +++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index 67589fcbea..df8db5d088 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -60,8 +60,8 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): try: - contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line", self._LINE_PACKAGE_NAME) - calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", self._LINE_PACKAGE_NAME) + contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line.db", True, self._LINE_PACKAGE_NAME) + calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", True, self._LINE_PACKAGE_NAME) for contact_and_message_db in contact_and_message_dbs: blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) @@ -98,9 +98,17 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): for calllog_db in calllog_dbs: blackboard_util = BlackboardUtil(self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) + calllog_db.attachDatabase(dataSource, "naver_line.db", True, calllog_db.getDBFile().getParentPath(), "naver") calllog_parser = LineCallLogsParser(calllog_db) while calllog_parser.next(): + print(calllog_parser.get_account_name()) + print(calllog_parser.get_contact_name()) + print(calllog_parser.get_call_direction()) + print(calllog_parser.get_phone_number_from()) + print(calllog_parser.get_phone_number_to()) + print(calllog_parser.get_call_start_date_time()) + print(calllog_parser.get_call_end_date_time()) blackboard_util.addCalllog( calllog_parser.get_account_name(), calllog_parser.get_call_direction(), @@ -111,6 +119,7 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): calllog_parser.get_contact_name() ) + calllog_db.detachDatabase("naver") calllog_parser.close() calllog_db.close() @@ -129,11 +138,14 @@ class LineCallLogsParser(TskCallLogsParser): def __init__(self, calllog_db): super(LineCallLogsParser, self).__init__(calllog_db.runQuery( """ - SELECT C.caller_mid AS mid, - substr(C.call_type, -1) AS direction, - C.start_time AS start_time, - C.end_time AS end_time - FROM call_history AS C + SELECT substr(CallH.call_type, -1) AS direction, + CallH.start_time AS start_time, + CallH.end_time AS end_time, + ConT.server_name AS account_name, + ConT.name AS contact_name + FROM call_history AS CallH + JOIN naver.contacts AS ConT + ON CallH.caller_mid = ConT.m_id """ ) ) @@ -153,6 +165,8 @@ class LineCallLogsParser(TskCallLogsParser): return long(start_time) / 1000 except ValueError as ve: self._had_error = True + print("bad_conversion") + return super(LineCallLogsParser, self).get_call_start_date_time() def get_call_end_date_time(self): end_time = self.result_set.getString("end_time") @@ -160,6 +174,14 @@ class LineCallLogsParser(TskCallLogsParser): return long(end_time) / 1000 except ValueError as ve: self._had_error = True + print("bad conversion") + return super(LineCallLogsParser, self).get_call_end_date_time() + + def get_account_name(self): + return self.result_set.getString("account_name") + + def get_contact_name(self): + return self.result_set.getString("contact_name") class LineContactsParser(TskContactsParser): """ From 8edaab679e626270263368758357d5a212efcd44 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 12:32:35 -0400 Subject: [PATCH 3/8] infra upgrades --- .../android/TskCallLogsParser.py | 26 +++++++++++-------- .../android/TskMessagesParser.py | 22 +++++++++------- InternalPythonModules/android/general.py | 11 ++++++++ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 77c7aa12da..763ba3c15f 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): """ @@ -32,28 +34,30 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) - self.INCOMING_MSG_STRING = "Incoming" - self.OUTGOING_MSG_STRING = "Outgoing" self._DEFAULT_STRING = "" - self._DEFAULT_LONG = -1L + self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_ADDRESS = None + self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - def get_account_name(self): - return self._DEFAULT_STRING + self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING + self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO + self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO def get_call_direction(self): - return self._DEFAULT_STRING + return self._DEFAULT_DIRECTION def get_phone_number_from(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_STRING + return self._DEFAULT_ADDRESS def get_call_start_date_time(self): return self._DEFAULT_LONG def get_call_end_date_time(self): return self._DEFAULT_LONG - - def get_contact_name(self): - return self._DEFAULT_STRING + + def get_call_type(self): + return self._DEFAULT_CALL_TYPE diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 0346a203e7..15c4166db7 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -17,6 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator +from org.sleuthkit.datamodel import Account +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper class TskMessagesParser(ResultSetIterator): """ @@ -31,32 +33,34 @@ class TskMessagesParser(ResultSetIterator): def __init__(self, result_set): super(TskMessagesParser, self).__init__(result_set) - self.INCOMING_MSG_STRING = "Incoming" - self.OUTGOING_MSG_STRING = "Outgoing" self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_INT = -1 + self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_ACCOUNT_ADDRESS = None + self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN - def get_account_id(self): - return self._DEFAULT_TEXT + self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING + self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING + self.READ = AppDBParserHelper.MessageReadStatusEnum.READ + self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD def get_message_type(self): return self._DEFAULT_TEXT def get_message_direction(self): - return self._DEFAULT_TEXT + return self._DEFAULT_COMMUNICATION_DIRECTION def get_phone_number_from(self): - return self._DEFAULT_TEXT + return self._DEFAULT_ACCOUNT_ADDRESS def get_phone_number_to(self): - return self._DEFAULT_TEXT + return self._DEFAULT_ACCOUNT_ADDRESS def get_message_date_time(self): return self._DEFAULT_LONG def get_message_read_status(self): - return self._DEFAULT_INT + return self._DEFAULT_MSG_READ_STATUS def get_message_subject(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/general.py b/InternalPythonModules/android/general.py index 28c96be9b9..53c123d13c 100644 --- a/InternalPythonModules/android/general.py +++ b/InternalPythonModules/android/general.py @@ -26,3 +26,14 @@ class AndroidComponentAnalyzer: # The Analyzer should implement this method def analyze(self, dataSource, fileManager, context): raise NotImplementedError + +""" +A utility method to append list of attachments to msg body +""" +def appendAttachmentList(msgBody, attachmentsList): + body = msgBody + if attachmentsList: + body = body + "\n\n------------Attachments------------\n" + body = body + "\n".join(attachmentsList) + + return body From 8259882b71b84518f60de7bc48ca87fbb19e5082 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 22:04:27 -0400 Subject: [PATCH 4/8] Fully implemented line parser --- .../android/TskCallLogsParser.py | 1 + InternalPythonModules/android/line.py | 211 +++++++++++------- 2 files changed, 136 insertions(+), 76 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 763ba3c15f..8c61070693 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -38,6 +38,7 @@ class TskCallLogsParser(ResultSetIterator): self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1 self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index df8db5d088..d7cab1b852 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -31,8 +31,8 @@ from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil -from org.sleuthkit.autopsy.coreutils import AppSQLiteDB as SQLiteUtil -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper as BlackboardUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile @@ -57,18 +57,22 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) self._LINE_PACKAGE_NAME = "jp.naver.line.android" self._PARSER_NAME = "Line Parser" + self._VERSION = "9.15.1" def analyze(self, dataSource, fileManager, context): try: - contact_and_message_dbs = SQLiteUtil.findAppDatabases(dataSource, "naver_line.db", True, self._LINE_PACKAGE_NAME) - calllog_dbs = SQLiteUtil.findAppDatabases(dataSource, "call_history", True, self._LINE_PACKAGE_NAME) + contact_and_message_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "naver_line", True, self._LINE_PACKAGE_NAME) + calllog_dbs = AppSQLiteDB.findAppDatabases(dataSource, + "call_history", True, self._LINE_PACKAGE_NAME) for contact_and_message_db in contact_and_message_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) + helper = AppDBParserHelper(self._PARSER_NAME, + contact_and_message_db.getDBFile(), Account.Type.LINE) contacts_parser = LineContactsParser(contact_and_message_db) while contacts_parser.next(): - blackboard_util.addContact( + helper.addContact( contacts_parser.get_account_name(), contacts_parser.get_contact_name(), contacts_parser.get_phone(), @@ -77,11 +81,10 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): contacts_parser.get_email() ) contacts_parser.close() - """ - messages_parser = LineMessagesParser(line_db) + + messages_parser = LineMessagesParser(contact_and_message_db) while messages_parser.next(): - blackboard_util.addMessage( - messages_parser.get_account_id(), + helper.addMessage( messages_parser.get_message_type(), messages_parser.get_message_direction(), messages_parser.get_phone_number_from(), @@ -93,36 +96,27 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): messages_parser.get_thread_id() ) messages_parser.close() - """ contact_and_message_db.close() for calllog_db in calllog_dbs: - blackboard_util = BlackboardUtil(self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) - calllog_db.attachDatabase(dataSource, "naver_line.db", True, calllog_db.getDBFile().getParentPath(), "naver") + helper = AppDBParserHelper(self._PARSER_NAME, + calllog_db.getDBFile(), Account.Type.LINE) + calllog_db.attachDatabase(dataSource, + "naver_line", calllog_db.getDBFile().getParentPath(), "naver") calllog_parser = LineCallLogsParser(calllog_db) while calllog_parser.next(): - print(calllog_parser.get_account_name()) - print(calllog_parser.get_contact_name()) - print(calllog_parser.get_call_direction()) - print(calllog_parser.get_phone_number_from()) - print(calllog_parser.get_phone_number_to()) - print(calllog_parser.get_call_start_date_time()) - print(calllog_parser.get_call_end_date_time()) - blackboard_util.addCalllog( - calllog_parser.get_account_name(), + helper.addCalllog( calllog_parser.get_call_direction(), calllog_parser.get_phone_number_from(), calllog_parser.get_phone_number_to(), calllog_parser.get_call_start_date_time(), calllog_parser.get_call_end_date_time(), - calllog_parser.get_contact_name() + calllog_parser.get_call_type() ) - - calllog_db.detachDatabase("naver") calllog_parser.close() - calllog_db.close() + calllog_db.close() except (SQLException, TskCoreException) as ex: # Error parsing Line databases. self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) @@ -141,47 +135,62 @@ class LineCallLogsParser(TskCallLogsParser): SELECT substr(CallH.call_type, -1) AS direction, CallH.start_time AS start_time, CallH.end_time AS end_time, - ConT.server_name AS account_name, - ConT.name AS contact_name - FROM call_history AS CallH - JOIN naver.contacts AS ConT - ON CallH.caller_mid = ConT.m_id + ConT.server_name AS name, + CallH.voip_type AS call_type, + ConT.m_id + FROM call_history AS CallH + JOIN naver.contacts AS ConT + ON CallH.caller_mid = ConT.m_id """ ) ) - self._OUTGOING_CALL = "O" - self._INCOMING_CALL = "I" + self._OUTGOING_CALL_TYPE = "O" + self._INCOMING_CALL_TYPE = "I" self._had_error = False + self._VIDEO_CALL_TYPE = "V" + self._AUDIO_CALL_TYPE = "A" - def get_call_direction(self): - direction = self.result_set.getString("direction") - if direction == self._OUTGOING_CALL: - return self.OUTGOING_MSG_STRING - return self.INCOMING_MSG_STRING + def get_call_direction(self): + direction = self.result_set.getString("direction") + if direction == self._OUTGOING_CALL_TYPE: + return self.OUTGOING_CALL + return self.INCOMING_CALL - def get_call_start_date_time(self): - start_time = self.result_set.getString("start_time") - try: - return long(start_time) / 1000 - except ValueError as ve: - self._had_error = True - print("bad_conversion") - return super(LineCallLogsParser, self).get_call_start_date_time() + def get_call_start_date_time(self): + try: + return long(self.result_set.getString("start_time")) / 1000 + except ValueError as ve: + self._had_error = True + return super(LineCallLogsParser, self).get_call_start_date_time() - def get_call_end_date_time(self): - end_time = self.result_set.getString("end_time") - try: - return long(end_time) / 1000 - except ValueError as ve: - self._had_error = True - print("bad conversion") - return super(LineCallLogsParser, self).get_call_end_date_time() + def get_call_end_date_time(self): + try: + return long(self.result_set.getString("end_time")) / 1000 + except ValueError as ve: + self._had_error = True + return super(LineCallLogsParser, self).get_call_end_date_time() + + def get_phone_number_to(self): + if self.get_call_direction() == self.OUTGOING_CALL: + return Account.Address(self.result_set.getString("m_id"), + self.result_set.getString("name")) + return super(LineCallLogsParser, self).get_phone_number_to() - def get_account_name(self): - return self.result_set.getString("account_name") + def get_phone_number_from(self): + if self.get_call_direction() == self.INCOMING_CALL: + return Account.Address(self.result_set.getString("m_id"), + self.result_set.getString("name")) + return super(LineCallLogsParser, self).get_phone_number_from() - def get_contact_name(self): - return self.result_set.getString("contact_name") + def get_call_type(self): + if self.result_set.getString("call_type") == self._VIDEO_CALL_TYPE: + return self.VIDEO_CALL + if self.result_set.getString("call_type") == self._AUDIO_CALL_TYPE: + return self.AUDIO_CALL + return super(LineCallLogsParser, self).get_call_type() + + def has_incomplete_results(self): + return self._had_error class LineContactsParser(TskContactsParser): """ @@ -193,17 +202,17 @@ class LineContactsParser(TskContactsParser): def __init__(self, contact_db): super(LineContactsParser, self).__init__(contact_db.runQuery( """ - SELECT name, + SELECT m_id, server_name FROM contacts """ ) ) def get_account_name(self): - return self.result_set.getString("server_name") + return self.result_set.getString("m_id") def get_contact_name(self): - return self.result_set.getString("name") + return self.result_set.getString("server_name") class LineMessagesParser(TskMessagesParser): """ @@ -213,23 +222,45 @@ class LineMessagesParser(TskMessagesParser): """ def __init__(self, message_db): - super().__init__(message_db.runQuery( - """SELECT created_time, content, contacts.server_name AS server_name, read_count - FROM chat_history - JOIN contacts - ON chat_history.from_mid = contacts.m_id""" - )) + super(LineMessagesParser, self).__init__(message_db.runQuery( + """ + SELECT all_contacts.name, + all_contacts.id, + all_contacts.members, + CH.from_mid, + CH.content, + CH.created_time, + CH.attachement_type, + CH.attachement_local_uri, + CH.status + FROM (SELECT G.name, + group_members.id, + group_members.members + FROM (SELECT id, + group_concat(m_id) AS members + FROM membership + GROUP BY id) AS group_members + JOIN groups AS G + ON G.id = group_members.id + UNION + SELECT server_name, + m_id, + NULL + FROM contacts) AS all_contacts + JOIN chat_history AS CH + ON CH.chat_id = all_contacts.id + """ + ) + ) self._LINE_MESSAGE_TYPE = "Line Message" + #From the limited test data, it appeared that incoming + #was only associated with a 1 status. Status # 3 and 7 + #was only associated with outgoing. + self._INCOMING_MESSAGE_TYPE = 1 self._had_error = False - def get_account_id(self): - return self.result_set.getString("server_name") - def get_message_type(self): - return self.LINE_MESSAGE_TYPE - - def get_phone_number_from(self): - return self.result_set("server_name") + return self._LINE_MESSAGE_TYPE def get_message_date_time(self): created_time = self.result_set.getString("created_time") @@ -242,6 +273,34 @@ class LineMessagesParser(TskMessagesParser): def get_message_text(self): content = self.result_set.getString("content") - if not LineContentUtil.is_text_message(content): - return "" return content + + def get_message_direction(self): + if self.result_set.getInt("status") == self._INCOMING_MESSAGE_TYPE: + return self.INCOMING + return self.OUTGOING + + def get_phone_number_from(self): + if self.get_message_direction() == self.INCOMING: + group = self.result_set.getString("members") + if group is None: + return Account.Address(self.result_set.getString("from_mid"), + self.result_set.getString("name")) + return Account.Address(self.result_set.getString("from_mid"), + self.result_set.getString("name")) + return super(LineMessagesParser, self).get_phone_number_from() + + def get_phone_number_to(self): + if self.get_message_direction() == self.OUTGOING: + group = self.result_set.getString("members") + if group is None: + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("name")) + return Account.Address(group, self.result_set.getString("name")) + return super(LineMessagesParser, self).get_phone_number_to() + + def get_thread_id(self): + members = self.result_set.getString("members") + if members is not None: + return self.result_set.getString("id") + return super(LineMessagesParser, self).get_thread_id() From 5b87544b97fe22b6bae72f8eb1c1aca52ea9c993 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Sun, 15 Sep 2019 23:29:10 -0400 Subject: [PATCH 5/8] Added attachments and removed call logs --- InternalPythonModules/android/line.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index d7cab1b852..1dc9871329 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -44,6 +44,7 @@ from org.sleuthkit.datamodel import Account from TskContactsParser import TskContactsParser from TskMessagesParser import TskMessagesParser from TskCallLogsParser import TskCallLogsParser +from general import appendAttachmentList import traceback import general @@ -114,6 +115,7 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): calllog_parser.get_call_end_date_time(), calllog_parser.get_call_type() ) + calllog_db.detachDatabase("naver") calllog_parser.close() calllog_db.close() @@ -249,6 +251,7 @@ class LineMessagesParser(TskMessagesParser): FROM contacts) AS all_contacts JOIN chat_history AS CH ON CH.chat_id = all_contacts.id + WHERE attachement_type != 6 """ ) ) @@ -273,6 +276,11 @@ class LineMessagesParser(TskMessagesParser): def get_message_text(self): content = self.result_set.getString("content") + attachment_uri = self.result_set.getString("attachement_local_uri") + if attachment_uri is not None and content is not None: + return appendAttachmentList(content, [attachment_uri]) + elif attachment_uri is not None and content is None: + return appendAttachmentList("", [attachment_uri]) return content def get_message_direction(self): From edd7fb99fd111093d5a9567f3feae448810b93b0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 19 Sep 2019 17:16:59 -0400 Subject: [PATCH 6/8] Updated line and the parser templates with the latest infra changes. Also implemented bug fixes for line --- .../android/TskCallLogsParser.py | 17 +- .../android/TskMessagesParser.py | 17 +- InternalPythonModules/android/line.py | 262 +++++++++++------- 3 files changed, 182 insertions(+), 114 deletions(-) diff --git a/InternalPythonModules/android/TskCallLogsParser.py b/InternalPythonModules/android/TskCallLogsParser.py index 8c61070693..d4e6942134 100644 --- a/InternalPythonModules/android/TskCallLogsParser.py +++ b/InternalPythonModules/android/TskCallLogsParser.py @@ -17,7 +17,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from ResultSetIterator import ResultSetIterator -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CallMediaType +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from org.sleuthkit.datamodel import Account class TskCallLogsParser(ResultSetIterator): @@ -35,15 +36,15 @@ class TskCallLogsParser(ResultSetIterator): def __init__(self, result_set): super(TskCallLogsParser, self).__init__(result_set) self._DEFAULT_STRING = "" - self._DEFAULT_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_DIRECTION = CommunicationDirection.UNKNOWN self._DEFAULT_ADDRESS = None - self._DEFAULT_CALL_TYPE = AppDBParserHelper.CallMediaType.UNKNOWN - self._DEFAULT_LONG = -1 + self._DEFAULT_CALL_TYPE = CallMediaType.UNKNOWN + self._DEFAULT_LONG = -1L - self.INCOMING_CALL = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING_CALL = AppDBParserHelper.CommunicationDirection.OUTGOING - self.AUDIO_CALL = AppDBParserHelper.CallMediaType.AUDIO - self.VIDEO_CALL = AppDBParserHelper.CallMediaType.VIDEO + self.INCOMING_CALL = CommunicationDirection.INCOMING + self.OUTGOING_CALL = CommunicationDirection.OUTGOING + self.AUDIO_CALL = CallMediaType.AUDIO + self.VIDEO_CALL = CallMediaType.VIDEO def get_call_direction(self): return self._DEFAULT_DIRECTION diff --git a/InternalPythonModules/android/TskMessagesParser.py b/InternalPythonModules/android/TskMessagesParser.py index 15c4166db7..4568a7400c 100644 --- a/InternalPythonModules/android/TskMessagesParser.py +++ b/InternalPythonModules/android/TskMessagesParser.py @@ -18,8 +18,9 @@ limitations under the License. """ from ResultSetIterator import ResultSetIterator from org.sleuthkit.datamodel import Account -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper - +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection + class TskMessagesParser(ResultSetIterator): """ Generic TSK_MESSAGE artifact template. Each of these methods @@ -35,14 +36,14 @@ class TskMessagesParser(ResultSetIterator): super(TskMessagesParser, self).__init__(result_set) self._DEFAULT_TEXT = "" self._DEFAULT_LONG = -1L - self._DEFAULT_MSG_READ_STATUS = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + self._DEFAULT_MSG_READ_STATUS = MessageReadStatus.UNKNOWN self._DEFAULT_ACCOUNT_ADDRESS = None - self._DEFAULT_COMMUNICATION_DIRECTION = AppDBParserHelper.CommunicationDirection.UNKNOWN + self._DEFAULT_COMMUNICATION_DIRECTION = CommunicationDirection.UNKNOWN - self.INCOMING = AppDBParserHelper.CommunicationDirection.INCOMING - self.OUTGOING = AppDBParserHelper.CommunicationDirection.OUTGOING - self.READ = AppDBParserHelper.MessageReadStatusEnum.READ - self.UNREAD = AppDBParserHelper.MessageReadStatusEnum.UNREAD + self.INCOMING = CommunicationDirection.INCOMING + self.OUTGOING = CommunicationDirection.OUTGOING + self.READ = MessageReadStatus.READ + self.UNREAD = MessageReadStatus.UNREAD def get_message_type(self): return self._DEFAULT_TEXT diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index 1dc9871329..f58448943f 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -16,7 +16,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - from java.io import File from java.lang import Class from java.lang import ClassNotFoundException @@ -31,8 +30,8 @@ from org.apache.commons.codec.binary import Base64 from org.sleuthkit.autopsy.casemodule import Case from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil -from org.sleuthkit.autopsy.coreutils import AppSQLiteDB -from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB + from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile @@ -40,11 +39,15 @@ from org.sleuthkit.datamodel import BlackboardArtifact from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import Content from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel.Blackboard import BlackboardException +from org.sleuthkit.autopsy.casemodule import NoCurrentCaseException from org.sleuthkit.datamodel import Account +from org.sleuthkit.datamodel.blackboardutils import CommunicationArtifactsHelper +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import MessageReadStatus +from org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper import CommunicationDirection from TskContactsParser import TskContactsParser from TskMessagesParser import TskMessagesParser from TskCallLogsParser import TskCallLogsParser -from general import appendAttachmentList import traceback import general @@ -63,67 +66,121 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): try: contact_and_message_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "naver_line", True, self._LINE_PACKAGE_NAME) + "naver_line", True, self._LINE_PACKAGE_NAME) calllog_dbs = AppSQLiteDB.findAppDatabases(dataSource, - "call_history", True, self._LINE_PACKAGE_NAME) + "call_history", True, self._LINE_PACKAGE_NAME) for contact_and_message_db in contact_and_message_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) - contacts_parser = LineContactsParser(contact_and_message_db) - while contacts_parser.next(): - helper.addContact( - contacts_parser.get_account_name(), - contacts_parser.get_contact_name(), - contacts_parser.get_phone(), - contacts_parser.get_home_phone(), - contacts_parser.get_mobile_phone(), - contacts_parser.get_email() - ) - contacts_parser.close() - - messages_parser = LineMessagesParser(contact_and_message_db) - while messages_parser.next(): - helper.addMessage( - messages_parser.get_message_type(), - messages_parser.get_message_direction(), - messages_parser.get_phone_number_from(), - messages_parser.get_phone_number_to(), - messages_parser.get_message_date_time(), - messages_parser.get_message_read_status(), - messages_parser.get_message_subject(), - messages_parser.get_message_text(), - messages_parser.get_thread_id() - ) - messages_parser.close() + self.parse_contacts(contact_and_message_db, helper) + self.parse_messages(contact_and_message_db, helper) contact_and_message_db.close() for calllog_db in calllog_dbs: - helper = AppDBParserHelper(self._PARSER_NAME, - calllog_db.getDBFile(), Account.Type.LINE) - calllog_db.attachDatabase(dataSource, - "naver_line", calllog_db.getDBFile().getParentPath(), "naver") + current_case = Case.getCurrentCaseThrows() + helper = CommunicationArtifactsHelper( + current_case.getSleuthkitCase(), self._PARSER_NAME, + calllog_db.getDBFile(), Account.Type.LINE) - calllog_parser = LineCallLogsParser(calllog_db) - while calllog_parser.next(): - helper.addCalllog( - calllog_parser.get_call_direction(), - calllog_parser.get_phone_number_from(), - calllog_parser.get_phone_number_to(), - calllog_parser.get_call_start_date_time(), - calllog_parser.get_call_end_date_time(), - calllog_parser.get_call_type() - ) - calllog_db.detachDatabase("naver") - calllog_parser.close() + calllog_db.attachDatabase( + dataSource, "naver_line", + calllog_db.getDBFile().getParentPath(), "naver") + self.parse_calllogs(calllog_db, helper) calllog_db.close() - except (SQLException, TskCoreException) as ex: + except NoCurrentCaseException as ex: # Error parsing Line databases. self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) self._logger.log(Level.WARNING, traceback.format_exc()) + def parse_contacts(self, contacts_db, helper): + try: + contacts_parser = LineContactsParser(contacts_db) + while contacts_parser.next(): + helper.addContact( + contacts_parser.get_account_name(), + contacts_parser.get_contact_name(), + contacts_parser.get_phone(), + contacts_parser.get_home_phone(), + contacts_parser.get_mobile_phone(), + contacts_parser.get_email() + ) + contacts_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing the Line App Database for contacts", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifact to case database... case is not complete. + self._logger.log(Level.SEVERE, + "Error adding Line contact artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard + self._logger.log(Level.WARNING, + "Error posting Line contact artifacts to blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_calllogs(self, calllogs_db, helper): + try: + calllog_parser = LineCallLogsParser(calllogs_db) + while calllog_parser.next(): + helper.addCalllog( + calllog_parser.get_call_direction(), + calllog_parser.get_phone_number_from(), + calllog_parser.get_phone_number_to(), + calllog_parser.get_call_start_date_time(), + calllog_parser.get_call_end_date_time(), + calllog_parser.get_call_type() + ) + calllog_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing the Line App Database for calllogs", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifact to case database... case is not complete. + self._logger.log(Level.SEVERE, + "Error adding Line calllog artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard + self._logger.log(Level.WARNING, + "Error posting Line calllog artifacts to blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + + def parse_messages(self, messages_db, helper): + try: + messages_parser = LineMessagesParser(messages_db) + while messages_parser.next(): + helper.addMessage( + messages_parser.get_message_type(), + messages_parser.get_message_direction(), + messages_parser.get_phone_number_from(), + messages_parser.get_phone_number_to(), + messages_parser.get_message_date_time(), + messages_parser.get_message_read_status(), + messages_parser.get_message_subject(), + messages_parser.get_message_text(), + messages_parser.get_thread_id() + ) + messages_parser.close() + except SQLException as ex: + self._logger.log(Level.WARNING, "Error parsing the Line App Database for messages.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + except TskCoreException as ex: + #Error adding artifact to case database... case is not complete. + self._logger.log(Level.SEVERE, + "Error adding Line message artifacts to the case database.", ex) + self._logger.log(Level.SEVERE, traceback.format_exc()) + except BlackboardException as ex: + #Error posting notification to blackboard + self._logger.log(Level.WARNING, + "Error posting Line message artifacts to blackboard.", ex) + self._logger.log(Level.WARNING, traceback.format_exc()) + class LineCallLogsParser(TskCallLogsParser): """ Parses out TSK_CALLLOG information from the Line database. @@ -148,7 +205,6 @@ class LineCallLogsParser(TskCallLogsParser): ) self._OUTGOING_CALL_TYPE = "O" self._INCOMING_CALL_TYPE = "I" - self._had_error = False self._VIDEO_CALL_TYPE = "V" self._AUDIO_CALL_TYPE = "A" @@ -162,14 +218,12 @@ class LineCallLogsParser(TskCallLogsParser): try: return long(self.result_set.getString("start_time")) / 1000 except ValueError as ve: - self._had_error = True return super(LineCallLogsParser, self).get_call_start_date_time() def get_call_end_date_time(self): try: return long(self.result_set.getString("end_time")) / 1000 except ValueError as ve: - self._had_error = True return super(LineCallLogsParser, self).get_call_end_date_time() def get_phone_number_to(self): @@ -191,9 +245,6 @@ class LineCallLogsParser(TskCallLogsParser): return self.AUDIO_CALL return super(LineCallLogsParser, self).get_call_type() - def has_incomplete_results(self): - return self._had_error - class LineContactsParser(TskContactsParser): """ Parses out TSK_CONTACT information from the Line database. @@ -225,42 +276,52 @@ class LineMessagesParser(TskMessagesParser): def __init__(self, message_db): super(LineMessagesParser, self).__init__(message_db.runQuery( - """ - SELECT all_contacts.name, - all_contacts.id, - all_contacts.members, - CH.from_mid, - CH.content, - CH.created_time, - CH.attachement_type, - CH.attachement_local_uri, - CH.status - FROM (SELECT G.name, - group_members.id, - group_members.members - FROM (SELECT id, - group_concat(m_id) AS members - FROM membership - GROUP BY id) AS group_members - JOIN groups AS G - ON G.id = group_members.id - UNION - SELECT server_name, - m_id, - NULL - FROM contacts) AS all_contacts - JOIN chat_history AS CH - ON CH.chat_id = all_contacts.id - WHERE attachement_type != 6 - """ - ) + """ + SELECT contact_list_with_groups.name, + contact_list_with_groups.id, + contact_list_with_groups.members, + contact_list_with_groups.member_names, + CH.from_mid, + C.server_name AS from_name, + CH.content, + CH.created_time, + CH.attachement_type, + CH.attachement_local_uri, + CH.status + FROM (SELECT G.name, + group_members.id, + group_members.members, + group_members.member_names + FROM (SELECT id, + group_concat(M.m_id) AS members, + group_concat(replace(C.server_name, + ",", + "")) as member_names + FROM membership AS M + JOIN contacts as C + ON M.m_id = C.m_id + GROUP BY id) AS group_members + JOIN groups AS G + ON G.id = group_members.id + UNION + SELECT server_name, + m_id, + NULL, + NULL + FROM contacts) AS contact_list_with_groups + JOIN chat_history AS CH + ON CH.chat_id = contact_list_with_groups.id + LEFT JOIN contacts as C + ON C.m_id = CH.from_mid + WHERE attachement_type != 6 + """ + ) ) self._LINE_MESSAGE_TYPE = "Line Message" #From the limited test data, it appeared that incoming #was only associated with a 1 status. Status # 3 and 7 #was only associated with outgoing. self._INCOMING_MESSAGE_TYPE = 1 - self._had_error = False def get_message_type(self): return self._LINE_MESSAGE_TYPE @@ -271,16 +332,15 @@ class LineMessagesParser(TskMessagesParser): #Get time in seconds (created_time is stored in ms from epoch) return long(created_time) / 1000 except ValueError as ve: - self._had_error = True return super(LineMessagesParser, self).get_message_date_time() def get_message_text(self): content = self.result_set.getString("content") attachment_uri = self.result_set.getString("attachement_local_uri") if attachment_uri is not None and content is not None: - return appendAttachmentList(content, [attachment_uri]) + return general.appendAttachmentList(content, [attachment_uri]) elif attachment_uri is not None and content is None: - return appendAttachmentList("", [attachment_uri]) + return general.appendAttachmentList("", [attachment_uri]) return content def get_message_direction(self): @@ -290,21 +350,27 @@ class LineMessagesParser(TskMessagesParser): def get_phone_number_from(self): if self.get_message_direction() == self.INCOMING: - group = self.result_set.getString("members") - if group is None: - return Account.Address(self.result_set.getString("from_mid"), - self.result_set.getString("name")) return Account.Address(self.result_set.getString("from_mid"), - self.result_set.getString("name")) + self.result_set.getString("from_name")) return super(LineMessagesParser, self).get_phone_number_from() def get_phone_number_to(self): if self.get_message_direction() == self.OUTGOING: group = self.result_set.getString("members") - if group is None: - return Account.Address(self.result_set.getString("id"), - self.result_set.getString("name")) - return Account.Address(group, self.result_set.getString("name")) + if group is not None: + group = group.split(",") + names = self.result_set.getString("member_names").split(",") + + recipients = [] + + for recipient_id, recipient_name in zip(group, names): + recipients.append(Account.Address(recipient_id, recipient_name)) + + return recipients + + return Account.Address(self.result_set.getString("id"), + self.result_set.getString("name")) + return super(LineMessagesParser, self).get_phone_number_to() def get_thread_id(self): From 6de1e2ab40920e7d8e1e84231b84d01ed9a73286 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:00:50 -0400 Subject: [PATCH 7/8] Fixed line bug --- InternalPythonModules/android/line.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index f58448943f..c6520edf43 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -350,8 +350,10 @@ class LineMessagesParser(TskMessagesParser): def get_phone_number_from(self): if self.get_message_direction() == self.INCOMING: - return Account.Address(self.result_set.getString("from_mid"), - self.result_set.getString("from_name")) + from_mid = self.result_set.getString("from_mid") + if from_mid is not None: + return Account.Address(from_mid, + self.result_set.getString("from_name")) return super(LineMessagesParser, self).get_phone_number_from() def get_phone_number_to(self): From 3dcab41cbd0d7e7e1a437cda88ba0f711dd39cdc Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 20 Sep 2019 10:06:49 -0400 Subject: [PATCH 8/8] Fixed sql exception and close statements --- InternalPythonModules/android/line.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/InternalPythonModules/android/line.py b/InternalPythonModules/android/line.py index c6520edf43..c87ef3477c 100644 --- a/InternalPythonModules/android/line.py +++ b/InternalPythonModules/android/line.py @@ -75,27 +75,26 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, contact_and_message_db.getDBFile(), Account.Type.LINE) - self.parse_contacts(contact_and_message_db, helper) self.parse_messages(contact_and_message_db, helper) - contact_and_message_db.close() for calllog_db in calllog_dbs: current_case = Case.getCurrentCaseThrows() helper = CommunicationArtifactsHelper( current_case.getSleuthkitCase(), self._PARSER_NAME, calllog_db.getDBFile(), Account.Type.LINE) + self.parse_calllogs(dataSource, calllog_db, helper) - calllog_db.attachDatabase( - dataSource, "naver_line", - calllog_db.getDBFile().getParentPath(), "naver") - - self.parse_calllogs(calllog_db, helper) - calllog_db.close() except NoCurrentCaseException as ex: # Error parsing Line databases. self._logger.log(Level.WARNING, "Error parsing the Line App Databases", ex) self._logger.log(Level.WARNING, traceback.format_exc()) + + for contact_and_message_db in contact_and_message_dbs: + contact_and_message_db.close() + + for calllog_db in calllog_dbs: + calllog_db.close() def parse_contacts(self, contacts_db, helper): try: @@ -124,8 +123,12 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): "Error posting Line contact artifacts to blackboard.", ex) self._logger.log(Level.WARNING, traceback.format_exc()) - def parse_calllogs(self, calllogs_db, helper): + def parse_calllogs(self, dataSource, calllogs_db, helper): try: + calllogs_db.attachDatabase( + dataSource, "naver_line", + calllogs_db.getDBFile().getParentPath(), "naver") + calllog_parser = LineCallLogsParser(calllogs_db) while calllog_parser.next(): helper.addCalllog( @@ -153,6 +156,7 @@ class LineAnalyzer(general.AndroidComponentAnalyzer): def parse_messages(self, messages_db, helper): try: + messages_parser = LineMessagesParser(messages_db) while messages_parser.next(): helper.addMessage(