mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5362 from dannysmyda/5682-simplify-app-db-queries

Browse Source 
5682 simplify app db queries
This commit is contained in:
Richard Cordovano 2019-10-23 12:46:15 -04:00 committed by GitHub
commit bc21a24caf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 137 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
InternalPythonModules/android/line.py
View File

@ -236,28 +236,23 @@ class LineCallLogsParser(TskCallLogsParser):
def __init__(self, calllog_db): def __init__(self, calllog_db):
super(LineCallLogsParser, self).__init__(calllog_db.runQuery( super(LineCallLogsParser, self).__init__(calllog_db.runQuery(
""" """
SELECT Substr(CH.call_type, -1) AS direction, SELECT Substr(calls.call_type, -1) AS direction,
CH.start_time AS start_time, calls.start_time AS start_time,
CH.end_time AS end_time, calls.end_time AS end_time,
contacts_list_with_groups.members AS group_members, contact_book_w_groups.members AS group_members,
contacts_list_with_groups.member_names AS names, calls.caller_mid,
CH.caller_mid, calls.voip_type AS call_type,
CH.voip_type AS call_type, calls.voip_gc_media_type AS group_call_type
CH.voip_gc_media_type AS group_call_type
FROM (SELECT id, FROM (SELECT id,
Group_concat(M.m_id) AS members, Group_concat(M.m_id) AS members
Group_concat(Replace(C.server_name, ",", "")) AS member_names
FROM membership AS M FROM membership AS M
JOIN naver.contacts AS C
ON M.m_id = C.m_id
GROUP BY id GROUP BY id
UNION UNION
SELECT m_id, SELECT m_id,
NULL, NULL
server_name FROM naver.contacts) AS contact_book_w_groups
FROM naver.contacts) AS contacts_list_with_groups JOIN call_history AS calls
JOIN call_history AS CH ON calls.caller_mid = contact_book_w_groups.id
ON CH.caller_mid = contacts_list_with_groups.id
""" """
) )
) )
@ -355,43 +350,25 @@ class LineMessagesParser(TskMessagesParser):
def __init__(self, message_db): def __init__(self, message_db):
super(LineMessagesParser, self).__init__(message_db.runQuery( super(LineMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT contact_list_with_groups.name, SELECT contact_book_w_groups.id,
contact_list_with_groups.id, contact_book_w_groups.members,
contact_list_with_groups.members, messages.from_mid,
contact_list_with_groups.member_names, messages.content,
CH.from_mid, messages.created_time,
C.server_name AS from_name, messages.attachement_type,
CH.content, messages.attachement_local_uri,
CH.created_time, messages.status
CH.attachement_type, FROM (SELECT id,
CH.attachement_local_uri, Group_concat(M.m_id) AS members
CH.status FROM membership AS M
FROM (SELECT G.name, GROUP BY id
group_members.id, UNION
group_members.members, SELECT m_id,
group_members.member_names NULL
FROM (SELECT id, FROM contacts) AS contact_book_w_groups
group_concat(M.m_id) AS members, JOIN chat_history AS messages
group_concat(replace(C.server_name, ON messages.chat_id = contact_book_w_groups.id
",", WHERE attachement_type != 6
"")) as member_names
FROM membership AS M
JOIN contacts as C
ON M.m_id = C.m_id
GROUP BY id) AS group_members
JOIN groups AS G
ON G.id = group_members.id
UNION
SELECT server_name,
m_id,
NULL,
NULL
FROM contacts) AS contact_list_with_groups
JOIN chat_history AS CH
ON CH.chat_id = contact_list_with_groups.id
LEFT JOIN contacts as C
ON C.m_id = CH.from_mid
WHERE attachement_type != 6
""" """
) )
) )

141
InternalPythonModules/android/skype.py
View File

@ -76,11 +76,8 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer):
as they would be excluded in the join. Since the chatItem table stores both the as they would be excluded in the join. Since the chatItem table stores both the
group id or skype_id in one column, an implementation decision was made to union group id or skype_id in one column, an implementation decision was made to union
the person and particiapnt table together so that all rows are matched in one join the person and particiapnt table together so that all rows are matched in one join
with chatItem. This result is consistently labeled contact_list_with_groups in the with chatItem. This result is consistently labeled contact_book_w_groups in the
following queries. following queries.
- In order to keep the formatting of the name consistent throughout each query,
a _format_user_name() function was created to encapsulate the CASE statement
that was being shared across them. Refer to the method for more details.
""" """
def __init__(self): def __init__(self):
@ -93,7 +90,12 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer):
account_query_result = skype_db.runQuery( account_query_result = skype_db.runQuery(
""" """
SELECT entry_id, SELECT entry_id,
"""+_format_user_name()+""" AS name CASE
WHEN Ifnull(first_name, "") == "" AND Ifnull(last_name, "") == "" THEN entry_id
WHEN first_name is NULL THEN replace(last_name, ",", "")
WHEN last_name is NULL THEN replace(first_name, ",", "")
ELSE replace(first_name, ",", "") || " " || replace(last_name, ",", "")
END AS name
FROM user FROM user
""" """
) )
@ -251,14 +253,6 @@ class SkypeCallLogsParser(TskCallLogsParser):
def __init__(self, calllog_db): def __init__(self, calllog_db):
""" """
Big picture:
The query below creates a contacts_list_with_groups table, which
represents the recipient info. A chatItem record holds ids for
both the recipient and sender. The first join onto chatItem fills
in the blanks for the recipients. The second join back onto person
handles the sender info. The result is a table with all of the
communication details.
Implementation details: Implementation details:
- message_type w/ value 3 appeared to be the call type, regardless - message_type w/ value 3 appeared to be the call type, regardless
of if it was audio or video. of if it was audio or video.
@ -266,37 +260,23 @@ class SkypeCallLogsParser(TskCallLogsParser):
""" """
super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery( super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery(
""" """
SELECT contacts_list_with_groups.conversation_id, SELECT contact_book_w_groups.conversation_id,
contacts_list_with_groups.participant_ids, contact_book_w_groups.participant_ids,
contacts_list_with_groups.participants, messages.time,
time, messages.duration,
duration, messages.is_sender_me,
is_sender_me, messages.person_id AS sender_id
person_id as sender_id,
sender_name.name as sender_name
FROM (SELECT conversation_id, FROM (SELECT conversation_id,
Group_concat(person_id) AS participant_ids, Group_concat(person_id) AS participant_ids
Group_concat("""+_format_user_name()+""") AS participants FROM particiapnt
FROM particiapnt AS PART
JOIN person AS P
ON PART.person_id = P.entry_id
GROUP BY conversation_id GROUP BY conversation_id
UNION UNION
SELECT entry_id, SELECT entry_id AS conversation_id,
NULL, NULL
"""+_format_user_name()+""" AS participant FROM person) AS contact_book_w_groups
FROM person) AS contacts_list_with_groups join chatitem AS messages
JOIN chatitem AS C ON messages.conversation_link = contact_book_w_groups.conversation_id
ON C.conversation_link = contacts_list_with_groups.conversation_id WHERE message_type == 3
JOIN (SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM person
UNION
SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM user) AS sender_name
ON sender_name.id = C.person_id
WHERE message_type == 3
""" """
) )
) )
@ -347,7 +327,12 @@ class SkypeContactsParser(TskContactsParser):
super(SkypeContactsParser, self).__init__(contact_db.runQuery( super(SkypeContactsParser, self).__init__(contact_db.runQuery(
""" """
SELECT entry_id, SELECT entry_id,
"""+_format_user_name()+""" AS name CASE
WHEN Ifnull(first_name, "") == "" AND Ifnull(last_name, "") == "" THEN entry_id
WHEN first_name is NULL THEN replace(last_name, ",", "")
WHEN last_name is NULL THEN replace(first_name, ",", "")
ELSE replace(first_name, ",", "") || " " || replace(last_name, ",", "")
END AS name
FROM person FROM person
""" """
) )
@ -379,39 +364,25 @@ class SkypeMessagesParser(TskMessagesParser):
""" """
super(SkypeMessagesParser, self).__init__(message_db.runQuery( super(SkypeMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT contacts_list_with_groups.conversation_id, SELECT contact_book_w_groups.conversation_id,
contacts_list_with_groups.participant_ids, contact_book_w_groups.participant_ids,
contacts_list_with_groups.participants, messages.time,
time, messages.content,
content, messages.device_gallery_path,
device_gallery_path, messages.is_sender_me,
is_sender_me, messages.person_id as sender_id
person_id as sender_id, FROM (SELECT conversation_id,
sender_name.name AS sender_name Group_concat(person_id) AS participant_ids
FROM (SELECT conversation_id, FROM particiapnt
Group_concat(person_id) AS participant_ids, GROUP BY conversation_id
Group_concat("""+_format_user_name()+""") AS participants UNION
FROM particiapnt AS PART SELECT entry_id as conversation_id,
JOIN person AS P NULL
ON PART.person_id = P.entry_id FROM person) AS contact_book_w_groups
GROUP BY conversation_id JOIN chatitem AS messages
UNION ON messages.conversation_link = contact_book_w_groups.conversation_id
SELECT entry_id as conversation_id,
NULL,
"""+_format_user_name()+""" AS participant
FROM person) AS contacts_list_with_groups
JOIN chatitem AS C
ON C.conversation_link = contacts_list_with_groups.conversation_id
JOIN (SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM person
UNION
SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM user) AS sender_name
ON sender_name.id = C.person_id
WHERE message_type != 3 WHERE message_type != 3
""" """
) )
) )
self._SKYPE_MESSAGE_TYPE = "Skype Message" self._SKYPE_MESSAGE_TYPE = "Skype Message"
@ -469,25 +440,3 @@ class SkypeMessagesParser(TskMessagesParser):
if group_ids is not None: if group_ids is not None:
return self.result_set.getString("conversation_id") return self.result_set.getString("conversation_id")
return super(SkypeMessagesParser, self).get_thread_id() return super(SkypeMessagesParser, self).get_thread_id()
def _format_user_name():
"""
This CASE SQL statement is used in many queries to
format the names of users. For a user, there is a first_name
column and a last_name column. Some of these columns can be null
and our goal is to produce the cleanest data possible. In the event
that both the first and last name columns are null, we return the skype_id
which is stored in the database as 'entry_id'. Commas are removed from the name
so that we can concatenate names into a comma seperate list for group chats.
"""
return """
CASE
WHEN Ifnull(first_name, "") == "" AND Ifnull(last_name, "") == "" THEN entry_id
WHEN first_name is NULL THEN replace(last_name, ",", "")
WHEN last_name is NULL THEN replace(first_name, ",", "")
ELSE replace(first_name, ",", "") || " " || replace(last_name, ",", "")
END
"""

78
InternalPythonModules/android/textnow.py
View File

@ -290,53 +290,50 @@ class TextNowMessagesParser(TskMessagesParser):
""" """
super(TextNowMessagesParser, self).__init__(message_db.runQuery( super(TextNowMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT CASE
SELECT CASE WHEN messages.message_direction == 2 THEN NULL
WHEN message_direction == 2 THEN "" WHEN contact_book_w_groups.to_addresses IS NULL THEN
WHEN to_addresses IS NULL THEN M.contact_value messages.contact_value
ELSE contact_name END from_address,
end from_address, CASE
CASE WHEN messages.message_direction == 1 THEN NULL
WHEN message_direction == 1 THEN "" WHEN contact_book_w_groups.to_addresses IS NULL THEN
WHEN to_addresses IS NULL THEN M.contact_value messages.contact_value
ELSE to_addresses ELSE contact_book_w_groups.to_addresses
end to_address, END to_address,
message_direction, messages.message_direction,
message_text, messages.message_text,
M.READ, messages.READ,
M.date, messages.DATE,
M.attach, messages.attach,
thread_id thread_id
FROM (SELECT group_info.contact_value, FROM (SELECT GM.contact_value,
group_info.to_addresses, Group_concat(GM.member_contact_value) AS to_addresses,
G.contact_value AS thread_id G.contact_value AS thread_id
FROM (SELECT GM.contact_value, FROM group_members AS GM
Group_concat(GM.member_contact_value) AS to_addresses join GROUPS AS G
FROM group_members AS GM ON G.contact_value = GM.contact_value
GROUP BY GM.contact_value) AS group_info GROUP BY GM.contact_value
JOIN groups AS G UNION
ON G.contact_value = group_info.contact_value SELECT contact_value,
UNION NULL,
SELECT c.contact_value, NULL
NULL, FROM contacts) AS contact_book_w_groups
"-1" join messages
FROM contacts AS c) AS to_from_map ON messages.contact_value = contact_book_w_groups.contact_value
JOIN messages AS M WHERE message_type NOT IN ( 102, 100 )
ON M.contact_value = to_from_map.contact_value
WHERE message_type NOT IN ( 102, 100 )
""" """
) )
) )
self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" self._TEXTNOW_MESSAGE_TYPE = "TextNow Message"
self._INCOMING_MESSAGE_TYPE = 1 self._INCOMING_MESSAGE_TYPE = 1
self._OUTGOING_MESSAGE_TYPE = 2 self._OUTGOING_MESSAGE_TYPE = 2
self._UNKNOWN_THREAD_ID = "-1"
def get_message_type(self): def get_message_type(self):
return self._TEXTNOW_MESSAGE_TYPE return self._TEXTNOW_MESSAGE_TYPE
def get_phone_number_from(self): def get_phone_number_from(self):
if self.result_set.getString("from_address") == "": if self.result_set.getString("from_address") is None:
return super(TextNowMessagesParser, self).get_phone_number_from() return super(TextNowMessagesParser, self).get_phone_number_from()
return self.result_set.getString("from_address") return self.result_set.getString("from_address")
@ -347,10 +344,9 @@ class TextNowMessagesParser(TskMessagesParser):
return self.OUTGOING return self.OUTGOING
def get_phone_number_to(self): def get_phone_number_to(self):
if self.result_set.getString("to_address") == "": if self.result_set.getString("to_address") is None:
return super(TextNowMessagesParser, self).get_phone_number_to() return super(TextNowMessagesParser, self).get_phone_number_to()
recipients = self.result_set.getString("to_address").split(",") return self.result_set.getString("to_address").split(",")
return recipients
def get_message_date_time(self): def get_message_date_time(self):
#convert ms to s #convert ms to s
@ -359,7 +355,7 @@ class TextNowMessagesParser(TskMessagesParser):
def get_message_read_status(self): def get_message_read_status(self):
read = self.result_set.getBoolean("read") read = self.result_set.getBoolean("read")
if self.get_message_direction() == self.INCOMING: if self.get_message_direction() == self.INCOMING:
if read == True: if read:
return self.READ return self.READ
return self.UNREAD return self.UNREAD
@ -375,6 +371,6 @@ class TextNowMessagesParser(TskMessagesParser):
def get_thread_id(self): def get_thread_id(self):
thread_id = self.result_set.getString("thread_id") thread_id = self.result_set.getString("thread_id")
if thread_id == self._UNKNOWN_THREAD_ID: if thread_id is None:
return super(TextNowMessagesParser, self).get_thread_id() return super(TextNowMessagesParser, self).get_thread_id()
return thread_id return thread_id

49
InternalPythonModules/android/whatsapp.py
View File

@ -433,31 +433,28 @@ class WhatsAppMessagesParser(TskMessagesParser):
def __init__(self, message_db): def __init__(self, message_db):
super(WhatsAppMessagesParser, self).__init__(message_db.runQuery( super(WhatsAppMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT M.key_remote_jid AS id, SELECT messages.key_remote_jid AS id,
contact_info.recipients, contact_book_w_groups.recipients,
key_from_me AS direction, key_from_me AS direction,
CASE messages.data AS content,
WHEN M.data IS NULL THEN "" messages.timestamp AS send_timestamp,
ELSE M.data messages.received_timestamp,
END AS content, messages.remote_resource AS group_sender,
M.timestamp AS send_timestamp, messages.media_url AS attachment
M.received_timestamp, FROM (SELECT jid,
M.remote_resource AS group_sender, recipients
M.media_url As attachment FROM wadb.wa_contacts AS contacts
FROM (SELECT jid, left join (SELECT gjid,
recipients Group_concat(CASE
FROM wadb.wa_contacts AS WC WHEN jid == "" THEN NULL
LEFT JOIN (SELECT gjid, ELSE jid
group_concat(CASE END) AS recipients
WHEN jid == "" THEN NULL FROM group_participants
ELSE jid GROUP BY gjid) AS groups
END) AS recipients ON contacts.jid = groups.gjid
FROM group_participants GROUP BY jid) AS contact_book_w_groups
GROUP BY gjid) AS group_map join messages
ON WC.jid = group_map.gjid ON messages.key_remote_jid = contact_book_w_groups.jid
GROUP BY jid) AS contact_info
JOIN messages AS M
ON M.key_remote_jid = contact_info.jid
""" """
) )
) )
@ -503,6 +500,8 @@ class WhatsAppMessagesParser(TskMessagesParser):
def get_message_text(self): def get_message_text(self):
message = self.result_set.getString("content") message = self.result_set.getString("content")
if message is None:
message = super(WhatsAppMessagesParser, self).get_message_text()
attachment = self.result_set.getString("attachment") attachment = self.result_set.getString("attachment")
if attachment is not None: if attachment is not None:
return general.appendAttachmentList(message, [attachment]) return general.appendAttachmentList(message, [attachment])