mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-11 23:46:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5362 from dannysmyda/5682-simplify-app-db-queries

Browse Source 
5682 simplify app db queries
This commit is contained in:
Richard Cordovano 2019-10-23 12:46:15 -04:00 committed by GitHub
commit bc21a24caf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 137 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
InternalPythonModules/android/line.py
View File

@ -236,28 +236,23 @@ class LineCallLogsParser(TskCallLogsParser):
def __init__(self, calllog_db): def __init__(self, calllog_db):
super(LineCallLogsParser, self).__init__(calllog_db.runQuery( super(LineCallLogsParser, self).__init__(calllog_db.runQuery(
""" """
SELECT Substr(CH.call_type, -1) AS direction, SELECT Substr(calls.call_type, -1) AS direction,
CH.start_time AS start_time, calls.start_time AS start_time,
CH.end_time AS end_time, calls.end_time AS end_time,
contacts_list_with_groups.members AS group_members, contact_book_w_groups.members AS group_members,
contacts_list_with_groups.member_names AS names, calls.caller_mid,
CH.caller_mid, calls.voip_type AS call_type,
CH.voip_type AS call_type, calls.voip_gc_media_type AS group_call_type
CH.voip_gc_media_type AS group_call_type
FROM (SELECT id, FROM (SELECT id,
Group_concat(M.m_id) AS members, Group_concat(M.m_id) AS members
Group_concat(Replace(C.server_name, ",", "")) AS member_names
FROM membership AS M FROM membership AS M
JOIN naver.contacts AS C
ON M.m_id = C.m_id
GROUP BY id GROUP BY id
UNION UNION
SELECT m_id, SELECT m_id,
NULL, NULL
server_name FROM naver.contacts) AS contact_book_w_groups
FROM naver.contacts) AS contacts_list_with_groups JOIN call_history AS calls
JOIN call_history AS CH ON calls.caller_mid = contact_book_w_groups.id
ON CH.caller_mid = contacts_list_with_groups.id
""" """
) )
) )
@ -355,42 +350,24 @@ class LineMessagesParser(TskMessagesParser):
def __init__(self, message_db): def __init__(self, message_db):
super(LineMessagesParser, self).__init__(message_db.runQuery( super(LineMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT contact_list_with_groups.name, SELECT contact_book_w_groups.id,
contact_list_with_groups.id, contact_book_w_groups.members,
contact_list_with_groups.members, messages.from_mid,
contact_list_with_groups.member_names, messages.content,
CH.from_mid, messages.created_time,
C.server_name AS from_name, messages.attachement_type,
CH.content, messages.attachement_local_uri,
CH.created_time, messages.status
CH.attachement_type,
CH.attachement_local_uri,
CH.status
FROM (SELECT G.name,
group_members.id,
group_members.members,
group_members.member_names
FROM (SELECT id, FROM (SELECT id,
group_concat(M.m_id) AS members, Group_concat(M.m_id) AS members
group_concat(replace(C.server_name,
",",
"")) as member_names
FROM membership AS M FROM membership AS M
JOIN contacts as C GROUP BY id
ON M.m_id = C.m_id
GROUP BY id) AS group_members
JOIN groups AS G
ON G.id = group_members.id
UNION UNION
SELECT server_name, SELECT m_id,
m_id,
NULL,
NULL NULL
FROM contacts) AS contact_list_with_groups FROM contacts) AS contact_book_w_groups
JOIN chat_history AS CH JOIN chat_history AS messages
ON CH.chat_id = contact_list_with_groups.id ON messages.chat_id = contact_book_w_groups.id
LEFT JOIN contacts as C
ON C.m_id = CH.from_mid
WHERE attachement_type != 6 WHERE attachement_type != 6
""" """
) )

129
InternalPythonModules/android/skype.py
View File

@ -76,11 +76,8 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer):
as they would be excluded in the join. Since the chatItem table stores both the as they would be excluded in the join. Since the chatItem table stores both the
group id or skype_id in one column, an implementation decision was made to union group id or skype_id in one column, an implementation decision was made to union
the person and particiapnt table together so that all rows are matched in one join the person and particiapnt table together so that all rows are matched in one join
with chatItem. This result is consistently labeled contact_list_with_groups in the with chatItem. This result is consistently labeled contact_book_w_groups in the
following queries. following queries.
- In order to keep the formatting of the name consistent throughout each query,
a _format_user_name() function was created to encapsulate the CASE statement
that was being shared across them. Refer to the method for more details.
""" """
def __init__(self): def __init__(self):
@ -93,7 +90,12 @@ class SkypeAnalyzer(general.AndroidComponentAnalyzer):
account_query_result = skype_db.runQuery( account_query_result = skype_db.runQuery(
""" """
SELECT entry_id, SELECT entry_id,
"""+_format_user_name()+""" AS name CASE
WHEN Ifnull(first_name, "") == "" AND Ifnull(last_name, "") == "" THEN entry_id
WHEN first_name is NULL THEN replace(last_name, ",", "")
WHEN last_name is NULL THEN replace(first_name, ",", "")
ELSE replace(first_name, ",", "") || " " || replace(last_name, ",", "")
END AS name
FROM user FROM user
""" """
) )
@ -251,14 +253,6 @@ class SkypeCallLogsParser(TskCallLogsParser):
def __init__(self, calllog_db): def __init__(self, calllog_db):
""" """
Big picture:
The query below creates a contacts_list_with_groups table, which
represents the recipient info. A chatItem record holds ids for
both the recipient and sender. The first join onto chatItem fills
in the blanks for the recipients. The second join back onto person
handles the sender info. The result is a table with all of the
communication details.
Implementation details: Implementation details:
- message_type w/ value 3 appeared to be the call type, regardless - message_type w/ value 3 appeared to be the call type, regardless
of if it was audio or video. of if it was audio or video.
@ -266,36 +260,22 @@ class SkypeCallLogsParser(TskCallLogsParser):
""" """
super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery( super(SkypeCallLogsParser, self).__init__(calllog_db.runQuery(
""" """
SELECT contacts_list_with_groups.conversation_id, SELECT contact_book_w_groups.conversation_id,
contacts_list_with_groups.participant_ids, contact_book_w_groups.participant_ids,
contacts_list_with_groups.participants, messages.time,
time, messages.duration,
duration, messages.is_sender_me,
is_sender_me, messages.person_id AS sender_id
person_id as sender_id,
sender_name.name as sender_name
FROM (SELECT conversation_id, FROM (SELECT conversation_id,
Group_concat(person_id) AS participant_ids, Group_concat(person_id) AS participant_ids
Group_concat("""+_format_user_name()+""") AS participants FROM particiapnt
FROM particiapnt AS PART
JOIN person AS P
ON PART.person_id = P.entry_id
GROUP BY conversation_id GROUP BY conversation_id
UNION UNION
SELECT entry_id, SELECT entry_id AS conversation_id,
NULL, NULL
"""+_format_user_name()+""" AS participant FROM person) AS contact_book_w_groups
FROM person) AS contacts_list_with_groups join chatitem AS messages
JOIN chatitem AS C ON messages.conversation_link = contact_book_w_groups.conversation_id
ON C.conversation_link = contacts_list_with_groups.conversation_id
JOIN (SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM person
UNION
SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM user) AS sender_name
ON sender_name.id = C.person_id
WHERE message_type == 3 WHERE message_type == 3
""" """
) )
@ -347,7 +327,12 @@ class SkypeContactsParser(TskContactsParser):
super(SkypeContactsParser, self).__init__(contact_db.runQuery( super(SkypeContactsParser, self).__init__(contact_db.runQuery(
""" """
SELECT entry_id, SELECT entry_id,
"""+_format_user_name()+""" AS name CASE
WHEN Ifnull(first_name, "") == "" AND Ifnull(last_name, "") == "" THEN entry_id
WHEN first_name is NULL THEN replace(last_name, ",", "")
WHEN last_name is NULL THEN replace(first_name, ",", "")
ELSE replace(first_name, ",", "") || " " || replace(last_name, ",", "")
END AS name
FROM person FROM person
""" """
) )
@ -379,37 +364,23 @@ class SkypeMessagesParser(TskMessagesParser):
""" """
super(SkypeMessagesParser, self).__init__(message_db.runQuery( super(SkypeMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT contacts_list_with_groups.conversation_id, SELECT contact_book_w_groups.conversation_id,
contacts_list_with_groups.participant_ids, contact_book_w_groups.participant_ids,
contacts_list_with_groups.participants, messages.time,
time, messages.content,
content, messages.device_gallery_path,
device_gallery_path, messages.is_sender_me,
is_sender_me, messages.person_id as sender_id
person_id as sender_id,
sender_name.name AS sender_name
FROM (SELECT conversation_id, FROM (SELECT conversation_id,
Group_concat(person_id) AS participant_ids, Group_concat(person_id) AS participant_ids
Group_concat("""+_format_user_name()+""") AS participants FROM particiapnt
FROM particiapnt AS PART
JOIN person AS P
ON PART.person_id = P.entry_id
GROUP BY conversation_id GROUP BY conversation_id
UNION UNION
SELECT entry_id as conversation_id, SELECT entry_id as conversation_id,
NULL, NULL
"""+_format_user_name()+""" AS participant FROM person) AS contact_book_w_groups
FROM person) AS contacts_list_with_groups JOIN chatitem AS messages
JOIN chatitem AS C ON messages.conversation_link = contact_book_w_groups.conversation_id
ON C.conversation_link = contacts_list_with_groups.conversation_id
JOIN (SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM person
UNION
SELECT entry_id as id,
"""+_format_user_name()+""" AS name
FROM user) AS sender_name
ON sender_name.id = C.person_id
WHERE message_type != 3 WHERE message_type != 3
""" """
) )
@ -469,25 +440,3 @@ class SkypeMessagesParser(TskMessagesParser):
if group_ids is not None: if group_ids is not None:
return self.result_set.getString("conversation_id") return self.result_set.getString("conversation_id")
return super(SkypeMessagesParser, self).get_thread_id() return super(SkypeMessagesParser, self).get_thread_id()
def _format_user_name():
"""
This CASE SQL statement is used in many queries to
format the names of users. For a user, there is a first_name
column and a last_name column. Some of these columns can be null
and our goal is to produce the cleanest data possible. In the event
that both the first and last name columns are null, we return the skype_id
which is stored in the database as 'entry_id'. Commas are removed from the name
so that we can concatenate names into a comma seperate list for group chats.
"""
return """
CASE
WHEN Ifnull(first_name, "") == "" AND Ifnull(last_name, "") == "" THEN entry_id
WHEN first_name is NULL THEN replace(last_name, ",", "")
WHEN last_name is NULL THEN replace(first_name, ",", "")
ELSE replace(first_name, ",", "") || " " || replace(last_name, ",", "")
END
"""

62
InternalPythonModules/android/textnow.py
View File

@ -290,39 +290,37 @@ class TextNowMessagesParser(TskMessagesParser):
""" """
super(TextNowMessagesParser, self).__init__(message_db.runQuery( super(TextNowMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT CASE SELECT CASE
WHEN message_direction == 2 THEN "" WHEN messages.message_direction == 2 THEN NULL
WHEN to_addresses IS NULL THEN M.contact_value WHEN contact_book_w_groups.to_addresses IS NULL THEN
ELSE contact_name messages.contact_value
end from_address, END from_address,
CASE CASE
WHEN message_direction == 1 THEN "" WHEN messages.message_direction == 1 THEN NULL
WHEN to_addresses IS NULL THEN M.contact_value WHEN contact_book_w_groups.to_addresses IS NULL THEN
ELSE to_addresses messages.contact_value
end to_address, ELSE contact_book_w_groups.to_addresses
message_direction, END to_address,
message_text, messages.message_direction,
M.READ, messages.message_text,
M.date, messages.READ,
M.attach, messages.DATE,
messages.attach,
thread_id thread_id
FROM (SELECT group_info.contact_value,
group_info.to_addresses,
G.contact_value AS thread_id
FROM (SELECT GM.contact_value, FROM (SELECT GM.contact_value,
Group_concat(GM.member_contact_value) AS to_addresses Group_concat(GM.member_contact_value) AS to_addresses,
G.contact_value AS thread_id
FROM group_members AS GM FROM group_members AS GM
GROUP BY GM.contact_value) AS group_info join GROUPS AS G
JOIN groups AS G ON G.contact_value = GM.contact_value
ON G.contact_value = group_info.contact_value GROUP BY GM.contact_value
UNION UNION
SELECT c.contact_value, SELECT contact_value,
NULL, NULL,
"-1" NULL
FROM contacts AS c) AS to_from_map FROM contacts) AS contact_book_w_groups
JOIN messages AS M join messages
ON M.contact_value = to_from_map.contact_value ON messages.contact_value = contact_book_w_groups.contact_value
WHERE message_type NOT IN ( 102, 100 ) WHERE message_type NOT IN ( 102, 100 )
""" """
) )
@ -330,13 +328,12 @@ class TextNowMessagesParser(TskMessagesParser):
self._TEXTNOW_MESSAGE_TYPE = "TextNow Message" self._TEXTNOW_MESSAGE_TYPE = "TextNow Message"
self._INCOMING_MESSAGE_TYPE = 1 self._INCOMING_MESSAGE_TYPE = 1
self._OUTGOING_MESSAGE_TYPE = 2 self._OUTGOING_MESSAGE_TYPE = 2
self._UNKNOWN_THREAD_ID = "-1"
def get_message_type(self): def get_message_type(self):
return self._TEXTNOW_MESSAGE_TYPE return self._TEXTNOW_MESSAGE_TYPE
def get_phone_number_from(self): def get_phone_number_from(self):
if self.result_set.getString("from_address") == "": if self.result_set.getString("from_address") is None:
return super(TextNowMessagesParser, self).get_phone_number_from() return super(TextNowMessagesParser, self).get_phone_number_from()
return self.result_set.getString("from_address") return self.result_set.getString("from_address")
@ -347,10 +344,9 @@ class TextNowMessagesParser(TskMessagesParser):
return self.OUTGOING return self.OUTGOING
def get_phone_number_to(self): def get_phone_number_to(self):
if self.result_set.getString("to_address") == "": if self.result_set.getString("to_address") is None:
return super(TextNowMessagesParser, self).get_phone_number_to() return super(TextNowMessagesParser, self).get_phone_number_to()
recipients = self.result_set.getString("to_address").split(",") return self.result_set.getString("to_address").split(",")
return recipients
def get_message_date_time(self): def get_message_date_time(self):
#convert ms to s #convert ms to s
@ -359,7 +355,7 @@ class TextNowMessagesParser(TskMessagesParser):
def get_message_read_status(self): def get_message_read_status(self):
read = self.result_set.getBoolean("read") read = self.result_set.getBoolean("read")
if self.get_message_direction() == self.INCOMING: if self.get_message_direction() == self.INCOMING:
if read == True: if read:
return self.READ return self.READ
return self.UNREAD return self.UNREAD
@ -375,6 +371,6 @@ class TextNowMessagesParser(TskMessagesParser):
def get_thread_id(self): def get_thread_id(self):
thread_id = self.result_set.getString("thread_id") thread_id = self.result_set.getString("thread_id")
if thread_id == self._UNKNOWN_THREAD_ID: if thread_id is None:
return super(TextNowMessagesParser, self).get_thread_id() return super(TextNowMessagesParser, self).get_thread_id()
return thread_id return thread_id

35
InternalPythonModules/android/whatsapp.py
View File

@ -433,31 +433,28 @@ class WhatsAppMessagesParser(TskMessagesParser):
def __init__(self, message_db): def __init__(self, message_db):
super(WhatsAppMessagesParser, self).__init__(message_db.runQuery( super(WhatsAppMessagesParser, self).__init__(message_db.runQuery(
""" """
SELECT M.key_remote_jid AS id, SELECT messages.key_remote_jid AS id,
contact_info.recipients, contact_book_w_groups.recipients,
key_from_me AS direction, key_from_me AS direction,
CASE messages.data AS content,
WHEN M.data IS NULL THEN "" messages.timestamp AS send_timestamp,
ELSE M.data messages.received_timestamp,
END AS content, messages.remote_resource AS group_sender,
M.timestamp AS send_timestamp, messages.media_url AS attachment
M.received_timestamp,
M.remote_resource AS group_sender,
M.media_url As attachment
FROM (SELECT jid, FROM (SELECT jid,
recipients recipients
FROM wadb.wa_contacts AS WC FROM wadb.wa_contacts AS contacts
LEFT JOIN (SELECT gjid, left join (SELECT gjid,
group_concat(CASE Group_concat(CASE
WHEN jid == "" THEN NULL WHEN jid == "" THEN NULL
ELSE jid ELSE jid
END) AS recipients END) AS recipients
FROM group_participants FROM group_participants
GROUP BY gjid) AS group_map GROUP BY gjid) AS groups
ON WC.jid = group_map.gjid ON contacts.jid = groups.gjid
GROUP BY jid) AS contact_info GROUP BY jid) AS contact_book_w_groups
JOIN messages AS M join messages
ON M.key_remote_jid = contact_info.jid ON messages.key_remote_jid = contact_book_w_groups.jid
""" """
) )
) )
@ -503,6 +500,8 @@ class WhatsAppMessagesParser(TskMessagesParser):
def get_message_text(self): def get_message_text(self):
message = self.result_set.getString("content") message = self.result_set.getString("content")
if message is None:
message = super(WhatsAppMessagesParser, self).get_message_text()
attachment = self.result_set.getString("attachment") attachment = self.result_set.getString("attachment")
if attachment is not None: if attachment is not None:
return general.appendAttachmentList(message, [attachment]) return general.appendAttachmentList(message, [attachment])