Merge pull request #5152 from raman-bt/5394-dbparser-helper

5394: Implement utility class supporting mobile app parsers and first…
This commit is contained in:
Richard Cordovano 2019-09-12 17:44:53 -04:00 committed by GitHub
commit 5ab6d5e56f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 1795 additions and 1 deletions

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,326 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.coreutils;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.logging.Level;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.casemodule.services.Services;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/**
* An abstraction around an SQLite app DB found in a data source.
* This class makes a copy of it, along with any meta files (WAL, SHM),
* opens a SQLite connection to it, and runs queries on it.
*/
public final class AppSQLiteDB implements Closeable {
private final Logger logger = Logger.getLogger(AppSQLiteDB.class.getName());
private final AbstractFile dbAbstractFile; // AbstractFile for the DB file
private final Connection connection;
private final Statement statement;
/**
* Class to abstract the abstract file for a DB file and its on disk copy
*
*/
private static final class AppSQLiteDBFileBundle {
private final AbstractFile dbAbstractFile;
private final File dbFileCopy;
AppSQLiteDBFileBundle(AbstractFile dbAbstractFile, File dbFileCopy) {
this.dbAbstractFile = dbAbstractFile;
this.dbFileCopy = dbFileCopy;
}
AbstractFile getAbstractFile() {
return dbAbstractFile;
}
File getFileCopy() {
return dbFileCopy;
}
}
private AppSQLiteDB(AppSQLiteDBFileBundle appSQLiteDBFileBundle) throws ClassNotFoundException, SQLException {
this.dbAbstractFile = appSQLiteDBFileBundle.getAbstractFile();
Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver
connection = DriverManager.getConnection("jdbc:sqlite:" + appSQLiteDBFileBundle.getFileCopy().getPath()); //NON-NLS
statement = connection.createStatement();
}
/**
* Looks for the given SQLIte database filename, with matching path substring.
* It looks for exact name or a pattern match based on a input parameter.
* It makes a copy of each matching file, and creates an instance of
* AppSQLiteDB to help query the DB.
*
* A list of AppSQLiteDB instances is returned, one for each
* match found.,
* .
* @param dataSource data source to search in
* @param dbName db file name to search
* @param matchExactName whether to look for exact file name or a pattern match
* @param parentPathSubstr path substring to match
*
* @return A list of abstract files matching the specified name and path.
* Returns an empty list if no matching database is found.
*/
public static Collection<AppSQLiteDB> findAppDatabases(DataSource dataSource,
String dbName, boolean matchExactName, String parentPathSubstr) {
List<AppSQLiteDB> appDbs = new ArrayList<> ();
try {
Collection<AppSQLiteDBFileBundle> dbFileBundles = findAndCopySQLiteDB( dataSource, dbName, matchExactName, parentPathSubstr, false);
dbFileBundles.forEach((dbFileBundle) -> {
try {
AppSQLiteDB appSQLiteDB = new AppSQLiteDB(dbFileBundle);
appDbs.add(appSQLiteDB);
} catch (ClassNotFoundException | SQLException ex) {
Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Failed to open a DB connection for file = '%s' and path = '%s'.", dbFileBundle.dbAbstractFile.getName(), dbFileBundle.getFileCopy().getPath()), ex); //NON-NLS
}
});
} catch (TskCoreException ex) {
Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error finding App database files with name = '%s' and path = '%s'.", dbName, parentPathSubstr), ex); //NON-NLS
}
return appDbs;
}
public AbstractFile getDBFile() {
return this.dbAbstractFile;
}
/**
* Attaches a database to the current connection.
*
* Finds the specified database file in the specified folder.
* If found, makes copy of the database in the case folder and
* run ATTACH DATABASE sql.
*
* @param dataSource data source in which to look file the db file
* @param dbName name of db file to look for
* @param dbPath path in which to look for the db file
* @param dbAlias alias name to attach the database as
*
* @return abstract file for the matching db file.
* null if no match is found.
*
* @throws SQLException in case of an SQL error
*/
public AbstractFile attachDatabase(DataSource dataSource, String dbName,
String dbPath, String dbAlias) throws SQLException {
try {
// find and copy DB files with exact name and path.
Collection<AppSQLiteDBFileBundle> dbFileBundles = findAndCopySQLiteDB(dataSource, dbName, true, dbPath, true);
if (!dbFileBundles.isEmpty()) {
AppSQLiteDBFileBundle dbFileBundle = dbFileBundles.iterator().next();
String attachDbSql = String.format("ATTACH DATABASE '%s' AS '%s'", dbFileBundle.getFileCopy().getPath(), dbAlias); //NON-NLS
statement.executeUpdate(attachDbSql);
return dbFileBundle.getAbstractFile();
}
} catch (TskCoreException ex) {
Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error attaching to App database files with name = '%s' and path = '%s'.", dbName, dbPath), ex); //NON-NLS
}
return null;
}
/**
* Finds database file with the specified name, makes a copy of the file in the case directory,
* and returns the AbstractFile as well as the file copy.
*
* @param dataSource data source to search in
* @param dbName db file name to search
* @param matchExactName whether to look for exact file name or a pattern match
* @param dbPath path to match
* @param matchExactName whether to look for exact path name or a substring match
*
* @return a collection of AppSQLiteDBFileBundle
*
* @throws TskCoreException
*/
private static Collection<AppSQLiteDBFileBundle> findAndCopySQLiteDB(DataSource dataSource, String dbName,
boolean matchExactName, String dbPath, boolean matchExactPath) throws TskCoreException {
Case openCase;
try {
openCase = Case.getCurrentCaseThrows();
} catch (NoCurrentCaseException ex) {
throw new TskCoreException("Failed to get current case.", ex);
}
List<AppSQLiteDBFileBundle> dbFileBundles = new ArrayList<> ();
long fileId = 0;
String localDiskPath = "";
SleuthkitCase skCase = openCase.getSleuthkitCase();
String parentPath = dbPath.replace("\\", "/");
parentPath = SleuthkitCase.escapeSingleQuotes(parentPath);
String whereClause;
if (matchExactName) {
whereClause = String.format("LOWER(name) = LOWER('%s')", dbName);
} else {
whereClause = String.format("LOWER(name) LIKE LOWER('%%%s%%') AND LOWER(name) NOT LIKE LOWER('%%journal%%')", dbName );
}
if (matchExactPath) {
whereClause += String.format(" AND LOWER(parent_path) = LOWER('%s')", parentPath );
} else {
whereClause += String.format(" AND LOWER(parent_path) LIKE LOWER('%%%s%%')", parentPath );
}
whereClause += String.format(" AND data_source_obj_id = %s", dataSource.getId());
List<AbstractFile> absFiles = skCase.findAllFilesWhere(whereClause);
for (AbstractFile absFile : absFiles) {
try {
localDiskPath = openCase.getTempDirectory()
+ File.separator + absFile.getId() + absFile.getName();
File jFile = new java.io.File(localDiskPath);
fileId = absFile.getId();
ContentUtils.writeToFile(absFile, jFile);
//Find and copy both WAL and SHM meta files
findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-wal");
findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-shm");
AppSQLiteDBFileBundle dbFileBundle = new AppSQLiteDBFileBundle(absFile, jFile);
dbFileBundles.add(dbFileBundle);
} catch (ReadContentInputStream.ReadContentInputStreamException ex) {
Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.WARNING, String.format("Error reading content from file '%s' (id=%d).", absFile.getName(), fileId), ex); //NON-NLS
} catch (IOException | NoCurrentCaseException | TskCoreException ex) {
Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error creating AppSQLiteDB for file '%s' (id=%d) to copied to '%s'.", absFile.getName(), fileId, localDiskPath), ex); //NON-NLS
}
}
return dbFileBundles;
}
/**
* Detaches the specified database from the connection
*
* @param dbAlias alias for database to detach
*
* @throws SQLException
*/
public void detachDatabase(String dbAlias) throws SQLException {
String detachDbSql = String.format("DETACH DATABASE '%s'", dbAlias);
statement.executeUpdate(detachDbSql); //NON-NLS
}
/**
* Runs the given query on the database and returns result set.
* @param queryStr SQL string for the query to run
*
* @return ResultSet from running the query.
*
* @throws SQLException in case of an error.
*
*/
public ResultSet runQuery(String queryStr) throws SQLException {
ResultSet resultSet = null;
if (null != queryStr) {
resultSet = statement.executeQuery(queryStr); //NON-NLS
}
return resultSet;
}
/**
* Closes the DB connection
*
* @throws IOException
*/
@Override
public void close() throws IOException {
// Close the DB connection
try {
statement.close();
connection.close();
} catch (SQLException e) {
logger.log(Level.SEVERE, "Error closing the database", e); //NON-NLS
}
}
/**
* Searches for a meta file associated with the give SQLite database. If
* found, it copies this file into the temp directory of the current case.
*
* @param sqliteFile file being processed
* @param metaFileName name of meta file to look for
*
* @throws NoCurrentCaseException Case has been closed.
* @throws TskCoreException fileManager cannot find AbstractFile
* files.
* @throws IOException Issue during writing to file.
*/
private static void findAndCopySQLiteMetaFile(AbstractFile sqliteFile,
String metaFileName) throws NoCurrentCaseException, TskCoreException, IOException {
Case openCase = Case.getCurrentCaseThrows();
SleuthkitCase sleuthkitCase = openCase.getSleuthkitCase();
Services services = new Services(sleuthkitCase);
FileManager fileManager = services.getFileManager();
List<AbstractFile> metaFiles = fileManager.findFiles(
sqliteFile.getDataSource(), metaFileName,
sqliteFile.getParent().getName());
if (metaFiles != null) {
for (AbstractFile metaFile : metaFiles) {
String localDiskPath = openCase.getTempDirectory()
+ File.separator + sqliteFile.getId() + metaFile.getName();
File localMetaFile = new File(localDiskPath);
if (!localMetaFile.exists()) {
ContentUtils.writeToFile(metaFile, localMetaFile);
}
}
}
}
}

View File

@ -0,0 +1,143 @@
"""
Autopsy Forensic Browser
Copyright 2019 Basis Technology Corp.
Contact: carrier <at> sleuthkit <dot> org
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from java.io import File
from java.lang import Class
from java.lang import ClassNotFoundException
from java.lang import Long
from java.lang import String
from java.sql import ResultSet
from java.sql import SQLException
from java.sql import Statement
from java.util.logging import Level
from java.util import ArrayList
from org.apache.commons.codec.binary import Base64
from org.sleuthkit.autopsy.casemodule import Case
from org.sleuthkit.autopsy.coreutils import Logger
from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil
from org.sleuthkit.autopsy.coreutils import AppSQLiteDB
from org.sleuthkit.autopsy.coreutils import AppDBParserHelper
from org.sleuthkit.autopsy.coreutils.AppDBParserHelper import MessageReadStatusEnum
from org.sleuthkit.autopsy.coreutils.AppDBParserHelper import CommunicationDirection
from org.sleuthkit.autopsy.datamodel import ContentUtils
from org.sleuthkit.autopsy.ingest import IngestJobContext
from org.sleuthkit.datamodel import AbstractFile
from org.sleuthkit.datamodel import BlackboardArtifact
from org.sleuthkit.datamodel import BlackboardAttribute
from org.sleuthkit.datamodel import Content
from org.sleuthkit.datamodel import TskCoreException
from org.sleuthkit.datamodel import Account
import traceback
import general
"""
Finds the SQLite DB for IMO, parses the DB for contacts & messages,
and adds artifacts to the case.
"""
class IMOAnalyzer(general.AndroidComponentAnalyzer):
def __init__(self):
self._logger = Logger.getLogger(self.__class__.__name__)
def analyze(self, dataSource, fileManager, context):
selfAccountAddress = None
accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", True, "com.imo.android.imous")
for accountDb in accountDbs:
try:
accountResultSet = accountDb.runQuery("SELECT uid, name FROM account")
if accountResultSet:
# We can determine the IMO user ID of the device owner.
# Therefore we can create and use a app account and use that
# as a 'self' account instead of a Device account
if not selfAccountAddress:
selfAccountAddress = Account.Address(accountResultSet.getString("uid"), accountResultSet.getString("name"))
except SQLException as ex:
self._logger.log(Level.SEVERE, "Error processing query result for account", ex)
finally:
accountDb.close()
friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", True, "com.imo.android.imous")
for friendsDb in friendsDbs:
try:
friendsDBHelper = AppDBParserHelper("IMO Parser", friendsDb.getDBFile(),
Account.Type.IMO, Account.Type.IMO, selfAccountAddress )
contactsResultSet = friendsDb.runQuery("SELECT buid, name FROM friends")
if contactsResultSet is not None:
while contactsResultSet.next():
friendsDBHelper.addContact( contactsResultSet.getString("buid"), ## unique id for account
contactsResultSet.getString("name"), ## contact name
"", ## phone
"", ## home phone
"", ## mobile
"") ## email
queryString = "SELECT messages.buid AS buid, imdata, last_message, timestamp, message_type, message_read, name FROM messages "\
"INNER JOIN friends ON friends.buid = messages.buid"
messagesResultSet = friendsDb.runQuery(queryString)
if messagesResultSet is not None:
while messagesResultSet.next():
direction = ""
fromAddress = None
toAddress = None
name = messagesResultSet.getString("name")
uniqueId = messagesResultSet.getString("buid")
if (messagesResultSet.getInt("message_type") == 1):
direction = CommunicationDirection.INCOMING
fromAddress = Account.Address(uniqueId, name)
else:
direction = CommunicationDirection.OUTGOING
toAddress = Account.Address(uniqueId, name)
message_read = messagesResultSet.getInt("message_read")
if (message_read == 1):
msgReadStatus = MessageReadStatusEnum.READ
elif (message_read == 0):
msgReadStatus = MessageReadStatusEnum.UNREAD
else:
msgReadStatus = MessageReadStatusEnum.UNKNOWN
timeStamp = messagesResultSet.getLong("timestamp") / 1000000000
messageArtifact = friendsDBHelper.addMessage(
"IMO Message",
direction,
fromAddress,
toAddress,
timeStamp,
msgReadStatus,
"", # subject
messagesResultSet.getString("last_message"),
"") # thread id
# TBD: parse the imdata JSON structure to figure out if there is an attachment.
# If one exists, add the attachment as a derived file and a child of the message artifact.
except SQLException as ex:
self._logger.log(Level.WARNING, "Error processing query result for IMO friends", ex)
except TskCoreException as ex:
self._logger.log(Level.WARNING, "Failed to create AppDBParserHelper for adding artifacts.", ex)
finally:
friendsDb.close()

View File

@ -46,6 +46,7 @@ import googlemaplocation
import tangomessage
import textmessage
import wwfmessage
import imo
class AndroidModuleFactory(IngestModuleFactoryAdapter):
@ -87,7 +88,10 @@ class AndroidIngestModule(DataSourceIngestModule):
errors = []
fileManager = Case.getCurrentCase().getServices().getFileManager()
analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), cachelocation.CacheLocationAnalyzer()]
analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(),
tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(),
googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(),
cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()]
self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers")
progressBar.switchToDeterminate(len(analyzers))