From c31044a6579ce3fd4fb5b0f6a36cbfcc429bd5fd Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 18 Sep 2018 12:10:30 -0400 Subject: [PATCH] Fixed the temp directory file collisions during reading and refactored the localDiskPath out --- .../tabulardatareader/AbstractReader.java | 41 +++- .../tabulardatareader/ExcelReader.java | 8 +- .../tabulardatareader/FileReaderFactory.java | 36 +-- .../tabulardatareader/SQLiteReader.java | 222 ++++++++++-------- 4 files changed, 184 insertions(+), 123 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java index f37bcf08ab..69e37cd825 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java @@ -22,8 +22,10 @@ import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.datamodel.ContentUtils; -import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; /** @@ -32,10 +34,17 @@ import org.sleuthkit.datamodel.TskCoreException; */ public abstract class AbstractReader implements AutoCloseable { - public AbstractReader(AbstractFile file, String localDiskPath) + private final String localDiskPath; + + public AbstractReader(Content file) throws FileReaderInitException { - writeDataSourceToLocalDisk(file, localDiskPath); + try { + localDiskPath = getLocalDiskPath(file); + writeDataSourceToLocalDisk(file); + } catch (FileReaderInitException ex) { + throw new FileReaderInitException(ex); + } } /** @@ -48,7 +57,7 @@ public abstract class AbstractReader implements AutoCloseable { * @throws NoCurrentCaseException Current case closed during file copying * @throws TskCoreException Exception finding files from abstract file */ - private void writeDataSourceToLocalDisk(AbstractFile file, String localDiskPath) + private void writeDataSourceToLocalDisk(Content file) throws FileReaderInitException { try { @@ -61,6 +70,30 @@ public abstract class AbstractReader implements AutoCloseable { } } + public String getLocalDiskPath() { + return localDiskPath; + } + + /** + * Generates a local disk path for abstract file contents to be copied. All + * file sources must be copied to local disk to be opened by abstract + * reader. + * + * @param file The database abstract file + * + * @return Valid local path for copying + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException + * + */ + private String getLocalDiskPath(Content file) throws FileReaderInitException { + try { + return Case.getCurrentCaseThrows().getTempDirectory() + + File.separator + file.getId() + file.getName(); + } catch(NoCurrentCaseException ex) { + throw new FileReaderInitException("No current case open when trying to get temp directory", ex); + } + } + /** * Return the a mapping of table names to table schemas (may be in the form of * headers or create table statements for databases). diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index 7eb4be7f8f..effd43d929 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -37,9 +37,9 @@ import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.datamodel.AbstractFile; import com.monitorjbl.xlsx.StreamingReader; import org.apache.poi.hssf.OldExcelFormatException; +import org.sleuthkit.datamodel.Content; /** * Reads excel files and implements the abstract reader api for interfacing with @@ -58,10 +58,10 @@ public final class ExcelReader extends AbstractReader { private String LOCAL_DISK_PATH; private String ACTIVE_MIME_TYPE; - public ExcelReader(AbstractFile file, String localDiskPath, String mimeType) + public ExcelReader(Content file, String mimeType) throws FileReaderInitException { - super(file, localDiskPath); - this.LOCAL_DISK_PATH = localDiskPath; + super(file); + this.LOCAL_DISK_PATH = super.getLocalDiskPath(); this.ACTIVE_MIME_TYPE = mimeType; try { diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java index e6af1673b2..173773aac8 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java @@ -19,40 +19,44 @@ package org.sleuthkit.autopsy.tabulardatareader; import org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException; -import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; /** - * Factory for creating the correct reader given the mime type of a file. + * Factory for creating the correct reader given the mime type of a file. */ public final class FileReaderFactory { - + private FileReaderFactory() { } + /** - * Instantiates the appropriate reader given the mimeType argument. Currently - * supports SQLite files and Excel files (.xls and .xlsx). BIFF5 format of .xls - * is not supported. - * - * @param mimeType mimeType passed in from the ingest module -g * @param file current file under inspection - * @param localDiskPath path for abstract file contents to be written + * Instantiates the appropriate reader given the mimeType argument. + * Currently supports SQLite files and Excel files (.xls and .xlsx). BIFF5 + * format of .xls is not supported. + * + * @param mimeType mimeType passed in from the ingest module g * @param file + * current file under inspection + * + * @param file Content file to be copied into + * * @return The correct reader class needed to read the file contents - * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException + * + * @throws + * org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException */ - public static AbstractReader createReader(String mimeType, AbstractFile file, - String localDiskPath) throws FileReaderInitException { + public static AbstractReader createReader(Content file, String mimeType) throws FileReaderInitException { switch (mimeType) { case "application/x-sqlite3": - return new SQLiteReader(file, localDiskPath); + return new SQLiteReader(file); case "application/vnd.ms-excel": case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": try { - return new ExcelReader(file, localDiskPath, mimeType); + return new ExcelReader(file, mimeType); //Catches runtime exceptions being emitted from Apache //POI (such as EncryptedDocumentException) and wraps them //into FileReaderInitException to be caught and logged //in the ingest module. - } catch(Exception poiInitException) { + } catch (Exception poiInitException) { throw new FileReaderInitException(poiInitException); } default: diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index cb67bdacc1..a1a065dd13 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -43,6 +43,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -50,152 +51,166 @@ import org.sleuthkit.datamodel.TskCoreException; * Reads sqlite databases and returns results in a list collection. */ public final class SQLiteReader extends AbstractReader { - + private final Connection connection; private final static IngestServices services = IngestServices.getInstance(); private final static Logger logger = services.getLogger(SQLiteReader.class.getName()); - + /** * Writes data source file contents to local disk and opens a sqlite JDBC - * connection. - * - * @param sqliteDbFile Data source abstract file - * @param localDiskPath Location for database contents to be copied to - * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException + * connection. + * + * @param sqliteDbFile Data source content + * + * @throws + * org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException */ - public SQLiteReader(AbstractFile sqliteDbFile, String localDiskPath) throws FileReaderInitException { - super(sqliteDbFile, localDiskPath); + public SQLiteReader(Content sqliteDbFile) throws FileReaderInitException { + super(sqliteDbFile); try { // Look for any meta files associated with this DB - WAL, SHM, etc. findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-wal"); findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-shm"); - connection = getDatabaseConnection(localDiskPath); - } catch (ClassNotFoundException | SQLException |IOException | - NoCurrentCaseException | TskCoreException ex) { + connection = getDatabaseConnection(super.getLocalDiskPath()); + } catch (ClassNotFoundException | SQLException | IOException + | NoCurrentCaseException | TskCoreException ex) { throw new FileReaderInitException(ex); } } - + /** - * Searches for a meta file associated with the give SQLite database. If found, - * copies the file to the local disk folder - * - * @param sqliteFile file being processed + * Searches for a meta file associated with the give SQLite database. If + * found, copies the file to the local disk folder + * + * @param sqliteFile file being processed * @param metaFileName name of meta file to look for + * * @throws NoCurrentCaseException Case has been closed. - * @throws TskCoreException fileManager cannot find AbstractFile files. - * @throws IOException Issue during writing to file. + * @throws TskCoreException fileManager cannot find AbstractFile + * files. + * @throws IOException Issue during writing to file. */ - private void findAndCopySQLiteMetaFile(AbstractFile sqliteFile, + private void findAndCopySQLiteMetaFile(Content sqliteFile, String metaFileName) throws NoCurrentCaseException, TskCoreException, IOException { - + Case openCase = Case.getCurrentCaseThrows(); SleuthkitCase sleuthkitCase = openCase.getSleuthkitCase(); Services services = new Services(sleuthkitCase); FileManager fileManager = services.getFileManager(); - + List metaFiles = fileManager.findFiles( - sqliteFile.getDataSource(), metaFileName, + sqliteFile.getDataSource(), metaFileName, sqliteFile.getParent().getName()); - + if (metaFiles != null) { for (AbstractFile metaFile : metaFiles) { - String tmpMetafilePathName = openCase.getTempDirectory() + - File.separator + metaFile.getName(); + String tmpMetafilePathName = openCase.getTempDirectory() + + File.separator + metaFile.getId() + metaFile.getName(); File tmpMetafile = new File(tmpMetafilePathName); ContentUtils.writeToFile(metaFile, tmpMetafile); } } } - + /** * Opens a JDBC connection to the sqlite database specified by the path * parameter. - * + * * @param databasePath Local path of sqlite database - * @return Connection JDBC connection, to be maintained and closed by the reader + * + * @return Connection JDBC connection, to be maintained and closed by the + * reader + * * @throws ClassNotFoundException missing SQLite JDBC class - * @throws SQLException Exception during opening database connection + * @throws SQLException Exception during opening database + * connection */ - private Connection getDatabaseConnection(String databasePath) + private Connection getDatabaseConnection(String databasePath) throws ClassNotFoundException, SQLException { - + // Load the SQLite JDBC driver, if necessary. Class.forName("org.sqlite.JDBC"); //NON-NLS return DriverManager.getConnection( "jdbc:sqlite:" + databasePath); //NON-NLS } - - + /** * Retrieves a map view of table names to table schemas (in the form of * CREATE TABLE statments). - * + * * @return A map of table names to table schemas - * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + * + * @throws + * org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override public Map getTableSchemas() throws FileReaderException { - + Map dbTablesMap = new TreeMap<>(); - + try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( - "SELECT name, sql FROM sqlite_master " //NON-NLS - + " WHERE type= 'table' " //NON-NLS - + " ORDER BY name;")){ //NON-NLS - - while (resultSet.next()) { - String tableName = resultSet.getString("name"); //NON-NLS - String tableSQL = resultSet.getString("sql"); //NON-NLS - dbTablesMap.put(tableName, tableSQL); - } - + "SELECT name, sql FROM sqlite_master " //NON-NLS + + " WHERE type= 'table' " //NON-NLS + + " ORDER BY name;")) { //NON-NLS + + while (resultSet.next()) { + String tableName = resultSet.getString("name"); //NON-NLS + String tableSQL = resultSet.getString("sql"); //NON-NLS + dbTablesMap.put(tableName, tableSQL); + } + } catch (SQLException ex) { throw new FileReaderException(ex); } - + return dbTablesMap; } - + /** * Retrieves the total number of rows from a table in the SQLite database. - * + * * @param tableName + * * @return Row count from tableName - * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + * + * @throws + * org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override - public Integer getRowCountFromTable(String tableName) + public Integer getRowCountFromTable(String tableName) throws FileReaderException { String quotedTableName = wrapTableNameStringWithQuotes(tableName); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( - "SELECT count (*) as count FROM " + quotedTableName)){ //NON-NLS + "SELECT count (*) as count FROM " + quotedTableName)) { //NON-NLS return resultSet.getInt("count"); //NON-NLS } catch (SQLException ex) { throw new FileReaderException(ex); } } - + /** - * Retrieves all rows from a given table in the SQLite database. If only a + * Retrieves all rows from a given table in the SQLite database. If only a * subset of rows are desired, see the overloaded function below. - * + * * @param tableName - * @return List of rows, where each row is - * represented as a column-value map. - * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + * + * @return List of rows, where each row is represented as a column-value + * map. + * + * @throws + * org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override - public List> getRowsFromTable(String tableName) + public List> getRowsFromTable(String tableName) throws FileReaderException { //This method does not directly call its overloaded counterpart //since the second parameter would need to be retreived from a call to //getTableRowCount(). String quotedTableName = wrapTableNameStringWithQuotes(tableName); - try(Statement statement = connection.createStatement(); + try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( "SELECT * FROM " + quotedTableName)) { //NON-NLS return resultSetToList(resultSet); @@ -203,24 +218,27 @@ public final class SQLiteReader extends AbstractReader { throw new FileReaderException(ex); } } - + /** * Retrieves a subset of the rows from a given table in the SQLite database. - * + * * @param tableName - * @param offset Desired start index (rows begin at 1) + * @param offset Desired start index (rows begin at 1) * @param numRowsToRead Number of rows past the start index - * @return List of rows, where each row is - * represented as a column-value map. - * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + * + * @return List of rows, where each row is represented as a column-value + * map. + * + * @throws + * org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override - public List> getRowsFromTable(String tableName, - int offset, int numRowsToRead) throws FileReaderException{ + public List> getRowsFromTable(String tableName, + int offset, int numRowsToRead) throws FileReaderException { String quotedTableName = wrapTableNameStringWithQuotes(tableName); - try(Statement statement = connection.createStatement(); + try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( - "SELECT * FROM " + quotedTableName //NON-NLS + "SELECT * FROM " + quotedTableName //NON-NLS + " LIMIT " + Integer.toString(numRowsToRead) //NON-NLS + " OFFSET " + Integer.toString(offset - 1))) { //NON-NLS return resultSetToList(resultSet); @@ -228,31 +246,34 @@ public final class SQLiteReader extends AbstractReader { throw new FileReaderException(ex); } } - + /** - * Wraps table name with quotation marks in case table name contains spaces. - * sqliteJDBC cannot read table names with spaces in them unless surrounded + * Wraps table name with quotation marks in case table name contains spaces. + * sqliteJDBC cannot read table names with spaces in them unless surrounded * by quotation marks. - * + * * @param tableName + * * @return Input name: Result Table -> "Result Table" */ private String wrapTableNameStringWithQuotes(String tableName) { - return "\"" + tableName +"\""; + return "\"" + tableName + "\""; } - + /** - * Converts a ResultSet (row results from a table read) into a list. - * + * Converts a ResultSet (row results from a table read) into a list. + * * @param resultSet row results from a table read - * @return List of rows, where each row is - * represented as a column-value map. - * @throws SQLException occurs if ResultSet is closed while attempting to - * access it's data. + * + * @return List of rows, where each row is represented as a column-value + * map. + * + * @throws SQLException occurs if ResultSet is closed while attempting to + * access it's data. */ @NbBundle.Messages("SQLiteReader.BlobNotShown.message=BLOB Data not shown") private List> resultSetToList(ResultSet resultSet) throws SQLException { - + ResultSetMetaData metaData = resultSet.getMetaData(); int columns = metaData.getColumnCount(); List> rowMap = new ArrayList<>(); @@ -274,33 +295,36 @@ public final class SQLiteReader extends AbstractReader { return rowMap; } - + /** - * Returns a column view of the table. Maps the column name to a list of that - * column's values. - * + * Returns a column view of the table. Maps the column name to a list of + * that column's values. + * * @param tableName + * * @return - * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + * + * @throws + * org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override - public Map> getColumnsFromTable(String tableName) + public Map> getColumnsFromTable(String tableName) throws FileReaderException { - + String quotedTableName = wrapTableNameStringWithQuotes(tableName); - try(Statement statement = connection.createStatement(); + try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( "SELECT * FROM " + quotedTableName)) { //NON-NLS - + Map> columnView = new HashMap<>(); ResultSetMetaData metaData = resultSet.getMetaData(); int columns = metaData.getColumnCount(); - for(int i = 1; i <= columns; i++) { + for (int i = 1; i <= columns; i++) { columnView.put(metaData.getColumnName(i), new LinkedList<>()); } - + while (resultSet.next()) { - for(int i = 1; i <= columns; i++) { + for (int i = 1; i <= columns; i++) { if (resultSet.getObject(i) == null) { columnView.get(metaData.getColumnName(i)).add(""); } else { @@ -314,7 +338,7 @@ public final class SQLiteReader extends AbstractReader { } } } - + return columnView; } catch (SQLException ex) { throw new FileReaderException(ex); @@ -322,7 +346,7 @@ public final class SQLiteReader extends AbstractReader { } /** - * Closes underlying JDBC connection. + * Closes underlying JDBC connection. */ @Override public void close() {