From cb7d20d1fed1cfbd647182222fe906d5aa3cf8f8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 6 Aug 2018 14:27:58 -0400 Subject: [PATCH 01/33] Setup necessary framework for creating the new module --- .../InterestingDatabasesIngestModule.java | 46 +++++++++++++ ...terestingDatabasesIngestModuleFactory.java | 66 +++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100755 Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java create mode 100755 Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java new file mode 100755 index 0000000000..a474b72ced --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java @@ -0,0 +1,46 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.interestingdatabases; + +import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.datamodel.AbstractFile; + +/** + * + * @author dsmyda + */ +public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter{ + + @Override + public void startUp(IngestJobContext context) throws IngestModuleException { + + } + + @Override + public ProcessResult process(AbstractFile file) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void shutDown() { + + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java new file mode 100755 index 0000000000..9e6b8bf9c6 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java @@ -0,0 +1,66 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.interestingdatabases; + +import org.sleuthkit.autopsy.coreutils.Version; +import org.openide.util.NbBundle; +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.ingest.FileIngestModule; +import org.sleuthkit.autopsy.ingest.IngestModuleFactory; +import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; + +/** + * + * @author dsmyda + */ + +@NbBundle.Messages({ + "InterestingDatabasesIngestModule.FlagDatabases.moduleName=Interesting Databases", + "InterestingDatabasesIngestModule.FlagDatabases.moduleDesc.text=Flags databases with interesting items (emails, phone numbers, gps coordinates, ip/mac addresses)" +}) +@ServiceProvider(service = IngestModuleFactory.class) +public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactoryAdapter { + + @Override + public String getModuleDisplayName() { + return Bundle.InterestingDatabasesIngestModule_FlagDatabases_moduleName(); + } + + @Override + public String getModuleDescription() { + return Bundle.InterestingDatabasesIngestModule_FlagDatabases_moduleDesc_text(); + } + + @Override + public String getModuleVersionNumber() { + return Version.getVersion(); + } + + @Override + public boolean isFileIngestModuleFactory() { + return true; + } + + @Override + public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { + return new InterestingDatabasesIngestModule(); + } + +} From bc4a432f94a9d3bad19b7fff9f30d260e573b732 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 6 Aug 2018 18:01:56 -0400 Subject: [PATCH 02/33] Fleshed out some functionality, but have dirty and untested changes --- .../InterestingDatabasesIngestModule.java | 98 ++++++++++++++++++- ...terestingDatabasesIngestModuleFactory.java | 9 +- 2 files changed, 100 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java index a474b72ced..749944b90d 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java @@ -18,29 +18,119 @@ */ package org.sleuthkit.autopsy.modules.interestingdatabases; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; +import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; /** * * @author dsmyda */ -public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter{ +public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter { + private static final String SUPPORTED_MIME_TYPE = "application/x-sqlite3"; + + private final IngestServices services = IngestServices.getInstance(); + private final Logger logger = services.getLogger( + InterestingDatabasesIngestModuleFactory.getModuleName()); + private FileTypeDetector fileTypeDetector; + + private String localDiskPath; + private SQLiteReader sqliteReader; + + /** + * + * @param context + * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException + */ + @NbBundle.Messages({ + "CannotRunFileTypeDetection=Unable to initialize file type detection.", + }) @Override public void startUp(IngestJobContext context) throws IngestModuleException { - + try { + fileTypeDetector = new FileTypeDetector(); + } catch (FileTypeDetector.FileTypeDetectorInitException ex) { + throw new IngestModuleException(Bundle.CannotRunFileTypeDetection(), ex); + } } + /** + * + * @param file + * @return + */ @Override public ProcessResult process(AbstractFile file) { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + String dataSourceMimeType = fileTypeDetector.getMIMEType(file); + if(SUPPORTED_MIME_TYPE.equals(dataSourceMimeType)) { + + if(successfulDependencyInitialization(file)) { + + } else { + return ProcessResult.ERROR; + } + } + + return ProcessResult.OK; } + private boolean successfulDependencyInitialization(AbstractFile file) { + return createLocalDiskPath(file) == ProcessResult.OK && + initalizeSQLiteReader(file) == ProcessResult.OK; + } + + private ProcessResult createLocalDiskPath(AbstractFile file) { + try { + localDiskPath = Case.getCurrentCaseThrows() + .getTempDirectory() + File.separator + file.getName(); + } catch (NoCurrentCaseException ex) { + // TODO -- personal note log about current case being closed or + //current case not existing. + return ProcessResult.ERROR; + } + + return ProcessResult.OK; + } + + private ProcessResult initalizeSQLiteReader(AbstractFile file) { + try { + sqliteReader = new SQLiteReader(file, localDiskPath); + return ProcessResult.OK; + } catch (ClassNotFoundException ex) { + //TODO add logging messages for this. + } catch (SQLException ex) { + //TODO add logging messages for this. + } catch (IOException ex) { + //TODO add logging messages for this. + } catch (NoCurrentCaseException ex) { + //TODO add logging messages for this. + } catch (TskCoreException ex) { + //TODO add logging messages for this. + } + return ProcessResult.ERROR; + } + + /** + * + */ @Override public void shutDown() { } - + + private void createArtifact() { + + } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java index 9e6b8bf9c6..56b46d099a 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java @@ -38,11 +38,15 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; @ServiceProvider(service = IngestModuleFactory.class) public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactoryAdapter { - @Override - public String getModuleDisplayName() { + static String getModuleName() { return Bundle.InterestingDatabasesIngestModule_FlagDatabases_moduleName(); } + @Override + public String getModuleDisplayName() { + return getModuleName(); + } + @Override public String getModuleDescription() { return Bundle.InterestingDatabasesIngestModule_FlagDatabases_moduleDesc_text(); @@ -62,5 +66,4 @@ public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactory public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { return new InterestingDatabasesIngestModule(); } - } From 8c8b3483d9ded7079565f37aac9e57da1ee20ea2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 8 Aug 2018 10:19:24 -0400 Subject: [PATCH 03/33] Added necessary framework for module to operate in Autopsy, still need to clean up logging and add email validator. Also swallowed SQLiteException in close function of sqlite reader --- .../interestingdatabases/CellType.java | 29 +++ .../CellTypeDetector.java | 33 ++++ .../InterestingDatabasesIngestModule.java | 184 ++++++++++++++---- ...terestingDatabasesIngestModuleFactory.java | 8 +- .../autopsy/sqlitereader/SQLiteReader.java | 17 +- 5 files changed, 226 insertions(+), 45 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellType.java create mode 100755 Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellTypeDetector.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellType.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellType.java new file mode 100755 index 0000000000..d491fcfadf --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellType.java @@ -0,0 +1,29 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.interestingdatabases; + +/** + * + * @author dsmyda + */ +public enum CellType { + EMAIL, + NOT_INTERESTING +} + diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellTypeDetector.java new file mode 100755 index 0000000000..e3301bccb8 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellTypeDetector.java @@ -0,0 +1,33 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.interestingdatabases; + +/** + * + * @author dsmyda + */ +public class CellTypeDetector { + + public CellTypeDetector() { + } + + public CellType getType(String cell) { + return CellType.EMAIL; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java index 749944b90d..b3c9d78ea6 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java @@ -21,16 +21,28 @@ package org.sleuthkit.autopsy.modules.interestingdatabases; import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.logging.Level; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.services.Blackboard; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; /** @@ -40,14 +52,15 @@ import org.sleuthkit.datamodel.TskCoreException; public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter { private static final String SUPPORTED_MIME_TYPE = "application/x-sqlite3"; + private static final String MODULE_NAME = InterestingDatabasesIngestModuleFactory.getModuleName(); private final IngestServices services = IngestServices.getInstance(); private final Logger logger = services.getLogger( InterestingDatabasesIngestModuleFactory.getModuleName()); private FileTypeDetector fileTypeDetector; + private CellTypeDetector cellTypeDetector; - private String localDiskPath; - private SQLiteReader sqliteReader; + private Blackboard blackboard; /** * @@ -61,6 +74,7 @@ public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter { public void startUp(IngestJobContext context) throws IngestModuleException { try { fileTypeDetector = new FileTypeDetector(); + cellTypeDetector = new CellTypeDetector(); } catch (FileTypeDetector.FileTypeDetectorInitException ex) { throw new IngestModuleException(Bundle.CannotRunFileTypeDetection(), ex); } @@ -72,54 +86,150 @@ public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter { * @return */ @Override + @NbBundle.Messages({ + "InterestingDatabasesIngestModule.indexError.message=Failed to index interesting artifact hit for keyword search." + }) public ProcessResult process(AbstractFile file) { + try { + blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return ProcessResult.ERROR; + } + String dataSourceMimeType = fileTypeDetector.getMIMEType(file); if(SUPPORTED_MIME_TYPE.equals(dataSourceMimeType)) { - if(successfulDependencyInitialization(file)) { - - } else { + String localDiskPath; + try { + localDiskPath = Case.getCurrentCaseThrows() + .getTempDirectory() + File.separator + file.getName(); + } catch (NoCurrentCaseException ex) { + // TODO -- personal note log about current case being closed or + //current case not existing. + Exceptions.printStackTrace(ex); return ProcessResult.ERROR; } + + + try (SQLiteReader sqliteReader = + new SQLiteReader(file, localDiskPath)){ + + Set aggregatedCellTypes = cellTypesInDatabase(sqliteReader); + String cellTypesMessage = aggregatedCellTypes.toString() + .replace("]", "") + .replace("[", ""); + + try { + BlackboardArtifact artifact = createArtifactGivenCellTypes( + file, cellTypesMessage); + + try { + // index the artifact for keyword search + blackboard.indexArtifact(artifact); + } catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, + "Unable to index blackboard artifact " + //NON-NLS + artifact.getArtifactID(), ex); + MessageNotifyUtil.Notify.error( + Bundle.InterestingDatabasesIngestModule_indexError_message(), + artifact.getDisplayName()); + } + + services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, + BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, + Collections.singletonList(artifact))); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error posting to the blackboard", ex); //NOI18N NON-NLS + } + + } catch (SQLException ex) { + //Could be thrown in init or close.. need to fix. + Exceptions.printStackTrace(ex); + } catch (ClassNotFoundException ex) { + Exceptions.printStackTrace(ex); + } catch (IOException ex) { + Exceptions.printStackTrace(ex); + } catch (NoCurrentCaseException ex) { + Exceptions.printStackTrace(ex); + } catch (TskCoreException ex) { + Exceptions.printStackTrace(ex); + } } return ProcessResult.OK; } - private boolean successfulDependencyInitialization(AbstractFile file) { - return createLocalDiskPath(file) == ProcessResult.OK && - initalizeSQLiteReader(file) == ProcessResult.OK; - } - - private ProcessResult createLocalDiskPath(AbstractFile file) { - try { - localDiskPath = Case.getCurrentCaseThrows() - .getTempDirectory() + File.separator + file.getName(); - } catch (NoCurrentCaseException ex) { - // TODO -- personal note log about current case being closed or - //current case not existing. - return ProcessResult.ERROR; + /** + * + * @param sqliteReader + * @return + * @throws SQLException + */ + private Set cellTypesInDatabase(SQLiteReader sqliteReader) throws SQLException { + Map tables = sqliteReader.getTableSchemas(); + Set aggregateCellTypes = new TreeSet<>(); + + //Aggregate all cell types from each table + for(String tableName : tables.keySet()) { + addCellTypesInTable(sqliteReader, tableName, aggregateCellTypes); } - return ProcessResult.OK; + return aggregateCellTypes; } - private ProcessResult initalizeSQLiteReader(AbstractFile file) { - try { - sqliteReader = new SQLiteReader(file, localDiskPath); - return ProcessResult.OK; - } catch (ClassNotFoundException ex) { - //TODO add logging messages for this. - } catch (SQLException ex) { - //TODO add logging messages for this. - } catch (IOException ex) { - //TODO add logging messages for this. - } catch (NoCurrentCaseException ex) { - //TODO add logging messages for this. - } catch (TskCoreException ex) { - //TODO add logging messages for this. - } - return ProcessResult.ERROR; + /** + * + * @param sqliteReader + * @param table + * @return + */ + private void addCellTypesInTable(SQLiteReader sqliteReader, String tableName, + Set aggregateCellTypes) throws SQLException { + + List> tableValues = sqliteReader.getRowsFromTable(tableName); + tableValues.forEach((row) -> { + addCellTypeInRow(row, aggregateCellTypes); + }); + } + + /** + * + * @param row + * @return + */ + private void addCellTypeInRow(Map row, + Set aggregateCellTypes) { + row.values().forEach((Object cell) -> { + if(cell instanceof String) { + aggregateCellTypes.add(cellTypeDetector.getType( (String) cell)); + } + }); + } + + /** + * + * @param type + */ + @NbBundle.Messages({ + "InterestingDatabasesIngestModule.FlagDatabases.setName=Selectors identified" + }) + private BlackboardArtifact createArtifactGivenCellTypes(AbstractFile file, + String cellTypesMessage) throws TskCoreException { + BlackboardArtifact artifact = file.newArtifact( + BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); + + BlackboardAttribute setNameAttribute = new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, + Bundle.InterestingDatabasesIngestModule_FlagDatabases_setName()); + artifact.addAttribute(setNameAttribute); + + BlackboardAttribute commentAttribute = new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, + cellTypesMessage); + artifact.addAttribute(commentAttribute); + + return artifact; } /** @@ -129,8 +239,4 @@ public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter { public void shutDown() { } - - private void createArtifact() { - - } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java index 56b46d099a..28c3e5eb2d 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java @@ -32,14 +32,14 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; */ @NbBundle.Messages({ - "InterestingDatabasesIngestModule.FlagDatabases.moduleName=Interesting Databases", - "InterestingDatabasesIngestModule.FlagDatabases.moduleDesc.text=Flags databases with interesting items (emails, phone numbers, gps coordinates, ip/mac addresses)" + "InterestingDatabasesIngestModuleFactory.FlagDatabases.moduleName=Interesting Databases", + "InterestingDatabasesIngestModuleFactory.FlagDatabases.moduleDesc.text=Flags databases with interesting items (emails, phone numbers, gps coordinates, ip/mac addresses)" }) @ServiceProvider(service = IngestModuleFactory.class) public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactoryAdapter { static String getModuleName() { - return Bundle.InterestingDatabasesIngestModule_FlagDatabases_moduleName(); + return Bundle.InterestingDatabasesIngestModuleFactory_FlagDatabases_moduleName(); } @Override @@ -49,7 +49,7 @@ public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactory @Override public String getModuleDescription() { - return Bundle.InterestingDatabasesIngestModule_FlagDatabases_moduleDesc_text(); + return Bundle.InterestingDatabasesIngestModuleFactory_FlagDatabases_moduleDesc_text(); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java index 34cd2969e7..9684d2c111 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java @@ -31,12 +31,15 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.casemodule.services.Services; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -44,9 +47,14 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Reads rows from SQLite tables and returns results in a list collection. */ +@NbBundle.Messages({ + "SQLiteReader.ReadSQLiteFiles.moduleName=SQLiteReader" +}) public class SQLiteReader implements AutoCloseable { private final Connection connection; + private final IngestServices services = IngestServices.getInstance(); + private final Logger logger = services.getLogger(Bundle.SQLiteReader_ReadSQLiteFiles_moduleName()); /** * Writes data source file contents to local disk and opens a sqlite JDBC @@ -260,13 +268,18 @@ public class SQLiteReader implements AutoCloseable { return rowMap; } + /** * Closes underlying JDBC connection. * * @throws SQLException */ @Override - public void close() throws SQLException { - connection.close(); + public void close() { + try { + connection.close(); + } catch (SQLException ex) { + logger.log(Level.WARNING, "Could not close JDBC connection", ex); + } } } From 6de21076032406547220c63e57975e0f30393d32 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 8 Aug 2018 17:23:53 -0400 Subject: [PATCH 04/33] Rename current module from interestingdatabases to databaseselector --- .../{interestingdatabases => databaseselector}/CellType.java | 0 .../CellTypeDetector.java | 0 .../DatabaseSelectorIngestModule.java} | 0 .../DatabaseSelectorIngestModuleFactory.java} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename Core/src/org/sleuthkit/autopsy/modules/{interestingdatabases => databaseselector}/CellType.java (100%) rename Core/src/org/sleuthkit/autopsy/modules/{interestingdatabases => databaseselector}/CellTypeDetector.java (100%) rename Core/src/org/sleuthkit/autopsy/modules/{interestingdatabases/InterestingDatabasesIngestModule.java => databaseselector/DatabaseSelectorIngestModule.java} (100%) rename Core/src/org/sleuthkit/autopsy/modules/{interestingdatabases/InterestingDatabasesIngestModuleFactory.java => databaseselector/DatabaseSelectorIngestModuleFactory.java} (100%) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellType.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java similarity index 100% rename from Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellType.java rename to Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java similarity index 100% rename from Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/CellTypeDetector.java rename to Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java similarity index 100% rename from Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModule.java rename to Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java similarity index 100% rename from Core/src/org/sleuthkit/autopsy/modules/interestingdatabases/InterestingDatabasesIngestModuleFactory.java rename to Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java From 67759a313900e53adb881dd8f421f9c2d10d5eb8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 8 Aug 2018 17:25:36 -0400 Subject: [PATCH 05/33] Fix SQLException duplication in close and init, so that try-with-resources can be used cleanly --- .../sleuthkit/autopsy/contentviewers/SQLiteViewer.java | 8 ++------ .../org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 6988e46527..4324a3dc06 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -339,12 +339,8 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { // close DB connection to file if (null != sqliteReader) { - try { - sqliteReader.close(); - sqliteReader = null; - } catch (SQLException ex) { - logger.log(Level.SEVERE, "Failed to close DB connection to file.", ex); //NON-NLS - } + sqliteReader.close(); + sqliteReader = null; } sqliteDbFile = null; diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java index 9684d2c111..94ac4c4e3e 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java @@ -270,15 +270,14 @@ public class SQLiteReader implements AutoCloseable { /** - * Closes underlying JDBC connection. - * - * @throws SQLException + * Closes underlying JDBC connection. */ @Override public void close() { try { connection.close(); } catch (SQLException ex) { + //Non-essential exception, logger.log(Level.WARNING, "Could not close JDBC connection", ex); } } From d74c8b6212da4592a25f0d897b70b2f06f8d9bf3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 8 Aug 2018 17:42:38 -0400 Subject: [PATCH 06/33] Add apache commons validator dependency to autopsy core --- Core/ivy.xml | 2 ++ Core/nbproject/project.properties | 1 + Core/nbproject/project.xml | 20 ++++++++++++-------- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Core/ivy.xml b/Core/ivy.xml index 601077eb91..5e0bcbbc4b 100644 --- a/Core/ivy.xml +++ b/Core/ivy.xml @@ -29,6 +29,8 @@ + + diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 80bb473653..46b4d6e90c 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -3,6 +3,7 @@ file.reference.c3p0-0.9.5.jar=release/modules/ext/c3p0-0.9.5.jar file.reference.commons-compress-1.14.jar=release/modules/ext/commons-compress-1.14.jar file.reference.commons-dbcp2-2.1.1.jar=release\\modules\\ext\\commons-dbcp2-2.1.1.jar file.reference.commons-pool2-2.4.2.jar=release\\modules\\ext\\commons-pool2-2.4.2.jar +file.reference.commons-validator-1.6.jar=release\\modules\\ext\\commons-validator-1.6.jar file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar file.reference.jdom-2.0.5-contrib.jar=release/modules/ext/jdom-2.0.5-contrib.jar file.reference.jdom-2.0.5.jar=release/modules/ext/jdom-2.0.5.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 859db9f0d2..ca87c4f2f4 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -355,6 +355,10 @@ ext/cxf-rt-transports-http-3.0.16.jar release/modules/ext/cxf-rt-transports-http-3.0.16.jar + + ext/commons-validator-1.6.jar + release\modules\ext\commons-validator-1.6.jar + ext/curator-framework-2.8.0.jar release/modules/ext/curator-framework-2.8.0.jar @@ -387,6 +391,10 @@ ext/sevenzipjbinding.jar release/modules/ext/sevenzipjbinding.jar + + ext/sleuthkit-postgresql-4.6.2.jar + release/modules/ext/sleuthkit-postgresql-4.6.2.jar + ext/mchange-commons-java-0.2.9.jar release/modules/ext/mchange-commons-java-0.2.9.jar @@ -411,10 +419,6 @@ ext/metadata-extractor-2.10.1.jar release/modules/ext/metadata-extractor-2.10.1.jar - - ext/sleuthkit-postgresql-4.6.2.jar - release/modules/ext/sleuthkit-postgresql-4.6.2.jar - ext/tika-core-1.17.jar release/modules/ext/tika-core-1.17.jar @@ -487,6 +491,10 @@ ext/jdom-2.0.5-contrib.jar release/modules/ext/jdom-2.0.5-contrib.jar + + ext/SparseBitSet-1.1.jar + release/modules/ext/SparseBitSet-1.1.jar + ext/pdfbox-2.0.8.jar release/modules/ext/pdfbox-2.0.8.jar @@ -499,10 +507,6 @@ ext/xmpcore-5.1.3.jar release/modules/ext/xmpcore-5.1.3.jar - - ext/SparseBitSet-1.1.jar - release/modules/ext/SparseBitSet-1.1.jar - From 8a3daf41100145c66d1a8155dfbd939c9982d66d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 8 Aug 2018 17:43:16 -0400 Subject: [PATCH 07/33] All the meat of the new module classes --- .../modules/databaseselector/CellType.java | 8 +- .../databaseselector/CellTypeDetector.java | 18 +- .../DatabaseSelectorIngestModule.java | 266 +++++++++++------- .../DatabaseSelectorIngestModuleFactory.java | 21 +- 4 files changed, 185 insertions(+), 128 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java index d491fcfadf..685c083eda 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java @@ -16,14 +16,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.modules.interestingdatabases; +package org.sleuthkit.autopsy.modules.databaseselector; /** - * - * @author dsmyda + * Cell types used by the CellTypeDetector class */ public enum CellType { EMAIL, NOT_INTERESTING -} - +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java index e3301bccb8..903bccca37 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java @@ -16,18 +16,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.modules.interestingdatabases; +package org.sleuthkit.autopsy.modules.databaseselector; + +import org.apache.commons.validator.routines.EmailValidator; /** - * - * @author dsmyda + * Determines the type of a database cell (EMAIL, PHONE, GPS COORD, MAC ADDRESS) */ public class CellTypeDetector { + + private final EmailValidator emailValidator; - public CellTypeDetector() { + public CellTypeDetector() { + emailValidator = EmailValidator.getInstance(); } public CellType getType(String cell) { - return CellType.EMAIL; + if(emailValidator.isValid(cell)) { + return CellType.EMAIL; + } else { + return CellType.NOT_INTERESTING; + } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java index b3c9d78ea6..043e85eaaa 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.modules.interestingdatabases; +package org.sleuthkit.autopsy.modules.databaseselector; import java.io.File; import java.io.IOException; @@ -39,6 +39,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector.FileTypeDetectorInitException; import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -46,197 +47,248 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; /** - * - * @author dsmyda + * Parses database files and marks them as having interesting records (emails, + * phone numbers, mac addresses, gps coordinates). */ -public class InterestingDatabasesIngestModule extends FileIngestModuleAdapter { +public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { private static final String SUPPORTED_MIME_TYPE = "application/x-sqlite3"; - private static final String MODULE_NAME = InterestingDatabasesIngestModuleFactory.getModuleName(); + private static final String MODULE_NAME = DatabaseSelectorIngestModuleFactory.getModuleName(); private final IngestServices services = IngestServices.getInstance(); private final Logger logger = services.getLogger( - InterestingDatabasesIngestModuleFactory.getModuleName()); + DatabaseSelectorIngestModuleFactory.getModuleName()); private FileTypeDetector fileTypeDetector; private CellTypeDetector cellTypeDetector; private Blackboard blackboard; + private String localDiskPath; + - /** - * - * @param context - * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException - */ @NbBundle.Messages({ - "CannotRunFileTypeDetection=Unable to initialize file type detection.", + "DatabaseSelectorIngestModule.CannotRunFileTypeDetection=Unable to initialize file type detection.", }) @Override public void startUp(IngestJobContext context) throws IngestModuleException { try { fileTypeDetector = new FileTypeDetector(); cellTypeDetector = new CellTypeDetector(); - } catch (FileTypeDetector.FileTypeDetectorInitException ex) { - throw new IngestModuleException(Bundle.CannotRunFileTypeDetection(), ex); + } catch (FileTypeDetectorInitException ex) { + throw new IngestModuleException(Bundle.DatabaseSelectorIngestModule_CannotRunFileTypeDetection(), ex); } } - /** - * - * @param file - * @return - */ @Override - @NbBundle.Messages({ - "InterestingDatabasesIngestModule.indexError.message=Failed to index interesting artifact hit for keyword search." - }) public ProcessResult process(AbstractFile file) { + if(getBlackboardInstanceFromServices().equals(ProcessResult.ERROR)) { + return ProcessResult.ERROR; + } + + //Qualify the MIMEType + String dataSourceMimeType = fileTypeDetector.getMIMEType(file); + if(SUPPORTED_MIME_TYPE.equals(dataSourceMimeType)) { + if(createLocalDiskPathFromCurrentCase(file).equals(ProcessResult.ERROR)) { + return ProcessResult.ERROR; + } + + try (SQLiteReader sqliteReader = new SQLiteReader(file, localDiskPath)){ + try { + Set databaseCellTypes = getCellTypesInDatabase(sqliteReader); + //No interesting hits, don't flag this database, skip artifact creation. + if(!databaseCellTypes.isEmpty()) { + String cellTypesComment = createCellTypeCommentString(databaseCellTypes); + try { + BlackboardArtifact artifact = createArtifactGivenCellTypes( + file, cellTypesComment); + indexArtifactAndFireModuleDataEvent(artifact); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS + } + } + } catch(SQLException ex) { + logger.log(Level.WARNING, "Error attempting to read sqlite " + + "file in DatabaseSelectorIngestModule", ex); + } + } catch (ClassNotFoundException | SQLException | IOException | + NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, "Cannot initialize sqliteReader class " + + "in DatabaseSelectorIngestModule", ex); + return ProcessResult.ERROR; + } + } + + //Whether we successfully read the sqlite database or determined the mime + //type is not supported, the process is OK. + return ProcessResult.OK; + } + + /** + * Get a pointer to the current case blackboard for indexing of artifacts + * + * @return ProcessResult indicating a success or failure in getting current case + */ + private ProcessResult getBlackboardInstanceFromServices() { try { - blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); + blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS return ProcessResult.ERROR; } - String dataSourceMimeType = fileTypeDetector.getMIMEType(file); - if(SUPPORTED_MIME_TYPE.equals(dataSourceMimeType)) { - - String localDiskPath; - try { - localDiskPath = Case.getCurrentCaseThrows() - .getTempDirectory() + File.separator + file.getName(); - } catch (NoCurrentCaseException ex) { - // TODO -- personal note log about current case being closed or - //current case not existing. - Exceptions.printStackTrace(ex); - return ProcessResult.ERROR; - } - - - try (SQLiteReader sqliteReader = - new SQLiteReader(file, localDiskPath)){ - - Set aggregatedCellTypes = cellTypesInDatabase(sqliteReader); - String cellTypesMessage = aggregatedCellTypes.toString() - .replace("]", "") - .replace("[", ""); - - try { - BlackboardArtifact artifact = createArtifactGivenCellTypes( - file, cellTypesMessage); - - try { - // index the artifact for keyword search - blackboard.indexArtifact(artifact); - } catch (Blackboard.BlackboardException ex) { - logger.log(Level.SEVERE, - "Unable to index blackboard artifact " + //NON-NLS - artifact.getArtifactID(), ex); - MessageNotifyUtil.Notify.error( - Bundle.InterestingDatabasesIngestModule_indexError_message(), - artifact.getDisplayName()); - } - - services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, - BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, - Collections.singletonList(artifact))); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error posting to the blackboard", ex); //NOI18N NON-NLS - } - - } catch (SQLException ex) { - //Could be thrown in init or close.. need to fix. - Exceptions.printStackTrace(ex); - } catch (ClassNotFoundException ex) { - Exceptions.printStackTrace(ex); - } catch (IOException ex) { - Exceptions.printStackTrace(ex); - } catch (NoCurrentCaseException ex) { - Exceptions.printStackTrace(ex); - } catch (TskCoreException ex) { - Exceptions.printStackTrace(ex); - } + return ProcessResult.OK; + } + + /** + * Generates a local disk path for abstract file contents to be copied. + * All database sources must be copied to local disk to be opened by + * SQLiteReader + * + * @param file The database abstract file + * @return ProcessResult indicating a success or failure in creating a disk path + */ + private ProcessResult createLocalDiskPathFromCurrentCase(AbstractFile file) { + try { + localDiskPath = Case.getCurrentCaseThrows() + .getTempDirectory() + File.separator + file.getName(); + } catch (NoCurrentCaseException ex) { + // TODO -- personal note log about current case being closed or + //current case not existing. + Exceptions.printStackTrace(ex); + return ProcessResult.ERROR; } return ProcessResult.OK; } /** + * Creates and populates a set of all CellTypes in the sqlite database * - * @param sqliteReader - * @return - * @throws SQLException + * @param sqliteReader Reader currently connected to database file + * @return A Set of distinct CellTypes + * @throws SQLException Caught during attempting to read sqlite database */ - private Set cellTypesInDatabase(SQLiteReader sqliteReader) throws SQLException { + private Set getCellTypesInDatabase(SQLiteReader sqliteReader) throws SQLException { Map tables = sqliteReader.getTableSchemas(); Set aggregateCellTypes = new TreeSet<>(); - //Aggregate all cell types from each table + //Aggregate cell types from all tables for(String tableName : tables.keySet()) { - addCellTypesInTable(sqliteReader, tableName, aggregateCellTypes); + aggregateCellTypes.addAll(getCellTypesInTable(sqliteReader, tableName)); } return aggregateCellTypes; } /** + * Creates and populates a set of all CellTypes in a sqlite table * - * @param sqliteReader - * @param table - * @return + * @param sqliteReader Reader currently connected to database file + * @param tableName database table to be opened and read + * @return Set of all unique cell types in table + * @throws SQLException Caught during attempting to read sqlite database */ - private void addCellTypesInTable(SQLiteReader sqliteReader, String tableName, - Set aggregateCellTypes) throws SQLException { - + private Set getCellTypesInTable(SQLiteReader sqliteReader, + String tableName) throws SQLException { + + Set tableCellTypes = new TreeSet<>(); List> tableValues = sqliteReader.getRowsFromTable(tableName); + + //Aggregate cell types from all table rows tableValues.forEach((row) -> { - addCellTypeInRow(row, aggregateCellTypes); + tableCellTypes.addAll(getCellTypesInRow(row)); }); + return tableCellTypes; } /** + * Creates and populates a set of all CellTypes in a table row * - * @param row + * @param row Table row is represented as a column-value map * @return */ - private void addCellTypeInRow(Map row, - Set aggregateCellTypes) { + private Set getCellTypesInRow(Map row) { + Set rowCellTypes = new TreeSet<>(); + + //Aggregate cell types from a row row.values().forEach((Object cell) -> { if(cell instanceof String) { - aggregateCellTypes.add(cellTypeDetector.getType( (String) cell)); + CellType type = cellTypeDetector.getType((String) cell); + if(!type.equals(CellType.NOT_INTERESTING)) { + rowCellTypes.add(type); + } } }); + return rowCellTypes; } /** + * Creates a comma seperated string of all the cell types found in a database + * file. Used as the comment string for the blackboard artifact. * - * @param type + * @param databaseCellTypes The set of all database cell types detected + * @return + */ + private String createCellTypeCommentString(Set databaseCellTypes) { + return databaseCellTypes.toString().replace("]", "").replace("[", ""); + } + + /** + * Initializes a new interesting file hit artifact and provides name and + * comment attributes + * + * @param file The database abstract file + * @param cellTypesComment String of all the cell types found in a database + * file + * @return Interesting file hit artifact + * @throws TskCoreException Thrown if the abstract file cannot create a blackboard + * artifact */ @NbBundle.Messages({ - "InterestingDatabasesIngestModule.FlagDatabases.setName=Selectors identified" + "DatabaseSelectorIngestModule.FlagDatabases.setName=Selectors identified" }) private BlackboardArtifact createArtifactGivenCellTypes(AbstractFile file, - String cellTypesMessage) throws TskCoreException { + String cellTypesComment) throws TskCoreException { BlackboardArtifact artifact = file.newArtifact( BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); BlackboardAttribute setNameAttribute = new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, - Bundle.InterestingDatabasesIngestModule_FlagDatabases_setName()); + Bundle.DatabaseSelectorIngestModule_FlagDatabases_setName()); artifact.addAttribute(setNameAttribute); BlackboardAttribute commentAttribute = new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, - cellTypesMessage); + cellTypesComment); artifact.addAttribute(commentAttribute); return artifact; } /** + * Pass the artifact to blackboard for indexing and fire module data event + * in the IngestServices * + * @param artifact Blackboard artifact created for the interesting file hit */ - @Override - public void shutDown() { - + @NbBundle.Messages({ + "DatabaseSelectorIngestModule.indexError.message=" + + "Failed to index interesting file hit artifact for keyword search." + }) + private void indexArtifactAndFireModuleDataEvent(BlackboardArtifact artifact) { + try { + // index the artifact for keyword search + blackboard.indexArtifact(artifact); + } catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, + "Unable to index blackboard artifact " + //NON-NLS + artifact.getArtifactID(), ex); + MessageNotifyUtil.Notify.error( + Bundle.DatabaseSelectorIngestModule_indexError_message(), + artifact.getDisplayName()); + } + + services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, + BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, + Collections.singletonList(artifact))); } -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java index 28c3e5eb2d..155b26c425 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.modules.interestingdatabases; +package org.sleuthkit.autopsy.modules.databaseselector; import org.sleuthkit.autopsy.coreutils.Version; import org.openide.util.NbBundle; @@ -27,19 +27,18 @@ import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; /** - * - * @author dsmyda + * A factory that creates a DatabaseSelectorModule to parse database files and + * mark them as having interesting records (emails, phone numbers, mac addresses, gps coordinates). */ - @NbBundle.Messages({ - "InterestingDatabasesIngestModuleFactory.FlagDatabases.moduleName=Interesting Databases", - "InterestingDatabasesIngestModuleFactory.FlagDatabases.moduleDesc.text=Flags databases with interesting items (emails, phone numbers, gps coordinates, ip/mac addresses)" + "DatabaseSelectorIngestModuleFactory.FlagDatabases.moduleName=Database Selector", + "DatabaseSelectorIngestModuleFactory.FlagDatabases.moduleDesc.text=Flag databases with interesting items (emails, phone numbers, gps coordinates, ip/mac addresses)" }) @ServiceProvider(service = IngestModuleFactory.class) -public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactoryAdapter { +public class DatabaseSelectorIngestModuleFactory extends IngestModuleFactoryAdapter { static String getModuleName() { - return Bundle.InterestingDatabasesIngestModuleFactory_FlagDatabases_moduleName(); + return Bundle.DatabaseSelectorIngestModuleFactory_FlagDatabases_moduleName(); } @Override @@ -49,7 +48,7 @@ public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactory @Override public String getModuleDescription() { - return Bundle.InterestingDatabasesIngestModuleFactory_FlagDatabases_moduleDesc_text(); + return Bundle.DatabaseSelectorIngestModuleFactory_FlagDatabases_moduleDesc_text(); } @Override @@ -64,6 +63,6 @@ public class InterestingDatabasesIngestModuleFactory extends IngestModuleFactory @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { - return new InterestingDatabasesIngestModule(); + return new DatabaseSelectorIngestModule(); } -} +} \ No newline at end of file From e5b9be89d658fe7cb04c56ea7172d4f89d529c08 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 9 Aug 2018 12:16:08 -0400 Subject: [PATCH 08/33] Added quotation marks around table names to fix jdbc bug and refactored where sql exceptions are caught so that all tables of a database file are read --- .../databaseselector/CellTypeDetector.java | 10 +- .../DatabaseSelectorIngestModule.java | 142 ++++++++---------- .../autopsy/sqlitereader/SQLiteReader.java | 20 ++- 3 files changed, 89 insertions(+), 83 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java index 903bccca37..085e2d3107 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java @@ -31,11 +31,17 @@ public class CellTypeDetector { emailValidator = EmailValidator.getInstance(); } + public CellType getType(Object cell) { + if (cell instanceof String) { + return getType((String) cell); + } + return CellType.NOT_INTERESTING; + } + public CellType getType(String cell) { if(emailValidator.isValid(cell)) { return CellType.EMAIL; - } else { - return CellType.NOT_INTERESTING; } + return CellType.NOT_INTERESTING; } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java index 043e85eaaa..f8be1ebb71 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.logging.Level; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -62,59 +61,57 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { private CellTypeDetector cellTypeDetector; private Blackboard blackboard; - private String localDiskPath; @NbBundle.Messages({ - "DatabaseSelectorIngestModule.CannotRunFileTypeDetection=Unable to initialize file type detection.", + "DatabaseSelectorIngestModule.CannotRunFileTypeDetection=" + + "Unable to initialize file type detection.", + "DatabaseSelectorIngestModule.CannotGetBlackboard=" + + "Exception while attempting to get Blackboard from current case." }) @Override public void startUp(IngestJobContext context) throws IngestModuleException { try { fileTypeDetector = new FileTypeDetector(); - cellTypeDetector = new CellTypeDetector(); } catch (FileTypeDetectorInitException ex) { throw new IngestModuleException(Bundle.DatabaseSelectorIngestModule_CannotRunFileTypeDetection(), ex); } + + cellTypeDetector = new CellTypeDetector(); + + try { + blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); + } catch (NoCurrentCaseException ex) { + throw new IngestModuleException(Bundle.DatabaseSelectorIngestModule_CannotGetBlackboard(), ex); + } } @Override public ProcessResult process(AbstractFile file) { - if(getBlackboardInstanceFromServices().equals(ProcessResult.ERROR)) { - return ProcessResult.ERROR; - } - - //Qualify the MIMEType + //Qualify the MIMEType, only process sqlite files. String dataSourceMimeType = fileTypeDetector.getMIMEType(file); - if(SUPPORTED_MIME_TYPE.equals(dataSourceMimeType)) { - if(createLocalDiskPathFromCurrentCase(file).equals(ProcessResult.ERROR)) { - return ProcessResult.ERROR; - } - - try (SQLiteReader sqliteReader = new SQLiteReader(file, localDiskPath)){ + if(!dataSourceMimeType.equals(SUPPORTED_MIME_TYPE)) { + return ProcessResult.OK; + } + + try (SQLiteReader sqliteReader = new SQLiteReader(file, createLocalDiskPath(file))){ + Set databaseCellTypes = getCellTypesInDatabase(file, sqliteReader); + //No interesting hits, don't flag this database, skip artifact creation. + if(!databaseCellTypes.isEmpty()) { + String cellTypesComment = createCellTypeCommentString(databaseCellTypes); try { - Set databaseCellTypes = getCellTypesInDatabase(sqliteReader); - //No interesting hits, don't flag this database, skip artifact creation. - if(!databaseCellTypes.isEmpty()) { - String cellTypesComment = createCellTypeCommentString(databaseCellTypes); - try { - BlackboardArtifact artifact = createArtifactGivenCellTypes( - file, cellTypesComment); - indexArtifactAndFireModuleDataEvent(artifact); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS - } - } - } catch(SQLException ex) { - logger.log(Level.WARNING, "Error attempting to read sqlite " - + "file in DatabaseSelectorIngestModule", ex); - } - } catch (ClassNotFoundException | SQLException | IOException | - NoCurrentCaseException | TskCoreException ex) { - logger.log(Level.SEVERE, "Cannot initialize sqliteReader class " - + "in DatabaseSelectorIngestModule", ex); - return ProcessResult.ERROR; - } + BlackboardArtifact artifact = createArtifactGivenCellTypes( + file, cellTypesComment); + indexArtifactAndFireModuleDataEvent(artifact); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS + } + } + } catch (ClassNotFoundException | SQLException | IOException | + NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Cannot initialize sqliteReader class " //NON-NLS + + "in DatabaseSelectorIngestModule for file [%s]", file.getName()), ex); //NON-NLS + return ProcessResult.ERROR; } //Whether we successfully read the sqlite database or determined the mime @@ -122,58 +119,49 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { return ProcessResult.OK; } - /** - * Get a pointer to the current case blackboard for indexing of artifacts - * - * @return ProcessResult indicating a success or failure in getting current case - */ - private ProcessResult getBlackboardInstanceFromServices() { - try { - blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return ProcessResult.ERROR; - } - - return ProcessResult.OK; - } - /** * Generates a local disk path for abstract file contents to be copied. * All database sources must be copied to local disk to be opened by * SQLiteReader * * @param file The database abstract file - * @return ProcessResult indicating a success or failure in creating a disk path + * @return Valid local path for copying + * @throws NoCurrentCaseException if the current case has been closed. */ - private ProcessResult createLocalDiskPathFromCurrentCase(AbstractFile file) { - try { - localDiskPath = Case.getCurrentCaseThrows() - .getTempDirectory() + File.separator + file.getName(); - } catch (NoCurrentCaseException ex) { - // TODO -- personal note log about current case being closed or - //current case not existing. - Exceptions.printStackTrace(ex); - return ProcessResult.ERROR; - } - - return ProcessResult.OK; + private String createLocalDiskPath(AbstractFile file) throws NoCurrentCaseException { + return Case.getCurrentCaseThrows().getTempDirectory() + + File.separator + file.getName(); } /** * Creates and populates a set of all CellTypes in the sqlite database * - * @param sqliteReader Reader currently connected to database file + * @param sqliteReader Reader instance currently connected to database file * @return A Set of distinct CellTypes - * @throws SQLException Caught during attempting to read sqlite database */ - private Set getCellTypesInDatabase(SQLiteReader sqliteReader) throws SQLException { - Map tables = sqliteReader.getTableSchemas(); + private Set getCellTypesInDatabase(AbstractFile file, SQLiteReader sqliteReader) { Set aggregateCellTypes = new TreeSet<>(); - + + Map tables; + try { + tables = sqliteReader.getTableSchemas(); + } catch (SQLException ex) { + logger.log(Level.WARNING, String.format("Error attempting to get tables from sqlite" //NON-NLS + + "file [%s] in DatabaseSelectorIngestModule", //NON-NLS + file.getName()), ex); + //Unable to get any cellTypes, return empty set to be ignored. + return aggregateCellTypes; + } //Aggregate cell types from all tables for(String tableName : tables.keySet()) { - aggregateCellTypes.addAll(getCellTypesInTable(sqliteReader, tableName)); + try { + aggregateCellTypes.addAll(getCellTypesInTable(sqliteReader, tableName)); + } catch (SQLException ex) { + logger.log(Level.WARNING, + String.format("Error attempting to read sqlite table [%s]" //NON-NLS + + " for file [%s] in DatabaseSelectorIngestModule", //NON-NLS + tableName, file.getName()), ex); + } } return aggregateCellTypes; @@ -211,11 +199,9 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { //Aggregate cell types from a row row.values().forEach((Object cell) -> { - if(cell instanceof String) { - CellType type = cellTypeDetector.getType((String) cell); - if(!type.equals(CellType.NOT_INTERESTING)) { - rowCellTypes.add(type); - } + CellType type = cellTypeDetector.getType(cell); + if(!type.equals(CellType.NOT_INTERESTING)) { + rowCellTypes.add(type); } }); return rowCellTypes; @@ -229,7 +215,7 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { * @return */ private String createCellTypeCommentString(Set databaseCellTypes) { - return databaseCellTypes.toString().replace("]", "").replace("[", ""); + return databaseCellTypes.toString().replace("]", "").replace("[", ""); //NON-NLS } /** diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java index 94ac4c4e3e..e48042a032 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java @@ -184,6 +184,7 @@ public class SQLiteReader implements AutoCloseable { * @throws SQLException */ public Integer getTableRowCount(String tableName) throws SQLException { + tableName = wrapTableNameStringWithQuotes(tableName); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( "SELECT count (*) as count FROM " + tableName)){ //NON-NLS @@ -201,10 +202,10 @@ public class SQLiteReader implements AutoCloseable { * @throws SQLException */ public List> getRowsFromTable(String tableName) throws SQLException { - //This method does not directly call its overloaded counterpart //since the second parameter would need to be retreived from a call to //getTableRowCount(). + tableName = wrapTableNameStringWithQuotes(tableName); try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( "SELECT * FROM " + tableName)) { //NON-NLS @@ -224,7 +225,7 @@ public class SQLiteReader implements AutoCloseable { */ public List> getRowsFromTable(String tableName, int startRow, int numRowsToRead) throws SQLException{ - + tableName = wrapTableNameStringWithQuotes(tableName); try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( "SELECT * FROM " + tableName //NON-NLS @@ -234,6 +235,18 @@ public class SQLiteReader implements AutoCloseable { } } + /** + * Wraps table name with quotation marks in case table name contains spaces. + * sqliteJDBC cannot read table names with spaces in them unless surrounded + * by quotation marks. + * + * @param tableName + * @return Input name: Result Table -> "Result Table" + */ + private String wrapTableNameStringWithQuotes(String tableName) { + return "\"" + tableName +"\""; + } + /** * Converts a ResultSet (row results from a table read) into a list. * @@ -277,7 +290,8 @@ public class SQLiteReader implements AutoCloseable { try { connection.close(); } catch (SQLException ex) { - //Non-essential exception, + //Non-essential exception, user has no need for the connection + //object at this stage so closing details are not important logger.log(Level.WARNING, "Could not close JDBC connection", ex); } } From 9afa1fa38cedb8531af761508175e62942f83184 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 9 Aug 2018 12:25:49 -0400 Subject: [PATCH 09/33] Refactored and renamed some functions --- .../DatabaseSelectorIngestModule.java | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java index f8be1ebb71..0933f7af7b 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java @@ -96,12 +96,11 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { try (SQLiteReader sqliteReader = new SQLiteReader(file, createLocalDiskPath(file))){ Set databaseCellTypes = getCellTypesInDatabase(file, sqliteReader); + //No interesting hits, don't flag this database, skip artifact creation. if(!databaseCellTypes.isEmpty()) { - String cellTypesComment = createCellTypeCommentString(databaseCellTypes); try { - BlackboardArtifact artifact = createArtifactGivenCellTypes( - file, cellTypesComment); + BlackboardArtifact artifact = createArtifact(file, databaseCellTypes); indexArtifactAndFireModuleDataEvent(artifact); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS @@ -110,12 +109,10 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { } catch (ClassNotFoundException | SQLException | IOException | NoCurrentCaseException | TskCoreException ex) { logger.log(Level.SEVERE, String.format("Cannot initialize sqliteReader class " //NON-NLS - + "in DatabaseSelectorIngestModule for file [%s]", file.getName()), ex); //NON-NLS + + "for file [%s].", file.getName()), ex); //NON-NLS return ProcessResult.ERROR; } - //Whether we successfully read the sqlite database or determined the mime - //type is not supported, the process is OK. return ProcessResult.OK; } @@ -147,11 +144,12 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { tables = sqliteReader.getTableSchemas(); } catch (SQLException ex) { logger.log(Level.WARNING, String.format("Error attempting to get tables from sqlite" //NON-NLS - + "file [%s] in DatabaseSelectorIngestModule", //NON-NLS + + "file [%s].", //NON-NLS file.getName()), ex); //Unable to get any cellTypes, return empty set to be ignored. return aggregateCellTypes; } + //Aggregate cell types from all tables for(String tableName : tables.keySet()) { try { @@ -159,7 +157,7 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { } catch (SQLException ex) { logger.log(Level.WARNING, String.format("Error attempting to read sqlite table [%s]" //NON-NLS - + " for file [%s] in DatabaseSelectorIngestModule", //NON-NLS + + " for file [%s].", //NON-NLS tableName, file.getName()), ex); } } @@ -207,17 +205,6 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { return rowCellTypes; } - /** - * Creates a comma seperated string of all the cell types found in a database - * file. Used as the comment string for the blackboard artifact. - * - * @param databaseCellTypes The set of all database cell types detected - * @return - */ - private String createCellTypeCommentString(Set databaseCellTypes) { - return databaseCellTypes.toString().replace("]", "").replace("[", ""); //NON-NLS - } - /** * Initializes a new interesting file hit artifact and provides name and * comment attributes @@ -232,8 +219,8 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { @NbBundle.Messages({ "DatabaseSelectorIngestModule.FlagDatabases.setName=Selectors identified" }) - private BlackboardArtifact createArtifactGivenCellTypes(AbstractFile file, - String cellTypesComment) throws TskCoreException { + private BlackboardArtifact createArtifact(AbstractFile file, + Set databaseCellTypes) throws TskCoreException { BlackboardArtifact artifact = file.newArtifact( BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); @@ -242,6 +229,7 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { Bundle.DatabaseSelectorIngestModule_FlagDatabases_setName()); artifact.addAttribute(setNameAttribute); + String cellTypesComment = createCellTypeCommentString(databaseCellTypes); BlackboardAttribute commentAttribute = new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, cellTypesComment); @@ -250,6 +238,17 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { return artifact; } + /** + * Creates a comma seperated string of all the cell types found in a database + * file. Used as the comment string for the blackboard artifact. + * + * @param databaseCellTypes The set of all database cell types detected + * @return + */ + private String createCellTypeCommentString(Set databaseCellTypes) { + return databaseCellTypes.toString().replace("]", "").replace("[", ""); //NON-NLS + } + /** * Pass the artifact to blackboard for indexing and fire module data event * in the IngestServices From defd0f6783268de05c672757af56dafd88ed240b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 9 Aug 2018 12:30:27 -0400 Subject: [PATCH 10/33] Final changes to comments --- .../databaseselector/DatabaseSelectorIngestModule.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java index 0933f7af7b..c73df94730 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java @@ -97,7 +97,7 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { try (SQLiteReader sqliteReader = new SQLiteReader(file, createLocalDiskPath(file))){ Set databaseCellTypes = getCellTypesInDatabase(file, sqliteReader); - //No interesting hits, don't flag this database, skip artifact creation. + //If empty, then no interesting hits, don't flag this database, skip artifact creation. if(!databaseCellTypes.isEmpty()) { try { BlackboardArtifact artifact = createArtifact(file, databaseCellTypes); @@ -240,7 +240,8 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { /** * Creates a comma seperated string of all the cell types found in a database - * file. Used as the comment string for the blackboard artifact. + * file. Used as the comment string for the blackboard artifact. TreeSet is + * used to ensure that CellTypes appear in the same order as the enum. * * @param databaseCellTypes The set of all database cell types detected * @return From 3c7eb3f371863da932dd6f6208e727358a3107f4 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 9 Aug 2018 15:40:37 -0400 Subject: [PATCH 11/33] More commits incoming --- .../modules/databaseselector/CellType.java | 27 ---- .../DataElementTypeDetector.java} | 34 +++-- .../FileSelectorIngestModule.java} | 132 ++++++++++-------- .../FileSelectorIngestModuleFactory.java} | 15 +- 4 files changed, 102 insertions(+), 106 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java rename Core/src/org/sleuthkit/autopsy/modules/{databaseselector/CellTypeDetector.java => fileselector/DataElementTypeDetector.java} (59%) rename Core/src/org/sleuthkit/autopsy/modules/{databaseselector/DatabaseSelectorIngestModule.java => fileselector/FileSelectorIngestModule.java} (64%) rename Core/src/org/sleuthkit/autopsy/modules/{databaseselector/DatabaseSelectorIngestModuleFactory.java => fileselector/FileSelectorIngestModuleFactory.java} (73%) diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java b/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java deleted file mode 100755 index 685c083eda..0000000000 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellType.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.modules.databaseselector; - -/** - * Cell types used by the CellTypeDetector class - */ -public enum CellType { - EMAIL, - NOT_INTERESTING -} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/DataElementTypeDetector.java similarity index 59% rename from Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java rename to Core/src/org/sleuthkit/autopsy/modules/fileselector/DataElementTypeDetector.java index 085e2d3107..ba532804b8 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/CellTypeDetector.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileselector/DataElementTypeDetector.java @@ -16,32 +16,40 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.modules.databaseselector; +package org.sleuthkit.autopsy.modules.fileselector; import org.apache.commons.validator.routines.EmailValidator; /** * Determines the type of a database cell (EMAIL, PHONE, GPS COORD, MAC ADDRESS) */ -public class CellTypeDetector { +final class DataElementTypeDetector { - private final EmailValidator emailValidator; - - public CellTypeDetector() { - emailValidator = EmailValidator.getInstance(); + private static final EmailValidator EMAIL_VALIDATOR; + + static { + EMAIL_VALIDATOR = EmailValidator.getInstance(); } - public CellType getType(Object cell) { + public static DataElementType getType(Object cell) { if (cell instanceof String) { return getType((String) cell); } - return CellType.NOT_INTERESTING; + return DataElementType.NOT_INTERESTING; } - public CellType getType(String cell) { - if(emailValidator.isValid(cell)) { - return CellType.EMAIL; + public static DataElementType getType(String cell) { + if(EMAIL_VALIDATOR.isValid(cell)) { + return DataElementType.EMAIL; } - return CellType.NOT_INTERESTING; + return DataElementType.NOT_INTERESTING; } -} + + /* + * Cell data that qualify as an "interesting" selector + */ + public enum DataElementType { + EMAIL, + NOT_INTERESTING + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java similarity index 64% rename from Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java rename to Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java index c73df94730..652030ba14 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java @@ -16,15 +16,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.modules.databaseselector; +package org.sleuthkit.autopsy.modules.fileselector; import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeSet; import java.util.logging.Level; import org.openide.util.NbBundle; @@ -37,6 +37,7 @@ import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.modules.fileselector.DataElementTypeDetector.DataElementType; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector.FileTypeDetectorInitException; import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; @@ -49,24 +50,23 @@ import org.sleuthkit.datamodel.TskCoreException; * Parses database files and marks them as having interesting records (emails, * phone numbers, mac addresses, gps coordinates). */ -public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { +public class FileSelectorIngestModule extends FileIngestModuleAdapter { private static final String SUPPORTED_MIME_TYPE = "application/x-sqlite3"; - private static final String MODULE_NAME = DatabaseSelectorIngestModuleFactory.getModuleName(); + private static final String MODULE_NAME = FileSelectorIngestModuleFactory.getModuleName(); private final IngestServices services = IngestServices.getInstance(); private final Logger logger = services.getLogger( - DatabaseSelectorIngestModuleFactory.getModuleName()); + FileSelectorIngestModuleFactory.getModuleName()); private FileTypeDetector fileTypeDetector; - private CellTypeDetector cellTypeDetector; private Blackboard blackboard; @NbBundle.Messages({ - "DatabaseSelectorIngestModule.CannotRunFileTypeDetection=" + "FileSelectorIngestModule.CannotRunFileTypeDetection=" + "Unable to initialize file type detection.", - "DatabaseSelectorIngestModule.CannotGetBlackboard=" + "FileSelectorIngestModule.CannotGetBlackboard=" + "Exception while attempting to get Blackboard from current case." }) @Override @@ -74,37 +74,37 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { try { fileTypeDetector = new FileTypeDetector(); } catch (FileTypeDetectorInitException ex) { - throw new IngestModuleException(Bundle.DatabaseSelectorIngestModule_CannotRunFileTypeDetection(), ex); + throw new IngestModuleException(Bundle.FileSelectorIngestModule_CannotRunFileTypeDetection(), ex); } - cellTypeDetector = new CellTypeDetector(); - try { blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); } catch (NoCurrentCaseException ex) { - throw new IngestModuleException(Bundle.DatabaseSelectorIngestModule_CannotGetBlackboard(), ex); + throw new IngestModuleException(Bundle.FileSelectorIngestModule_CannotGetBlackboard(), ex); } } @Override public ProcessResult process(AbstractFile file) { //Qualify the MIMEType, only process sqlite files. - String dataSourceMimeType = fileTypeDetector.getMIMEType(file); - if(!dataSourceMimeType.equals(SUPPORTED_MIME_TYPE)) { + String fileMimeType = fileTypeDetector.getMIMEType(file); + if(!fileMimeType.equals(SUPPORTED_MIME_TYPE)) { return ProcessResult.OK; } try (SQLiteReader sqliteReader = new SQLiteReader(file, createLocalDiskPath(file))){ - Set databaseCellTypes = getCellTypesInDatabase(file, sqliteReader); - //If empty, then no interesting hits, don't flag this database, skip artifact creation. - if(!databaseCellTypes.isEmpty()) { - try { - BlackboardArtifact artifact = createArtifact(file, databaseCellTypes); - indexArtifactAndFireModuleDataEvent(artifact); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS - } + Collection dataElementTypesInFile = readFileAndFindTypes(file, sqliteReader); + //No interesting types found, no artifact to create + if(dataElementTypesInFile.isEmpty()) { + return ProcessResult.OK; + } + + try { + BlackboardArtifact artifact = createArtifact(file, dataElementTypesInFile); + indexArtifactAndFireModuleDataEvent(artifact); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS } } catch (ClassNotFoundException | SQLException | IOException | NoCurrentCaseException | TskCoreException ex) { @@ -131,13 +131,13 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { } /** - * Creates and populates a set of all CellTypes in the sqlite database + * Creates and populates a collection of all data element types in the file * - * @param sqliteReader Reader instance currently connected to database file - * @return A Set of distinct CellTypes + * @param sqliteReader Reader instance currently connected to local file contents + * @return A collection of data element types */ - private Set getCellTypesInDatabase(AbstractFile file, SQLiteReader sqliteReader) { - Set aggregateCellTypes = new TreeSet<>(); + private Collection readFileAndFindTypes(AbstractFile file, SQLiteReader sqliteReader) { + Collection currentTypesFound = new TreeSet<>(); Map tables; try { @@ -146,14 +146,15 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { logger.log(Level.WARNING, String.format("Error attempting to get tables from sqlite" //NON-NLS + "file [%s].", //NON-NLS file.getName()), ex); - //Unable to get any cellTypes, return empty set to be ignored. - return aggregateCellTypes; + //Unable to read anything, return empty collection. + return currentTypesFound; } //Aggregate cell types from all tables for(String tableName : tables.keySet()) { try { - aggregateCellTypes.addAll(getCellTypesInTable(sqliteReader, tableName)); + Collection typesFoundInTable = readTableAndFindTypes(sqliteReader, tableName); + currentTypesFound.addAll(typesFoundInTable); } catch (SQLException ex) { logger.log(Level.WARNING, String.format("Error attempting to read sqlite table [%s]" //NON-NLS @@ -162,47 +163,60 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { } } - return aggregateCellTypes; + return currentTypesFound; } /** - * Creates and populates a set of all CellTypes in a sqlite table + * Creates and populates a collection of all data element types in a sqlite + * database table * - * @param sqliteReader Reader currently connected to database file + * @param sqliteReader Reader currently connected to local file contents * @param tableName database table to be opened and read - * @return Set of all unique cell types in table + * @return collection of all types in table * @throws SQLException Caught during attempting to read sqlite database */ - private Set getCellTypesInTable(SQLiteReader sqliteReader, + private Collection readTableAndFindTypes(SQLiteReader sqliteReader, String tableName) throws SQLException { - Set tableCellTypes = new TreeSet<>(); + Collection typesFoundReadingTable = new TreeSet<>(); List> tableValues = sqliteReader.getRowsFromTable(tableName); - //Aggregate cell types from all table rows + //Aggregate cell types from all rows tableValues.forEach((row) -> { - tableCellTypes.addAll(getCellTypesInRow(row)); + Collection typesFoundInRow = readRowAndFindTypes(row); + typesFoundReadingTable.addAll(typesFoundInRow); }); - return tableCellTypes; + return typesFoundReadingTable; } /** - * Creates and populates a set of all CellTypes in a table row + * Creates and populates a collection of all data element types in a table row * * @param row Table row is represented as a column-value map * @return */ - private Set getCellTypesInRow(Map row) { - Set rowCellTypes = new TreeSet<>(); + private Collection readRowAndFindTypes(Map row) { + Collection typesFoundReadingRow = new TreeSet<>(); //Aggregate cell types from a row - row.values().forEach((Object cell) -> { - CellType type = cellTypeDetector.getType(cell); - if(!type.equals(CellType.NOT_INTERESTING)) { - rowCellTypes.add(type); + row.values().forEach((Object dataElement) -> { + DataElementType type = DataElementTypeDetector.getType(dataElement); + if(isAnInterestingDataType(type)) { + typesFoundReadingRow.add(type); } }); - return rowCellTypes; + return typesFoundReadingRow; + } + + /** + * Boolean function purely for readability. Statement below is read as: + * if type is not not interesting -> isAnInterestingDataType. + * + * @param type + * @return + */ + private boolean isAnInterestingDataType(DataElementType type) { + return !type.equals(DataElementType.NOT_INTERESTING); } /** @@ -210,29 +224,29 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { * comment attributes * * @param file The database abstract file - * @param cellTypesComment String of all the cell types found in a database + * @param dataElementTypesInFile Collection of data types found during reading * file * @return Interesting file hit artifact * @throws TskCoreException Thrown if the abstract file cannot create a blackboard * artifact */ @NbBundle.Messages({ - "DatabaseSelectorIngestModule.FlagDatabases.setName=Selectors identified" + "FileSelectorIngestModule.setName=Selectors identified" }) private BlackboardArtifact createArtifact(AbstractFile file, - Set databaseCellTypes) throws TskCoreException { + Collection dataElementTypesInFile) throws TskCoreException { BlackboardArtifact artifact = file.newArtifact( BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); BlackboardAttribute setNameAttribute = new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, - Bundle.DatabaseSelectorIngestModule_FlagDatabases_setName()); + Bundle.FileSelectorIngestModule_setName()); artifact.addAttribute(setNameAttribute); - String cellTypesComment = createCellTypeCommentString(databaseCellTypes); + String dateTypesComment = dataElementTypesToString(dataElementTypesInFile); BlackboardAttribute commentAttribute = new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, - cellTypesComment); + dateTypesComment); artifact.addAttribute(commentAttribute); return artifact; @@ -243,11 +257,11 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { * file. Used as the comment string for the blackboard artifact. TreeSet is * used to ensure that CellTypes appear in the same order as the enum. * - * @param databaseCellTypes The set of all database cell types detected + * @param dataElementTypesInFile Collection of data types found during reading * @return */ - private String createCellTypeCommentString(Set databaseCellTypes) { - return databaseCellTypes.toString().replace("]", "").replace("[", ""); //NON-NLS + private String dataElementTypesToString(Collection dataElementTypesInFile) { + return dataElementTypesInFile.toString().replace("]", "").replace("[", ""); //NON-NLS } /** @@ -257,7 +271,7 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { * @param artifact Blackboard artifact created for the interesting file hit */ @NbBundle.Messages({ - "DatabaseSelectorIngestModule.indexError.message=" + "FileSelectorIngestModule.indexError.message=" + "Failed to index interesting file hit artifact for keyword search." }) private void indexArtifactAndFireModuleDataEvent(BlackboardArtifact artifact) { @@ -269,7 +283,7 @@ public class DatabaseSelectorIngestModule extends FileIngestModuleAdapter { "Unable to index blackboard artifact " + //NON-NLS artifact.getArtifactID(), ex); MessageNotifyUtil.Notify.error( - Bundle.DatabaseSelectorIngestModule_indexError_message(), + Bundle.FileSelectorIngestModule_indexError_message(), artifact.getDisplayName()); } diff --git a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModuleFactory.java similarity index 73% rename from Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java rename to Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModuleFactory.java index 155b26c425..02cc90f2e9 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/databaseselector/DatabaseSelectorIngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModuleFactory.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.modules.databaseselector; +package org.sleuthkit.autopsy.modules.fileselector; import org.sleuthkit.autopsy.coreutils.Version; import org.openide.util.NbBundle; @@ -31,14 +31,15 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; * mark them as having interesting records (emails, phone numbers, mac addresses, gps coordinates). */ @NbBundle.Messages({ - "DatabaseSelectorIngestModuleFactory.FlagDatabases.moduleName=Database Selector", - "DatabaseSelectorIngestModuleFactory.FlagDatabases.moduleDesc.text=Flag databases with interesting items (emails, phone numbers, gps coordinates, ip/mac addresses)" + "FileSelectorIngestModuleFactory.FlagFilesWithInterestingContents.moduleName=File Selector", + "FileSelectorIngestModuleFactory.FlagFilesWithInterestingContents.moduleDesc.text=" + + "Flag file with interesting contents (emails, phone numbers, gps coordinates, ip/mac addresses)" }) @ServiceProvider(service = IngestModuleFactory.class) -public class DatabaseSelectorIngestModuleFactory extends IngestModuleFactoryAdapter { +public class FileSelectorIngestModuleFactory extends IngestModuleFactoryAdapter { static String getModuleName() { - return Bundle.DatabaseSelectorIngestModuleFactory_FlagDatabases_moduleName(); + return Bundle.FileSelectorIngestModuleFactory_FlagFilesWithInterestingContents_moduleName(); } @Override @@ -48,7 +49,7 @@ public class DatabaseSelectorIngestModuleFactory extends IngestModuleFactoryAdap @Override public String getModuleDescription() { - return Bundle.DatabaseSelectorIngestModuleFactory_FlagDatabases_moduleDesc_text(); + return Bundle.FileSelectorIngestModuleFactory_FlagFilesWithInterestingContents_moduleDesc_text(); } @Override @@ -63,6 +64,6 @@ public class DatabaseSelectorIngestModuleFactory extends IngestModuleFactoryAdap @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { - return new DatabaseSelectorIngestModule(); + return new FileSelectorIngestModule(); } } \ No newline at end of file From ef76511f6b9d0b2dbce8a5c4b8ebc3fe87b1de73 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 10 Aug 2018 10:19:55 -0400 Subject: [PATCH 12/33] Drowning in checked exceptions --- .../FileSelectorIngestModule.java | 21 +++--- .../autopsy/sqlitereader/ExcelReader.java | 61 +++++++++++++++++ .../sqlitereader/FileReaderFactory.java | 38 +++++++++++ .../autopsy/sqlitereader/SQLiteReader.java | 60 ++++++++--------- .../sqlitereader/TabularFileReader.java | 66 +++++++++++++++++++ 5 files changed, 204 insertions(+), 42 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java create mode 100755 Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java create mode 100755 Core/src/org/sleuthkit/autopsy/sqlitereader/TabularFileReader.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java index 652030ba14..265910b93e 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.modules.fileselector; import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.sql.SQLException; import java.util.Collection; import java.util.Collections; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -40,7 +42,9 @@ import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.modules.fileselector.DataElementTypeDetector.DataElementType; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector.FileTypeDetectorInitException; +import org.sleuthkit.autopsy.sqlitereader.FileReaderFactory; import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; +import org.sleuthkit.autopsy.sqlitereader.TabularFileReader; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -92,9 +96,9 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { return ProcessResult.OK; } - try (SQLiteReader sqliteReader = new SQLiteReader(file, createLocalDiskPath(file))){ + try (TabularFileReader fileReader = FileReaderFactory.createReader(fileMimeType, file, createLocalDiskPath(file))){ - Collection dataElementTypesInFile = readFileAndFindTypes(file, sqliteReader); + Collection dataElementTypesInFile = readFileAndFindTypes(file, fileReader); //No interesting types found, no artifact to create if(dataElementTypesInFile.isEmpty()) { return ProcessResult.OK; @@ -106,9 +110,10 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS } - } catch (ClassNotFoundException | SQLException | IOException | - NoCurrentCaseException | TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Cannot initialize sqliteReader class " //NON-NLS + } catch (InstantiationException | IllegalAccessException | + IllegalArgumentException | InvocationTargetException | + NoSuchMethodException | NoCurrentCaseException ex) { + logger.log(Level.SEVERE, String.format("Cannot initialize fileReader class " //NON-NLS + "for file [%s].", file.getName()), ex); //NON-NLS return ProcessResult.ERROR; } @@ -136,7 +141,7 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { * @param sqliteReader Reader instance currently connected to local file contents * @return A collection of data element types */ - private Collection readFileAndFindTypes(AbstractFile file, SQLiteReader sqliteReader) { + private Collection readFileAndFindTypes(AbstractFile file, TabularFileReader sqliteReader) { Collection currentTypesFound = new TreeSet<>(); Map tables; @@ -150,7 +155,7 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { return currentTypesFound; } - //Aggregate cell types from all tables + //Aggregate data element types from all tables for(String tableName : tables.keySet()) { try { Collection typesFoundInTable = readTableAndFindTypes(sqliteReader, tableName); @@ -175,7 +180,7 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { * @return collection of all types in table * @throws SQLException Caught during attempting to read sqlite database */ - private Collection readTableAndFindTypes(SQLiteReader sqliteReader, + private Collection readTableAndFindTypes(TabularFileReader sqliteReader, String tableName) throws SQLException { Collection typesFoundReadingTable = new TreeSet<>(); diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java new file mode 100755 index 0000000000..eadc4d8697 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java @@ -0,0 +1,61 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.sqlitereader; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * + * @author dsmyda + */ +public class ExcelReader extends TabularFileReader { + static { + final String SUPPORTED_MIME_TYPE = "application/vnd.ms-excel"; + FileReaderFactory.registerReaderType(SUPPORTED_MIME_TYPE, ExcelReader.class); + } + + public ExcelReader(AbstractFile file, String localDiskPath) + throws IOException, TskCoreException { + super(file, localDiskPath); + } + + @Override + public Map getTableSchemas() throws Exception { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public Integer getRowCountFromTable(String tableName) throws Exception { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public List> getRowsFromTable(String tableName) throws Exception { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public void close() { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java new file mode 100755 index 0000000000..68baf4e612 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java @@ -0,0 +1,38 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.sqlitereader; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; +import org.sleuthkit.datamodel.AbstractFile; + +/** + * + * @author dsmyda + */ +public final class FileReaderFactory { + + private final static Map REGISTERED_READER_TYPES; + + static { + REGISTERED_READER_TYPES = new HashMap(); + } + + public static void registerReaderType(String mimeType, Class reader) { + REGISTERED_READER_TYPES.put(mimeType, reader); + } + + public static TabularFileReader createReader(String mimeType, AbstractFile file, String localDiskPath) + throws InstantiationException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException, NoSuchMethodException { + + Class readerClass = (Class) REGISTERED_READER_TYPES.get(mimeType); + Constructor readerConstructor = Class.class.getDeclaredConstructor(new Class[] {String.class, String.class}); + return (TabularFileReader) readerConstructor.newInstance(new Object[] {file, localDiskPath}); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java index e48042a032..ad747cf002 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java @@ -50,7 +50,12 @@ import org.sleuthkit.datamodel.TskCoreException; @NbBundle.Messages({ "SQLiteReader.ReadSQLiteFiles.moduleName=SQLiteReader" }) -public class SQLiteReader implements AutoCloseable { +public class SQLiteReader extends TabularFileReader { + + static { + final String SUPPORTED_MIME_TYPE = "application/x-sqlite3"; + FileReaderFactory.registerReaderType(SUPPORTED_MIME_TYPE, SQLiteReader.class); + } private final Connection connection; private final IngestServices services = IngestServices.getInstance(); @@ -71,37 +76,19 @@ public class SQLiteReader implements AutoCloseable { public SQLiteReader(AbstractFile sqliteDbFile, String localDiskPath) throws ClassNotFoundException, SQLException, IOException, NoCurrentCaseException, TskCoreException{ - writeDataSourceToLocalDisk(sqliteDbFile, localDiskPath); - connection = getDatabaseConnection(localDiskPath); - } - - /** - * Copies the data source file contents to local drive for processing. - * - * @param file AbstractFile from the data source - * @param localDiskPath Local drive path to copy AbstractFile contents - * @throws IOException Exception writing file contents - * @throws NoCurrentCaseException Current case closed during file copying - * @throws TskCoreException Exception finding files from abstract file - */ - private void writeDataSourceToLocalDisk(AbstractFile file, String localDiskPath) - throws IOException, NoCurrentCaseException, TskCoreException { + super(sqliteDbFile, localDiskPath); + // Look for any meta files associated with this DB - WAL, SHM, etc. + findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-wal"); + findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-shm"); - File localDatabaseFile = new File(localDiskPath); - if (!localDatabaseFile.exists()) { - ContentUtils.writeToFile(file, localDatabaseFile); - - // Look for any meta files associated with this DB - WAL, SHM, etc. - findAndCopySQLiteMetaFile(file, file.getName() + "-wal"); - findAndCopySQLiteMetaFile(file, file.getName() + "-shm"); - } + connection = getDatabaseConnection(localDiskPath); } /** * Searches for a meta file associated with the give SQLite database. If found, * copies the file to the local disk folder * - * @param sqliteFile SQLIte db file being processed + * @param file file being processed * @param metaFileName name of meta file to look for * @throws NoCurrentCaseException Case has been closed. * @throws TskCoreException fileManager cannot find AbstractFile files. @@ -128,7 +115,7 @@ public class SQLiteReader implements AutoCloseable { } } } - + /** * Opens a JDBC connection to the sqlite database specified by the path * parameter. @@ -155,8 +142,8 @@ public class SQLiteReader implements AutoCloseable { * @return A map of table names to table schemas * @throws SQLException */ - public Map getTableSchemas() - throws SQLException { + @Override + public Map getTableSchemas() { Map dbTablesMap = new TreeMap<>(); @@ -166,11 +153,14 @@ public class SQLiteReader implements AutoCloseable { + " WHERE type= 'table' " //NON-NLS + " ORDER BY name;")){ //NON-NLS - while (resultSet.next()) { - String tableName = resultSet.getString("name"); //NON-NLS - String tableSQL = resultSet.getString("sql"); //NON-NLS - dbTablesMap.put(tableName, tableSQL); - } + while (resultSet.next()) { + String tableName = resultSet.getString("name"); //NON-NLS + String tableSQL = resultSet.getString("sql"); //NON-NLS + dbTablesMap.put(tableName, tableSQL); + } + + } catch (SQLException ex) { + throw new RuntimeException(ex); } return dbTablesMap; @@ -183,7 +173,8 @@ public class SQLiteReader implements AutoCloseable { * @return Row count from tableName * @throws SQLException */ - public Integer getTableRowCount(String tableName) throws SQLException { + @Override + public Integer getRowCountFromTable(String tableName) throws SQLException { tableName = wrapTableNameStringWithQuotes(tableName); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( @@ -201,6 +192,7 @@ public class SQLiteReader implements AutoCloseable { * represented as a column-value map. * @throws SQLException */ + @Override public List> getRowsFromTable(String tableName) throws SQLException { //This method does not directly call its overloaded counterpart //since the second parameter would need to be retreived from a call to diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/TabularFileReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/TabularFileReader.java new file mode 100755 index 0000000000..da31e2eb02 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/TabularFileReader.java @@ -0,0 +1,66 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.sqlitereader; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * + * @author dsmyda + */ +public abstract class TabularFileReader implements AutoCloseable { + + public TabularFileReader(AbstractFile file, String localDiskPath) + throws IOException, TskCoreException { + writeDataSourceToLocalDisk(file, localDiskPath); + } + + /** + * Copies the data source file contents to local drive for processing. + * + * @param file AbstractFile from the data source + * @param localDiskPath Local drive path to copy AbstractFile contents + * @throws IOException Exception writing file contents + * @throws NoCurrentCaseException Current case closed during file copying + * @throws TskCoreException Exception finding files from abstract file + */ + private void writeDataSourceToLocalDisk(AbstractFile file, String localDiskPath) + throws IOException, TskCoreException { + + File localDatabaseFile = new File(localDiskPath); + if (!localDatabaseFile.exists()) { + ContentUtils.writeToFile(file, localDatabaseFile); + } + } + + public abstract Map getTableSchemas(); + + public abstract Integer getRowCountFromTable(String tableName); + + public abstract List> getRowsFromTable(String tableName); + + @Override + public abstract void close(); +} From f535be8e3e2e6c698e14ccbc191752bac7e040bc Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 10 Aug 2018 18:17:16 -0400 Subject: [PATCH 13/33] Adjusted infrastructure and added support for reading excel files --- .../autopsy/contentviewers/SQLiteViewer.java | 33 +++-- .../FileSelectorIngestModule.java | 48 +++---- ...larFileReader.java => AbstractReader.java} | 55 ++++++-- .../autopsy/sqlitereader/ExcelReader.java | 122 +++++++++++++++--- .../sqlitereader/FileReaderFactory.java | 33 ++--- .../autopsy/sqlitereader/SQLiteReader.java | 44 ++++--- 6 files changed, 225 insertions(+), 110 deletions(-) rename Core/src/org/sleuthkit/autopsy/sqlitereader/{TabularFileReader.java => AbstractReader.java} (54%) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 4324a3dc06..076748d2af 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -38,6 +38,7 @@ import javax.swing.JFileChooser; import javax.swing.JOptionPane; import javax.swing.filechooser.FileNameExtensionFilter; import org.apache.commons.io.FilenameUtils; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; @@ -46,6 +47,9 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException; +import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException; +import org.sleuthkit.autopsy.sqlitereader.FileReaderFactory; import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; /** @@ -362,7 +366,8 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { try { String localDiskPath = Case.getCurrentCaseThrows().getTempDirectory() + File.separator + sqliteDbFile.getName(); - sqliteReader = new SQLiteReader(sqliteDbFile, localDiskPath); + + sqliteReader = (SQLiteReader) FileReaderFactory.createReader(SUPPORTED_MIMETYPES[0], sqliteDbFile, localDiskPath); Map dbTablesMap = sqliteReader.getTableSchemas(); @@ -377,24 +382,16 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Current case has been closed", ex); //NON-NLS MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_errorMessage_noCurrentCase()); - } catch (IOException | TskCoreException ex) { - logger.log(Level.SEVERE, String.format( - "Failed to create temp copy of DB file '%s' (objId=%d)", //NON-NLS - sqliteDbFile.getName(), sqliteDbFile.getId()), ex); - MessageNotifyUtil.Message.error( - Bundle.SQLiteViewer_errorMessage_failedToExtractFile()); - } catch (ClassNotFoundException ex) { - logger.log(Level.SEVERE, String.format( - "Failed to initialize JDBC SQLite '%s' (objId=%d)", //NON-NLS - sqliteDbFile.getName(), sqliteDbFile.getId()), ex); - MessageNotifyUtil.Message.error( - Bundle.SQLiteViewer_errorMessage_failedToinitJDBCDriver()); - } catch (SQLException ex) { + } catch (FileReaderException ex) { logger.log(Level.SEVERE, String.format( "Failed to get tables from DB file '%s' (objId=%d)", //NON-NLS sqliteDbFile.getName(), sqliteDbFile.getId()), ex); MessageNotifyUtil.Message.error( Bundle.SQLiteViewer_errorMessage_failedToQueryDatabase()); + } catch (FileReaderInitException ex) { + logger.log(Level.SEVERE, String.format( + "Failed to create a SQLiteReader '%s' (objId=%d)", //NON-NLS + sqliteDbFile.getName(), sqliteDbFile.getId()), ex); } } @@ -403,7 +400,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { }) private void selectTable(String tableName) { try { - numRows = sqliteReader.getTableRowCount(tableName); + numRows = sqliteReader.getRowCountFromTable(tableName); numEntriesField.setText(numRows + " entries"); currPage = 1; @@ -422,7 +419,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { selectedTableView.setupTable(Collections.emptyList()); } - } catch (SQLException ex) { + } catch (FileReaderException ex) { logger.log(Level.SEVERE, String.format( "Failed to load table %s from DB file '%s' (objId=%d)", tableName, //NON-NLS sqliteDbFile.getName(), sqliteDbFile.getId()), ex); @@ -443,7 +440,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } else { selectedTableView.setupTable(Collections.emptyList()); } - } catch (SQLException ex) { + } catch (FileReaderException ex) { logger.log(Level.SEVERE, String.format( "Failed to read table %s from DB file '%s' (objId=%d)", tableName, //NON-NLS sqliteDbFile.getName(), sqliteDbFile.getId()), ex); @@ -512,7 +509,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } else { exportTableToCSV(file, tableName, currentTableRows); } - } catch (SQLException ex) { + } catch (FileReaderException ex) { logger.log(Level.SEVERE, String.format( "Failed to read table %s from DB file '%s' (objId=%d)", //NON-NLS tableName, sqliteDbFile.getName(), sqliteDbFile.getId()), ex); diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java index 265910b93e..54c91fc34a 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.sql.SQLException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -43,8 +44,9 @@ import org.sleuthkit.autopsy.modules.fileselector.DataElementTypeDetector.DataEl import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector.FileTypeDetectorInitException; import org.sleuthkit.autopsy.sqlitereader.FileReaderFactory; -import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; -import org.sleuthkit.autopsy.sqlitereader.TabularFileReader; +import org.sleuthkit.autopsy.sqlitereader.AbstractReader; +import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException; +import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -56,7 +58,7 @@ import org.sleuthkit.datamodel.TskCoreException; */ public class FileSelectorIngestModule extends FileIngestModuleAdapter { - private static final String SUPPORTED_MIME_TYPE = "application/x-sqlite3"; + private static final List SUPPORTED_MIMETYPES = Arrays.asList(new String[]{"application/x-sqlite3", "application/vnd.ms-excel"}); private static final String MODULE_NAME = FileSelectorIngestModuleFactory.getModuleName(); private final IngestServices services = IngestServices.getInstance(); @@ -66,37 +68,35 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { private Blackboard blackboard; - @NbBundle.Messages({ "FileSelectorIngestModule.CannotRunFileTypeDetection=" + "Unable to initialize file type detection.", "FileSelectorIngestModule.CannotGetBlackboard=" - + "Exception while attempting to get Blackboard from current case." + + "Exception while attempting to get Blackboard from current case.", + "FileSelectorIngestModule.CannotLoadReaderClasses=" + + "Cannot load reader class dependencies." }) @Override public void startUp(IngestJobContext context) throws IngestModuleException { try { fileTypeDetector = new FileTypeDetector(); + blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); } catch (FileTypeDetectorInitException ex) { throw new IngestModuleException(Bundle.FileSelectorIngestModule_CannotRunFileTypeDetection(), ex); - } - - try { - blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); } catch (NoCurrentCaseException ex) { throw new IngestModuleException(Bundle.FileSelectorIngestModule_CannotGetBlackboard(), ex); - } + } } @Override public ProcessResult process(AbstractFile file) { //Qualify the MIMEType, only process sqlite files. String fileMimeType = fileTypeDetector.getMIMEType(file); - if(!fileMimeType.equals(SUPPORTED_MIME_TYPE)) { + if(!SUPPORTED_MIMETYPES.contains(fileMimeType)) { return ProcessResult.OK; } - try (TabularFileReader fileReader = FileReaderFactory.createReader(fileMimeType, file, createLocalDiskPath(file))){ + try (AbstractReader fileReader = FileReaderFactory.createReader(fileMimeType, file, createLocalDiskPath(file))){ Collection dataElementTypesInFile = readFileAndFindTypes(file, fileReader); //No interesting types found, no artifact to create @@ -110,9 +110,7 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS } - } catch (InstantiationException | IllegalAccessException | - IllegalArgumentException | InvocationTargetException | - NoSuchMethodException | NoCurrentCaseException ex) { + } catch (FileReaderInitException | NoCurrentCaseException ex) { logger.log(Level.SEVERE, String.format("Cannot initialize fileReader class " //NON-NLS + "for file [%s].", file.getName()), ex); //NON-NLS return ProcessResult.ERROR; @@ -141,16 +139,18 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { * @param sqliteReader Reader instance currently connected to local file contents * @return A collection of data element types */ - private Collection readFileAndFindTypes(AbstractFile file, TabularFileReader sqliteReader) { + private Collection readFileAndFindTypes(AbstractFile file, AbstractReader sqliteReader) { Collection currentTypesFound = new TreeSet<>(); Map tables; try { - tables = sqliteReader.getTableSchemas(); - } catch (SQLException ex) { + tables = sqliteReader.getTableSchemas(); //NON-NLS + //NON-NLS + //Unable to read anything, return empty collection. + } catch (FileReaderException ex) { logger.log(Level.WARNING, String.format("Error attempting to get tables from sqlite" //NON-NLS - + "file [%s].", //NON-NLS - file.getName()), ex); + + "file [%s].", //NON-NLS + file.getName()), ex); //Unable to read anything, return empty collection. return currentTypesFound; } @@ -160,7 +160,7 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { try { Collection typesFoundInTable = readTableAndFindTypes(sqliteReader, tableName); currentTypesFound.addAll(typesFoundInTable); - } catch (SQLException ex) { + } catch (FileReaderException ex) { logger.log(Level.WARNING, String.format("Error attempting to read sqlite table [%s]" //NON-NLS + " for file [%s].", //NON-NLS @@ -180,8 +180,8 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { * @return collection of all types in table * @throws SQLException Caught during attempting to read sqlite database */ - private Collection readTableAndFindTypes(TabularFileReader sqliteReader, - String tableName) throws SQLException { + private Collection readTableAndFindTypes(AbstractReader sqliteReader, + String tableName) throws FileReaderException { Collection typesFoundReadingTable = new TreeSet<>(); List> tableValues = sqliteReader.getRowsFromTable(tableName); @@ -204,7 +204,7 @@ public class FileSelectorIngestModule extends FileIngestModuleAdapter { Collection typesFoundReadingRow = new TreeSet<>(); //Aggregate cell types from a row - row.values().forEach((Object dataElement) -> { + row.values().forEach((dataElement) -> { DataElementType type = DataElementTypeDetector.getType(dataElement); if(isAnInterestingDataType(type)) { typesFoundReadingRow.add(type); diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/TabularFileReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/AbstractReader.java similarity index 54% rename from Core/src/org/sleuthkit/autopsy/sqlitereader/TabularFileReader.java rename to Core/src/org/sleuthkit/autopsy/sqlitereader/AbstractReader.java index da31e2eb02..46f4242454 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/TabularFileReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/AbstractReader.java @@ -30,10 +30,11 @@ import org.sleuthkit.datamodel.TskCoreException; * * @author dsmyda */ -public abstract class TabularFileReader implements AutoCloseable { +public abstract class AbstractReader implements AutoCloseable { - public TabularFileReader(AbstractFile file, String localDiskPath) - throws IOException, TskCoreException { + public AbstractReader(AbstractFile file, String localDiskPath) + throws FileReaderInitException { + writeDataSourceToLocalDisk(file, localDiskPath); } @@ -47,20 +48,52 @@ public abstract class TabularFileReader implements AutoCloseable { * @throws TskCoreException Exception finding files from abstract file */ private void writeDataSourceToLocalDisk(AbstractFile file, String localDiskPath) - throws IOException, TskCoreException { + throws FileReaderInitException { - File localDatabaseFile = new File(localDiskPath); - if (!localDatabaseFile.exists()) { - ContentUtils.writeToFile(file, localDatabaseFile); + try { + File localDatabaseFile = new File(localDiskPath); + if (!localDatabaseFile.exists()) { + ContentUtils.writeToFile(file, localDatabaseFile); + } + } catch (IOException ex) { + throw new FileReaderInitException(ex); } } - public abstract Map getTableSchemas(); + public abstract Map getTableSchemas() throws FileReaderException; - public abstract Integer getRowCountFromTable(String tableName); + public abstract Integer getRowCountFromTable(String tableName) throws FileReaderException; - public abstract List> getRowsFromTable(String tableName); + public abstract List> getRowsFromTable(String tableName) throws FileReaderException; @Override - public abstract void close(); + public abstract void close(); + + public static class FileReaderInitException extends Exception { + public FileReaderInitException(String message, Throwable cause) { + super(message, cause); + } + + public FileReaderInitException(Throwable cause) { + super(cause); + } + + public FileReaderInitException(String message) { + super(message); + } + } + + public class FileReaderException extends Exception { + public FileReaderException(String message, Throwable cause) { + super(message, cause); + } + + public FileReaderException(Throwable cause) { + super(cause); + } + + public FileReaderException(String message) { + super(message); + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java index eadc4d8697..4222a679af 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java @@ -18,44 +18,132 @@ */ package org.sleuthkit.autopsy.sqlitereader; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; /** * * @author dsmyda */ -public class ExcelReader extends TabularFileReader { - static { - final String SUPPORTED_MIME_TYPE = "application/vnd.ms-excel"; - FileReaderFactory.registerReaderType(SUPPORTED_MIME_TYPE, ExcelReader.class); - } - +public class ExcelReader extends AbstractReader { + + private Workbook xlsWorkbook; + private final IngestServices services = IngestServices.getInstance(); + private final Logger logger = services.getLogger(Bundle.SQLiteReader_ReadSQLiteFiles_moduleName()); + public ExcelReader(AbstractFile file, String localDiskPath) - throws IOException, TskCoreException { + throws FileReaderInitException { super(file, localDiskPath); + + try { + getWorkbookFromLocalDisk(localDiskPath); + } catch (IOException ex) { + throw new FileReaderInitException(ex); + } + } + + private void getWorkbookFromLocalDisk(String localDiskPath) throws IOException{ + xlsWorkbook = new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); } @Override - public Map getTableSchemas() throws Exception { - throw new UnsupportedOperationException("Not supported yet."); + public Integer getRowCountFromTable(String tableName) throws FileReaderException { + return xlsWorkbook.getSheet(tableName).getLastRowNum(); } @Override - public Integer getRowCountFromTable(String tableName) throws Exception { - throw new UnsupportedOperationException("Not supported yet."); + public List> getRowsFromTable(String tableName) throws FileReaderException { + List> rowContents = new ArrayList<>(); + Iterator iterator = xlsWorkbook.getSheet(tableName).rowIterator(); + //Consume header + if(iterator.hasNext()) { + iterator.next(); + } + while(iterator.hasNext()) { + Map contents = new HashMap<>(); + Row r = iterator.next(); + for(Cell c : r) { + addCellValueToMap(c, contents); + } + rowContents.add(contents); + } + return rowContents; + } + + private void addCellValueToMap(Cell cell, Map contents){ + switch (cell.getCellTypeEnum()) { + case BOOLEAN: + contents.put(getCellValuesColumnName(cell), cell.getBooleanCellValue()); + break; + case STRING: + contents.put(getCellValuesColumnName(cell), cell.getRichStringCellValue().getString()); + break; + case NUMERIC: + if (DateUtil.isCellDateFormatted(cell)) { + contents.put(getCellValuesColumnName(cell), cell.getDateCellValue()); + } else { + contents.put(getCellValuesColumnName(cell), cell.getNumericCellValue()); + } + break; + case FORMULA: + contents.put(getCellValuesColumnName(cell), cell.getCellFormula()); + break; + default: + contents.put(getCellValuesColumnName(cell),""); + } + } + + private String getCellValuesColumnName(Cell cell) { + Row header = cell.getSheet().getRow(0); + // if(header != null) { + // header.getCell(cell.getRowIndex()).getRichStringCellValue().toString(); + //} + return ""; } @Override - public List> getRowsFromTable(String tableName) throws Exception { - throw new UnsupportedOperationException("Not supported yet."); + public Map getTableSchemas() throws FileReaderException { + Map tableSchemas = new HashMap<>(); + for(Sheet sheet : xlsWorkbook) { + Row header = sheet.getRow(0); + if(header != null) { + tableSchemas.put(sheet.getSheetName(), poiRowToString(sheet.getRow(0))); + } else { + tableSchemas.put(sheet.getSheetName(), ""); + } + } + return tableSchemas; } - - @Override + + private String poiRowToString(Row row) { + return StringUtils.join(row.cellIterator(), ", "); + } + + @Override public void close() { - throw new UnsupportedOperationException("Not supported yet."); + try { + xlsWorkbook.close(); + } catch (IOException ex) { + //Non-essential exception, user has no need for the connection + //object at this stage so closing details are not important + logger.log(Level.WARNING, "Could not close excel file input stream", ex); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java index 68baf4e612..a6cee9f917 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java @@ -5,10 +5,7 @@ */ package org.sleuthkit.autopsy.sqlitereader; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.HashMap; -import java.util.Map; +import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException; import org.sleuthkit.datamodel.AbstractFile; /** @@ -17,22 +14,16 @@ import org.sleuthkit.datamodel.AbstractFile; */ public final class FileReaderFactory { - private final static Map REGISTERED_READER_TYPES; - - static { - REGISTERED_READER_TYPES = new HashMap(); - } - - public static void registerReaderType(String mimeType, Class reader) { - REGISTERED_READER_TYPES.put(mimeType, reader); - } - - public static TabularFileReader createReader(String mimeType, AbstractFile file, String localDiskPath) - throws InstantiationException, IllegalAccessException, - IllegalArgumentException, InvocationTargetException, NoSuchMethodException { - - Class readerClass = (Class) REGISTERED_READER_TYPES.get(mimeType); - Constructor readerConstructor = Class.class.getDeclaredConstructor(new Class[] {String.class, String.class}); - return (TabularFileReader) readerConstructor.newInstance(new Object[] {file, localDiskPath}); + public static AbstractReader createReader(String mimeType, AbstractFile file, + String localDiskPath) throws FileReaderInitException { + switch(mimeType) { + case "application/x-sqlite3": + return new SQLiteReader(file, localDiskPath); + case "application/vnd.ms-excel": + return new ExcelReader(file, localDiskPath); + default: + throw new FileReaderInitException(String.format("Reader for mime " + + "type [%s] is not supported", mimeType)); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java index ad747cf002..f3d0aec277 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java @@ -50,12 +50,7 @@ import org.sleuthkit.datamodel.TskCoreException; @NbBundle.Messages({ "SQLiteReader.ReadSQLiteFiles.moduleName=SQLiteReader" }) -public class SQLiteReader extends TabularFileReader { - - static { - final String SUPPORTED_MIME_TYPE = "application/x-sqlite3"; - FileReaderFactory.registerReaderType(SUPPORTED_MIME_TYPE, SQLiteReader.class); - } +public class SQLiteReader extends AbstractReader { private final Connection connection; private final IngestServices services = IngestServices.getInstance(); @@ -73,15 +68,18 @@ public class SQLiteReader extends TabularFileReader { * @throws NoCurrentCaseException Current case closed during file copying * @throws TskCoreException Exception finding files from abstract file */ - public SQLiteReader(AbstractFile sqliteDbFile, String localDiskPath) throws ClassNotFoundException, - SQLException, IOException, NoCurrentCaseException, TskCoreException{ - + public SQLiteReader(AbstractFile sqliteDbFile, String localDiskPath) throws FileReaderInitException { super(sqliteDbFile, localDiskPath); - // Look for any meta files associated with this DB - WAL, SHM, etc. - findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-wal"); - findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-shm"); - - connection = getDatabaseConnection(localDiskPath); + try { + // Look for any meta files associated with this DB - WAL, SHM, etc. + findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-wal"); + findAndCopySQLiteMetaFile(sqliteDbFile, sqliteDbFile.getName() + "-shm"); + + connection = getDatabaseConnection(localDiskPath); + } catch (ClassNotFoundException | SQLException |IOException | + NoCurrentCaseException | TskCoreException ex) { + throw new FileReaderInitException(ex); + } } /** @@ -143,7 +141,7 @@ public class SQLiteReader extends TabularFileReader { * @throws SQLException */ @Override - public Map getTableSchemas() { + public Map getTableSchemas() throws FileReaderException { Map dbTablesMap = new TreeMap<>(); @@ -160,7 +158,7 @@ public class SQLiteReader extends TabularFileReader { } } catch (SQLException ex) { - throw new RuntimeException(ex); + throw new FileReaderException(ex); } return dbTablesMap; @@ -174,12 +172,15 @@ public class SQLiteReader extends TabularFileReader { * @throws SQLException */ @Override - public Integer getRowCountFromTable(String tableName) throws SQLException { + public Integer getRowCountFromTable(String tableName) + throws FileReaderException { tableName = wrapTableNameStringWithQuotes(tableName); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( "SELECT count (*) as count FROM " + tableName)){ //NON-NLS return resultSet.getInt("count"); //NON-NLS + } catch (SQLException ex) { + throw new FileReaderException(ex); } } @@ -193,7 +194,8 @@ public class SQLiteReader extends TabularFileReader { * @throws SQLException */ @Override - public List> getRowsFromTable(String tableName) throws SQLException { + public List> getRowsFromTable(String tableName) + throws FileReaderException { //This method does not directly call its overloaded counterpart //since the second parameter would need to be retreived from a call to //getTableRowCount(). @@ -202,6 +204,8 @@ public class SQLiteReader extends TabularFileReader { ResultSet resultSet = statement.executeQuery( "SELECT * FROM " + tableName)) { //NON-NLS return resultSetToList(resultSet); + } catch (SQLException ex) { + throw new FileReaderException(ex); } } @@ -216,7 +220,7 @@ public class SQLiteReader extends TabularFileReader { * @throws SQLException */ public List> getRowsFromTable(String tableName, - int startRow, int numRowsToRead) throws SQLException{ + int startRow, int numRowsToRead) throws FileReaderException{ tableName = wrapTableNameStringWithQuotes(tableName); try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( @@ -224,6 +228,8 @@ public class SQLiteReader extends TabularFileReader { + " LIMIT " + Integer.toString(numRowsToRead) //NON-NLS + " OFFSET " + Integer.toString(startRow - 1))) { //NON-NLS return resultSetToList(resultSet); + } catch (SQLException ex) { + throw new FileReaderException(ex); } } From 6ecffa3f02b635c4269040f173fe67a081b8db58 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 10 Aug 2018 18:23:00 -0400 Subject: [PATCH 14/33] Small bundle comment change --- .../src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java index 4222a679af..7c08e309cb 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java @@ -34,6 +34,7 @@ import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; @@ -42,11 +43,14 @@ import org.sleuthkit.datamodel.AbstractFile; * * @author dsmyda */ +@NbBundle.Messages({ + "ExcelReader.ReadExcelFiles.moduleName=ExcelReader" +}) public class ExcelReader extends AbstractReader { private Workbook xlsWorkbook; private final IngestServices services = IngestServices.getInstance(); - private final Logger logger = services.getLogger(Bundle.SQLiteReader_ReadSQLiteFiles_moduleName()); + private final Logger logger = services.getLogger(Bundle.ExcelReader_ReadExcelFiles_moduleName()); public ExcelReader(AbstractFile file, String localDiskPath) throws FileReaderInitException { From 3fd1d1cf0549ea8884b6a20c1964405dd55e05c9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 14 Aug 2018 13:08:16 -0400 Subject: [PATCH 15/33] More commits coming, need to resolve ivy dependencies --- .../fileselector/DataElementTypeDetector.java | 55 ---- .../FileSelectorIngestModule.java | 299 ------------------ .../FileSelectorIngestModuleFactory.java | 69 ---- .../autopsy/sqlitereader/ExcelReader.java | 84 +++-- .../sqlitereader/FileReaderFactory.java | 30 +- .../autopsy/sqlitereader/SQLiteReader.java | 16 +- 6 files changed, 96 insertions(+), 457 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/modules/fileselector/DataElementTypeDetector.java delete mode 100755 Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java delete mode 100755 Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModuleFactory.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileselector/DataElementTypeDetector.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/DataElementTypeDetector.java deleted file mode 100755 index ba532804b8..0000000000 --- a/Core/src/org/sleuthkit/autopsy/modules/fileselector/DataElementTypeDetector.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.modules.fileselector; - -import org.apache.commons.validator.routines.EmailValidator; - -/** - * Determines the type of a database cell (EMAIL, PHONE, GPS COORD, MAC ADDRESS) - */ -final class DataElementTypeDetector { - - private static final EmailValidator EMAIL_VALIDATOR; - - static { - EMAIL_VALIDATOR = EmailValidator.getInstance(); - } - - public static DataElementType getType(Object cell) { - if (cell instanceof String) { - return getType((String) cell); - } - return DataElementType.NOT_INTERESTING; - } - - public static DataElementType getType(String cell) { - if(EMAIL_VALIDATOR.isValid(cell)) { - return DataElementType.EMAIL; - } - return DataElementType.NOT_INTERESTING; - } - - /* - * Cell data that qualify as an "interesting" selector - */ - public enum DataElementType { - EMAIL, - NOT_INTERESTING - } -} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java deleted file mode 100755 index 54c91fc34a..0000000000 --- a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModule.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.modules.fileselector; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.TreeSet; -import java.util.logging.Level; -import org.openide.util.Exceptions; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.casemodule.services.Blackboard; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter; -import org.sleuthkit.autopsy.ingest.IngestJobContext; -import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.autopsy.ingest.ModuleDataEvent; -import org.sleuthkit.autopsy.modules.fileselector.DataElementTypeDetector.DataElementType; -import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; -import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector.FileTypeDetectorInitException; -import org.sleuthkit.autopsy.sqlitereader.FileReaderFactory; -import org.sleuthkit.autopsy.sqlitereader.AbstractReader; -import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException; -import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.TskCoreException; - -/** - * Parses database files and marks them as having interesting records (emails, - * phone numbers, mac addresses, gps coordinates). - */ -public class FileSelectorIngestModule extends FileIngestModuleAdapter { - - private static final List SUPPORTED_MIMETYPES = Arrays.asList(new String[]{"application/x-sqlite3", "application/vnd.ms-excel"}); - private static final String MODULE_NAME = FileSelectorIngestModuleFactory.getModuleName(); - - private final IngestServices services = IngestServices.getInstance(); - private final Logger logger = services.getLogger( - FileSelectorIngestModuleFactory.getModuleName()); - private FileTypeDetector fileTypeDetector; - - private Blackboard blackboard; - - @NbBundle.Messages({ - "FileSelectorIngestModule.CannotRunFileTypeDetection=" - + "Unable to initialize file type detection.", - "FileSelectorIngestModule.CannotGetBlackboard=" - + "Exception while attempting to get Blackboard from current case.", - "FileSelectorIngestModule.CannotLoadReaderClasses=" - + "Cannot load reader class dependencies." - }) - @Override - public void startUp(IngestJobContext context) throws IngestModuleException { - try { - fileTypeDetector = new FileTypeDetector(); - blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); - } catch (FileTypeDetectorInitException ex) { - throw new IngestModuleException(Bundle.FileSelectorIngestModule_CannotRunFileTypeDetection(), ex); - } catch (NoCurrentCaseException ex) { - throw new IngestModuleException(Bundle.FileSelectorIngestModule_CannotGetBlackboard(), ex); - } - } - - @Override - public ProcessResult process(AbstractFile file) { - //Qualify the MIMEType, only process sqlite files. - String fileMimeType = fileTypeDetector.getMIMEType(file); - if(!SUPPORTED_MIMETYPES.contains(fileMimeType)) { - return ProcessResult.OK; - } - - try (AbstractReader fileReader = FileReaderFactory.createReader(fileMimeType, file, createLocalDiskPath(file))){ - - Collection dataElementTypesInFile = readFileAndFindTypes(file, fileReader); - //No interesting types found, no artifact to create - if(dataElementTypesInFile.isEmpty()) { - return ProcessResult.OK; - } - - try { - BlackboardArtifact artifact = createArtifact(file, dataElementTypesInFile); - indexArtifactAndFireModuleDataEvent(artifact); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error creating blackboard artifact", ex); //NON-NLS - } - } catch (FileReaderInitException | NoCurrentCaseException ex) { - logger.log(Level.SEVERE, String.format("Cannot initialize fileReader class " //NON-NLS - + "for file [%s].", file.getName()), ex); //NON-NLS - return ProcessResult.ERROR; - } - - return ProcessResult.OK; - } - - /** - * Generates a local disk path for abstract file contents to be copied. - * All database sources must be copied to local disk to be opened by - * SQLiteReader - * - * @param file The database abstract file - * @return Valid local path for copying - * @throws NoCurrentCaseException if the current case has been closed. - */ - private String createLocalDiskPath(AbstractFile file) throws NoCurrentCaseException { - return Case.getCurrentCaseThrows().getTempDirectory() + - File.separator + file.getName(); - } - - /** - * Creates and populates a collection of all data element types in the file - * - * @param sqliteReader Reader instance currently connected to local file contents - * @return A collection of data element types - */ - private Collection readFileAndFindTypes(AbstractFile file, AbstractReader sqliteReader) { - Collection currentTypesFound = new TreeSet<>(); - - Map tables; - try { - tables = sqliteReader.getTableSchemas(); //NON-NLS - //NON-NLS - //Unable to read anything, return empty collection. - } catch (FileReaderException ex) { - logger.log(Level.WARNING, String.format("Error attempting to get tables from sqlite" //NON-NLS - + "file [%s].", //NON-NLS - file.getName()), ex); - //Unable to read anything, return empty collection. - return currentTypesFound; - } - - //Aggregate data element types from all tables - for(String tableName : tables.keySet()) { - try { - Collection typesFoundInTable = readTableAndFindTypes(sqliteReader, tableName); - currentTypesFound.addAll(typesFoundInTable); - } catch (FileReaderException ex) { - logger.log(Level.WARNING, - String.format("Error attempting to read sqlite table [%s]" //NON-NLS - + " for file [%s].", //NON-NLS - tableName, file.getName()), ex); - } - } - - return currentTypesFound; - } - - /** - * Creates and populates a collection of all data element types in a sqlite - * database table - * - * @param sqliteReader Reader currently connected to local file contents - * @param tableName database table to be opened and read - * @return collection of all types in table - * @throws SQLException Caught during attempting to read sqlite database - */ - private Collection readTableAndFindTypes(AbstractReader sqliteReader, - String tableName) throws FileReaderException { - - Collection typesFoundReadingTable = new TreeSet<>(); - List> tableValues = sqliteReader.getRowsFromTable(tableName); - - //Aggregate cell types from all rows - tableValues.forEach((row) -> { - Collection typesFoundInRow = readRowAndFindTypes(row); - typesFoundReadingTable.addAll(typesFoundInRow); - }); - return typesFoundReadingTable; - } - - /** - * Creates and populates a collection of all data element types in a table row - * - * @param row Table row is represented as a column-value map - * @return - */ - private Collection readRowAndFindTypes(Map row) { - Collection typesFoundReadingRow = new TreeSet<>(); - - //Aggregate cell types from a row - row.values().forEach((dataElement) -> { - DataElementType type = DataElementTypeDetector.getType(dataElement); - if(isAnInterestingDataType(type)) { - typesFoundReadingRow.add(type); - } - }); - return typesFoundReadingRow; - } - - /** - * Boolean function purely for readability. Statement below is read as: - * if type is not not interesting -> isAnInterestingDataType. - * - * @param type - * @return - */ - private boolean isAnInterestingDataType(DataElementType type) { - return !type.equals(DataElementType.NOT_INTERESTING); - } - - /** - * Initializes a new interesting file hit artifact and provides name and - * comment attributes - * - * @param file The database abstract file - * @param dataElementTypesInFile Collection of data types found during reading - * file - * @return Interesting file hit artifact - * @throws TskCoreException Thrown if the abstract file cannot create a blackboard - * artifact - */ - @NbBundle.Messages({ - "FileSelectorIngestModule.setName=Selectors identified" - }) - private BlackboardArtifact createArtifact(AbstractFile file, - Collection dataElementTypesInFile) throws TskCoreException { - BlackboardArtifact artifact = file.newArtifact( - BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); - - BlackboardAttribute setNameAttribute = new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, - Bundle.FileSelectorIngestModule_setName()); - artifact.addAttribute(setNameAttribute); - - String dateTypesComment = dataElementTypesToString(dataElementTypesInFile); - BlackboardAttribute commentAttribute = new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, - dateTypesComment); - artifact.addAttribute(commentAttribute); - - return artifact; - } - - /** - * Creates a comma seperated string of all the cell types found in a database - * file. Used as the comment string for the blackboard artifact. TreeSet is - * used to ensure that CellTypes appear in the same order as the enum. - * - * @param dataElementTypesInFile Collection of data types found during reading - * @return - */ - private String dataElementTypesToString(Collection dataElementTypesInFile) { - return dataElementTypesInFile.toString().replace("]", "").replace("[", ""); //NON-NLS - } - - /** - * Pass the artifact to blackboard for indexing and fire module data event - * in the IngestServices - * - * @param artifact Blackboard artifact created for the interesting file hit - */ - @NbBundle.Messages({ - "FileSelectorIngestModule.indexError.message=" - + "Failed to index interesting file hit artifact for keyword search." - }) - private void indexArtifactAndFireModuleDataEvent(BlackboardArtifact artifact) { - try { - // index the artifact for keyword search - blackboard.indexArtifact(artifact); - } catch (Blackboard.BlackboardException ex) { - logger.log(Level.SEVERE, - "Unable to index blackboard artifact " + //NON-NLS - artifact.getArtifactID(), ex); - MessageNotifyUtil.Notify.error( - Bundle.FileSelectorIngestModule_indexError_message(), - artifact.getDisplayName()); - } - - services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, - BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, - Collections.singletonList(artifact))); - } -} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModuleFactory.java deleted file mode 100755 index 02cc90f2e9..0000000000 --- a/Core/src/org/sleuthkit/autopsy/modules/fileselector/FileSelectorIngestModuleFactory.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.modules.fileselector; - -import org.sleuthkit.autopsy.coreutils.Version; -import org.openide.util.NbBundle; -import org.openide.util.lookup.ServiceProvider; -import org.sleuthkit.autopsy.ingest.FileIngestModule; -import org.sleuthkit.autopsy.ingest.IngestModuleFactory; -import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; - -/** - * A factory that creates a DatabaseSelectorModule to parse database files and - * mark them as having interesting records (emails, phone numbers, mac addresses, gps coordinates). - */ -@NbBundle.Messages({ - "FileSelectorIngestModuleFactory.FlagFilesWithInterestingContents.moduleName=File Selector", - "FileSelectorIngestModuleFactory.FlagFilesWithInterestingContents.moduleDesc.text=" - + "Flag file with interesting contents (emails, phone numbers, gps coordinates, ip/mac addresses)" -}) -@ServiceProvider(service = IngestModuleFactory.class) -public class FileSelectorIngestModuleFactory extends IngestModuleFactoryAdapter { - - static String getModuleName() { - return Bundle.FileSelectorIngestModuleFactory_FlagFilesWithInterestingContents_moduleName(); - } - - @Override - public String getModuleDisplayName() { - return getModuleName(); - } - - @Override - public String getModuleDescription() { - return Bundle.FileSelectorIngestModuleFactory_FlagFilesWithInterestingContents_moduleDesc_text(); - } - - @Override - public String getModuleVersionNumber() { - return Version.getVersion(); - } - - @Override - public boolean isFileIngestModuleFactory() { - return true; - } - - @Override - public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { - return new FileSelectorIngestModule(); - } -} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java index 7c08e309cb..d9c1a2f2c2 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java @@ -40,8 +40,7 @@ import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; /** - * - * @author dsmyda + * Reads Excel files and returns results in a list collection. */ @NbBundle.Messages({ "ExcelReader.ReadExcelFiles.moduleName=ExcelReader" @@ -50,8 +49,14 @@ public class ExcelReader extends AbstractReader { private Workbook xlsWorkbook; private final IngestServices services = IngestServices.getInstance(); - private final Logger logger = services.getLogger(Bundle.ExcelReader_ReadExcelFiles_moduleName()); + private final Logger logger = services.getLogger(ExcelReader.class.getName()); + /** + * + * @param file + * @param localDiskPath + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException + */ public ExcelReader(AbstractFile file, String localDiskPath) throws FileReaderInitException { super(file, localDiskPath); @@ -63,23 +68,46 @@ public class ExcelReader extends AbstractReader { } } + /** + * Opens an apache poi workbook instance on the file contents copied from the + * abstract file. + * + * @param localDiskPath Location of the file contents on local disk + * @throws IOException Error opening file at local disk path + */ private void getWorkbookFromLocalDisk(String localDiskPath) throws IOException{ xlsWorkbook = new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); } + /** + * Returns the number of rows in a given excel table (aka sheet). + * + * @param tableName Name of table to count total rows from + * @return row count for requested table name + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + */ @Override public Integer getRowCountFromTable(String tableName) throws FileReaderException { return xlsWorkbook.getSheet(tableName).getLastRowNum(); } + /** + * Returns a collection of all the rows from a given table in an excel document. + * + * @param tableName + * @return + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + */ @Override public List> getRowsFromTable(String tableName) throws FileReaderException { List> rowContents = new ArrayList<>(); Iterator iterator = xlsWorkbook.getSheet(tableName).rowIterator(); //Consume header if(iterator.hasNext()) { + //Consume header iterator.next(); } + while(iterator.hasNext()) { Map contents = new HashMap<>(); Row r = iterator.next(); @@ -91,44 +119,63 @@ public class ExcelReader extends AbstractReader { return rowContents; } + /** + * + * @param cell + * @param contents + */ private void addCellValueToMap(Cell cell, Map contents){ + String columnName = getCellValuesColumnName(cell); switch (cell.getCellTypeEnum()) { case BOOLEAN: - contents.put(getCellValuesColumnName(cell), cell.getBooleanCellValue()); + contents.put(columnName, cell.getBooleanCellValue()); break; case STRING: - contents.put(getCellValuesColumnName(cell), cell.getRichStringCellValue().getString()); + contents.put(columnName, cell.getRichStringCellValue().getString()); break; case NUMERIC: if (DateUtil.isCellDateFormatted(cell)) { - contents.put(getCellValuesColumnName(cell), cell.getDateCellValue()); + contents.put(columnName, cell.getDateCellValue()); } else { - contents.put(getCellValuesColumnName(cell), cell.getNumericCellValue()); + contents.put(columnName, cell.getNumericCellValue()); } break; case FORMULA: - contents.put(getCellValuesColumnName(cell), cell.getCellFormula()); + contents.put(columnName, cell.getCellFormula()); break; default: - contents.put(getCellValuesColumnName(cell),""); + contents.put(columnName,""); } } + /** + * + * @param cell + * @return + */ private String getCellValuesColumnName(Cell cell) { Row header = cell.getSheet().getRow(0); - // if(header != null) { - // header.getCell(cell.getRowIndex()).getRichStringCellValue().toString(); - //} + if(header != null) { + Cell columnCell = header.getCell(cell.getRowIndex()); + + } return ""; } + /** + * + * + * @return + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + */ @Override public Map getTableSchemas() throws FileReaderException { Map tableSchemas = new HashMap<>(); for(Sheet sheet : xlsWorkbook) { Row header = sheet.getRow(0); if(header != null) { - tableSchemas.put(sheet.getSheetName(), poiRowToString(sheet.getRow(0))); + String headerStringFormat = StringUtils.join(header.cellIterator(), ", "); + tableSchemas.put(sheet.getSheetName(), headerStringFormat); } else { tableSchemas.put(sheet.getSheetName(), ""); } @@ -136,11 +183,10 @@ public class ExcelReader extends AbstractReader { return tableSchemas; } - private String poiRowToString(Row row) { - return StringUtils.join(row.cellIterator(), ", "); - } - - @Override + /** + * + */ + @Override public void close() { try { xlsWorkbook.close(); @@ -150,4 +196,4 @@ public class ExcelReader extends AbstractReader { logger.log(Level.WARNING, "Could not close excel file input stream", ex); } } -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java index a6cee9f917..a650bc3b0f 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.sqlitereader; @@ -9,11 +22,18 @@ import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException import org.sleuthkit.datamodel.AbstractFile; /** - * - * @author dsmyda + * Factory for creating the correct reader given the mime type of a file. */ public final class FileReaderFactory { + /** + * + * @param mimeType + * @param file + * @param localDiskPath + * @return + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException + */ public static AbstractReader createReader(String mimeType, AbstractFile file, String localDiskPath) throws FileReaderInitException { switch(mimeType) { diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java index f3d0aec277..dff4d21269 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java @@ -45,7 +45,7 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** - * Reads rows from SQLite tables and returns results in a list collection. + * Reads sqlite databases and returns results in a list collection. */ @NbBundle.Messages({ "SQLiteReader.ReadSQLiteFiles.moduleName=SQLiteReader" @@ -62,11 +62,7 @@ public class SQLiteReader extends AbstractReader { * * @param sqliteDbFile Data source abstract file * @param localDiskPath Location for database contents to be copied to - * @throws ClassNotFoundException missing SQLite JDBC class - * @throws SQLException Exception opening JDBC connection - * @throws IOException Exception writing file contents - * @throws NoCurrentCaseException Current case closed during file copying - * @throws TskCoreException Exception finding files from abstract file + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException */ public SQLiteReader(AbstractFile sqliteDbFile, String localDiskPath) throws FileReaderInitException { super(sqliteDbFile, localDiskPath); @@ -138,7 +134,7 @@ public class SQLiteReader extends AbstractReader { * CREATE TABLE statments). * * @return A map of table names to table schemas - * @throws SQLException + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException */ @Override public Map getTableSchemas() throws FileReaderException { @@ -169,7 +165,7 @@ public class SQLiteReader extends AbstractReader { * * @param tableName * @return Row count from tableName - * @throws SQLException + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException */ @Override public Integer getRowCountFromTable(String tableName) @@ -191,7 +187,7 @@ public class SQLiteReader extends AbstractReader { * @param tableName * @return List of rows, where each row is * represented as a column-value map. - * @throws SQLException + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException */ @Override public List> getRowsFromTable(String tableName) @@ -217,7 +213,7 @@ public class SQLiteReader extends AbstractReader { * @param numRowsToRead Number of rows past the start index * @return List of rows, where each row is * represented as a column-value map. - * @throws SQLException + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException */ public List> getRowsFromTable(String tableName, int startRow, int numRowsToRead) throws FileReaderException{ From 9bb103359b25dd2f2cadb415e780afb43ea7edae Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 15 Aug 2018 13:07:53 -0400 Subject: [PATCH 16/33] Added xlsx support (XML model) --- .../autopsy/sqlitereader/ExcelReader.java | 62 ++++++++----------- .../sqlitereader/FileReaderFactory.java | 5 +- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java index d9c1a2f2c2..c67d620c9c 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java @@ -34,51 +34,46 @@ import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; -import org.openide.util.NbBundle; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.datamodel.AbstractFile; + /** - * Reads Excel files and returns results in a list collection. + * + * @author dsmyda */ -@NbBundle.Messages({ - "ExcelReader.ReadExcelFiles.moduleName=ExcelReader" -}) -public class ExcelReader extends AbstractReader { +public class ExcelReader extends AbstractReader { - private Workbook xlsWorkbook; + private Workbook workbook; private final IngestServices services = IngestServices.getInstance(); private final Logger logger = services.getLogger(ExcelReader.class.getName()); - - /** - * - * @param file - * @param localDiskPath - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException - */ - public ExcelReader(AbstractFile file, String localDiskPath) + private String XLSXMimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private String XLSMimeType = "application/vnd.ms-excel"; + + public ExcelReader(AbstractFile file, String localDiskPath, String mimeType) throws FileReaderInitException { super(file, localDiskPath); - try { - getWorkbookFromLocalDisk(localDiskPath); + this.workbook = createWorkbook(localDiskPath, mimeType); } catch (IOException ex) { throw new FileReaderInitException(ex); } } - /** - * Opens an apache poi workbook instance on the file contents copied from the - * abstract file. - * - * @param localDiskPath Location of the file contents on local disk - * @throws IOException Error opening file at local disk path - */ - private void getWorkbookFromLocalDisk(String localDiskPath) throws IOException{ - xlsWorkbook = new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); + private Workbook createWorkbook(String localDiskPath, String mimeType) throws + IOException, FileReaderInitException { + if(mimeType.equals(XLSMimeType)) { + return new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); + } else if(mimeType.equals(XLSXMimeType)) { + return new XSSFWorkbook(new FileInputStream(new File(localDiskPath))); + } else { + throw new FileReaderInitException(String.format("Excel reader for mime " + + "type [%s] is not supported", mimeType)); + } } - + /** * Returns the number of rows in a given excel table (aka sheet). * @@ -88,7 +83,7 @@ public class ExcelReader extends AbstractReader { */ @Override public Integer getRowCountFromTable(String tableName) throws FileReaderException { - return xlsWorkbook.getSheet(tableName).getLastRowNum(); + return workbook.getSheet(tableName).getLastRowNum(); } /** @@ -101,7 +96,7 @@ public class ExcelReader extends AbstractReader { @Override public List> getRowsFromTable(String tableName) throws FileReaderException { List> rowContents = new ArrayList<>(); - Iterator iterator = xlsWorkbook.getSheet(tableName).rowIterator(); + Iterator iterator = workbook.getSheet(tableName).rowIterator(); //Consume header if(iterator.hasNext()) { //Consume header @@ -171,7 +166,7 @@ public class ExcelReader extends AbstractReader { @Override public Map getTableSchemas() throws FileReaderException { Map tableSchemas = new HashMap<>(); - for(Sheet sheet : xlsWorkbook) { + for(Sheet sheet : workbook) { Row header = sheet.getRow(0); if(header != null) { String headerStringFormat = StringUtils.join(header.cellIterator(), ", "); @@ -183,17 +178,14 @@ public class ExcelReader extends AbstractReader { return tableSchemas; } - /** - * - */ @Override public void close() { try { - xlsWorkbook.close(); + workbook.close(); } catch (IOException ex) { //Non-essential exception, user has no need for the connection //object at this stage so closing details are not important logger.log(Level.WARNING, "Could not close excel file input stream", ex); } } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java index a650bc3b0f..af9d657277 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java @@ -36,11 +36,12 @@ public final class FileReaderFactory { */ public static AbstractReader createReader(String mimeType, AbstractFile file, String localDiskPath) throws FileReaderInitException { - switch(mimeType) { + switch (mimeType) { case "application/x-sqlite3": return new SQLiteReader(file, localDiskPath); case "application/vnd.ms-excel": - return new ExcelReader(file, localDiskPath); + case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + return new ExcelReader(file, localDiskPath, mimeType); default: throw new FileReaderInitException(String.format("Reader for mime " + "type [%s] is not supported", mimeType)); From 91aa55838b218b661f1ee9fd1e225ab88180e335 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 15 Aug 2018 14:31:49 -0400 Subject: [PATCH 17/33] Added streaming verison of apache poi for xlsx files --- .../org/sleuthkit/autopsy/sqlitereader/ExcelReader.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java index c67d620c9c..7ddcc7aca8 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java @@ -34,6 +34,7 @@ import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -67,7 +68,7 @@ public class ExcelReader extends AbstractReader { if(mimeType.equals(XLSMimeType)) { return new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); } else if(mimeType.equals(XLSXMimeType)) { - return new XSSFWorkbook(new FileInputStream(new File(localDiskPath))); + return new SXSSFWorkbook(new XSSFWorkbook(new FileInputStream(new File(localDiskPath)))); } else { throw new FileReaderInitException(String.format("Excel reader for mime " + "type [%s] is not supported", mimeType)); @@ -97,11 +98,6 @@ public class ExcelReader extends AbstractReader { public List> getRowsFromTable(String tableName) throws FileReaderException { List> rowContents = new ArrayList<>(); Iterator iterator = workbook.getSheet(tableName).rowIterator(); - //Consume header - if(iterator.hasNext()) { - //Consume header - iterator.next(); - } while(iterator.hasNext()) { Map contents = new HashMap<>(); From 11b482f4eed39374cae973d37a7e07a34cedce06 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 14:40:42 -0400 Subject: [PATCH 18/33] Fixed the out of memory issue with excel xml document reading, caught the old excel format exception, and refactored the code --- Core/ivy.xml | 1 + Core/nbproject/project.properties | 3 +- Core/nbproject/project.xml | 7 +- .../autopsy/contentviewers/SQLiteViewer.java | 8 +- .../autopsy/sqlitereader/ExcelReader.java | 187 -------------- .../AbstractReader.java | 7 +- .../tabulardatareader/ExcelReader.java | 230 ++++++++++++++++++ .../FileReaderFactory.java | 11 +- .../SQLiteReader.java | 2 +- 9 files changed, 254 insertions(+), 202 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java rename Core/src/org/sleuthkit/autopsy/{sqlitereader => tabulardatareader}/AbstractReader.java (97%) create mode 100755 Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java rename Core/src/org/sleuthkit/autopsy/{sqlitereader => tabulardatareader}/FileReaderFactory.java (80%) rename Core/src/org/sleuthkit/autopsy/{sqlitereader => tabulardatareader}/SQLiteReader.java (99%) diff --git a/Core/ivy.xml b/Core/ivy.xml index 5e0bcbbc4b..5124aa2fd8 100644 --- a/Core/ivy.xml +++ b/Core/ivy.xml @@ -30,6 +30,7 @@ + diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 46b4d6e90c..2d65b137b8 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -16,7 +16,7 @@ file.reference.postgresql-9.4.1211.jre7.jar=release/modules/ext/postgresql-9.4.1 file.reference.Rejistry-1.0-SNAPSHOT.jar=release/modules/ext/Rejistry-1.0-SNAPSHOT.jar file.reference.sevenzipjbinding-AllPlatforms.jar=release/modules/ext/sevenzipjbinding-AllPlatforms.jar file.reference.sevenzipjbinding.jar=release/modules/ext/sevenzipjbinding.jar -file.reference.sqlite-jdbc-3.8.11.jar=release/modules/ext/sqlite-jdbc-3.8.11.jar +file.reference.sqlite-jdbc-3.8.11.jar=release\\modules\\ext\\sqlite-jdbc-3.8.11.jar file.reference.StixLib.jar=release/modules/ext/StixLib.jar file.reference.bcprov-jdk15on-1.54.jar=release/modules/ext/bcprov-jdk15on-1.54.jar file.reference.jackcess-2.1.8.jar=release/modules/ext/jackcess-2.1.8.jar @@ -36,6 +36,7 @@ file.reference.tika-parsers-1.17.jar=release/modules/ext/tika-parsers-1.17.jar file.reference.curator-client-2.8.0.jar=release/modules/ext/curator-client-2.8.0.jar file.reference.curator-framework-2.8.0.jar=release/modules/ext/curator-framework-2.8.0.jar file.reference.curator-recipes-2.8.0.jar=release/modules/ext/curator-recipes-2.8.0.jar +file.reference.xlsx-streamer-1.2.1.jar=release\\modules\\ext\\xlsx-streamer-1.2.1.jar file.reference.xmpcore-5.1.3.jar=release/modules/ext/xmpcore-5.1.3.jar file.reference.xz-1.6.jar=release/modules/ext/xz-1.6.jar file.reference.zookeeper-3.4.6.jar=release/modules/ext/zookeeper-3.4.6.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index ca87c4f2f4..dbd89f2761 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -337,6 +337,7 @@ org.sleuthkit.autopsy.modules.vmextractor org.sleuthkit.autopsy.progress org.sleuthkit.autopsy.report + org.sleuthkit.autopsy.tabulardatareader org.sleuthkit.datamodel @@ -445,7 +446,7 @@ ext/sqlite-jdbc-3.8.11.jar - release/modules/ext/sqlite-jdbc-3.8.11.jar + release\modules\ext\sqlite-jdbc-3.8.11.jar ext/activemq-all-5.11.1.jar @@ -495,6 +496,10 @@ ext/SparseBitSet-1.1.jar release/modules/ext/SparseBitSet-1.1.jar + + ext/xlsx-streamer-1.2.1.jar + release\modules\ext\xlsx-streamer-1.2.1.jar + ext/pdfbox-2.0.8.jar release/modules/ext/pdfbox-2.0.8.jar diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 076748d2af..23aa450c4c 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -47,10 +47,10 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException; -import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException; -import org.sleuthkit.autopsy.sqlitereader.FileReaderFactory; -import org.sleuthkit.autopsy.sqlitereader.SQLiteReader; +import org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException; +import org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException; +import org.sleuthkit.autopsy.tabulardatareader.FileReaderFactory; +import org.sleuthkit.autopsy.tabulardatareader.SQLiteReader; /** * A file content viewer for SQLite database files. diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java deleted file mode 100755 index 7ddcc7aca8..0000000000 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/ExcelReader.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.sqlitereader; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import org.apache.commons.lang3.StringUtils; -import org.apache.poi.hssf.usermodel.HSSFWorkbook; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.DateUtil; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.datamodel.AbstractFile; - - -/** - * - * @author dsmyda - */ -public class ExcelReader extends AbstractReader { - - private Workbook workbook; - private final IngestServices services = IngestServices.getInstance(); - private final Logger logger = services.getLogger(ExcelReader.class.getName()); - private String XLSXMimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; - private String XLSMimeType = "application/vnd.ms-excel"; - - public ExcelReader(AbstractFile file, String localDiskPath, String mimeType) - throws FileReaderInitException { - super(file, localDiskPath); - try { - this.workbook = createWorkbook(localDiskPath, mimeType); - } catch (IOException ex) { - throw new FileReaderInitException(ex); - } - } - - private Workbook createWorkbook(String localDiskPath, String mimeType) throws - IOException, FileReaderInitException { - if(mimeType.equals(XLSMimeType)) { - return new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); - } else if(mimeType.equals(XLSXMimeType)) { - return new SXSSFWorkbook(new XSSFWorkbook(new FileInputStream(new File(localDiskPath)))); - } else { - throw new FileReaderInitException(String.format("Excel reader for mime " - + "type [%s] is not supported", mimeType)); - } - } - - /** - * Returns the number of rows in a given excel table (aka sheet). - * - * @param tableName Name of table to count total rows from - * @return row count for requested table name - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException - */ - @Override - public Integer getRowCountFromTable(String tableName) throws FileReaderException { - return workbook.getSheet(tableName).getLastRowNum(); - } - - /** - * Returns a collection of all the rows from a given table in an excel document. - * - * @param tableName - * @return - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException - */ - @Override - public List> getRowsFromTable(String tableName) throws FileReaderException { - List> rowContents = new ArrayList<>(); - Iterator iterator = workbook.getSheet(tableName).rowIterator(); - - while(iterator.hasNext()) { - Map contents = new HashMap<>(); - Row r = iterator.next(); - for(Cell c : r) { - addCellValueToMap(c, contents); - } - rowContents.add(contents); - } - return rowContents; - } - - /** - * - * @param cell - * @param contents - */ - private void addCellValueToMap(Cell cell, Map contents){ - String columnName = getCellValuesColumnName(cell); - switch (cell.getCellTypeEnum()) { - case BOOLEAN: - contents.put(columnName, cell.getBooleanCellValue()); - break; - case STRING: - contents.put(columnName, cell.getRichStringCellValue().getString()); - break; - case NUMERIC: - if (DateUtil.isCellDateFormatted(cell)) { - contents.put(columnName, cell.getDateCellValue()); - } else { - contents.put(columnName, cell.getNumericCellValue()); - } - break; - case FORMULA: - contents.put(columnName, cell.getCellFormula()); - break; - default: - contents.put(columnName,""); - } - } - - /** - * - * @param cell - * @return - */ - private String getCellValuesColumnName(Cell cell) { - Row header = cell.getSheet().getRow(0); - if(header != null) { - Cell columnCell = header.getCell(cell.getRowIndex()); - - } - return ""; - } - - /** - * - * - * @return - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException - */ - @Override - public Map getTableSchemas() throws FileReaderException { - Map tableSchemas = new HashMap<>(); - for(Sheet sheet : workbook) { - Row header = sheet.getRow(0); - if(header != null) { - String headerStringFormat = StringUtils.join(header.cellIterator(), ", "); - tableSchemas.put(sheet.getSheetName(), headerStringFormat); - } else { - tableSchemas.put(sheet.getSheetName(), ""); - } - } - return tableSchemas; - } - - @Override - public void close() { - try { - workbook.close(); - } catch (IOException ex) { - //Non-essential exception, user has no need for the connection - //object at this stage so closing details are not important - logger.log(Level.WARNING, "Could not close excel file input stream", ex); - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/AbstractReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java similarity index 97% rename from Core/src/org/sleuthkit/autopsy/sqlitereader/AbstractReader.java rename to Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java index 46f4242454..981552ce27 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/AbstractReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.sqlitereader; +package org.sleuthkit.autopsy.tabulardatareader; import java.io.File; import java.io.IOException; @@ -27,8 +27,7 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * - * @author dsmyda + * An abstract reader interface to */ public abstract class AbstractReader implements AutoCloseable { @@ -96,4 +95,4 @@ public abstract class AbstractReader implements AutoCloseable { super(message); } } -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java new file mode 100755 index 0000000000..3b5afe452c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -0,0 +1,230 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.tabulardatareader; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DateUtil; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.datamodel.AbstractFile; +import com.monitorjbl.xlsx.StreamingReader; +import org.apache.poi.hssf.OldExcelFormatException; + + +/** + * + */ +public class ExcelReader extends AbstractReader { + /* Boilerplate code */ + private final IngestServices services = IngestServices.getInstance(); + private final Logger logger = services.getLogger(ExcelReader.class.getName()); + + private Workbook workbook; + private final String XLSXMimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private final String XLSMimeType = "application/vnd.ms-excel"; + private final String EMPTY_CELL_STRING = ""; + + public ExcelReader(AbstractFile file, String localDiskPath, String mimeType) + throws FileReaderInitException { + super(file, localDiskPath); + try { + this.workbook = createWorkbook(localDiskPath, mimeType); + } catch (IOException ex) { + throw new FileReaderInitException(ex); + } + } + + /** + * Internal factory for creating the correct workbook given the mime type. The + * file reader factory in this module passes both the XLSMimeType and XLSXMimeType + * into this constructor for the reader to handle. This avoided the need for creating + * an AbstractExcelReader class and two sub classes overriding the workbook field. + * Additionally, I don't forsee needing to support more than these two mime types. + * + * @param localDiskPath To open an input stream for poi to read from + * @param mimeType The mimeType passed to the constructor + * @return The corrent workbook instance + * @throws IOException Issue with input stream and opening file location at + * localDiskPath + * @throws FileReaderInitException mimetype unsupported + */ + private Workbook createWorkbook(String localDiskPath, String mimeType) throws + IOException, FileReaderInitException { + if(mimeType.equals(XLSMimeType)) { + try { + //Apache POI only supports BIFF8 format, anything below is considered + //old excel format and is not a concern for us. + return new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); + } catch (OldExcelFormatException e) { + throw new FileReaderInitException(e); + } + } else if(mimeType.equals(XLSXMimeType)) { + InputStream is = new FileInputStream(new File(localDiskPath)); + //StreamingReader is part of the xlsx streamer dependency that creates + //a streaming version of XSSFWorkbook for reading (SXSSFWorkbook is only for writing + //large workbooks, not reading). This libary provides a workbook interface + //that is mostly identical to the poi workbook api, hence both the HSSFWorkbook + //and this can use the same functions below. + return StreamingReader.builder().rowCacheSize(500).bufferSize(4096).open(is); + } else { + throw new FileReaderInitException(String.format("Excel reader for mime " + + "type [%s] is not supported", mimeType)); + } + } + + /** + * Returns the number of rows in a given excel table (aka sheet). + * + * @param tableName Name of table to count total rows from + * @return row count for requested table name + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + */ + @Override + public Integer getRowCountFromTable(String tableName) throws FileReaderException { + return workbook.getSheet(tableName).getLastRowNum(); + } + + /** + * Returns a collection of all the rows from a given table in an excel document. + * + * @param tableName Current sheet name being read + * @return A collection of row maps + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + */ + @Override + public List> getRowsFromTable(String tableName) throws FileReaderException { + List> rowList = new ArrayList<>(); + Iterator iterator = workbook.getSheet(tableName).rowIterator(); + + while(iterator.hasNext()) { + Map row = new HashMap<>(); + Row currRow = iterator.next(); + for(Cell cell : currRow) { + String columnName = getColumnName(cell, tableName); + Object value = getCellValue(cell); + row.put(columnName, value); + } + rowList.add(row); + } + + return rowList; + } + + /** + * Returns the value of a given cell. The correct value function must be + * called on a cell depending on its type, hence the switch. + * + * @param cell Cell object containing a getter function for its value type + * @return A generic object pointer to the cell's value + */ + private Object getCellValue(Cell cell){ + switch (cell.getCellTypeEnum()) { + case BOOLEAN: + return cell.getBooleanCellValue(); + case STRING: + return cell.getRichStringCellValue().getString(); + case NUMERIC: + if (DateUtil.isCellDateFormatted(cell)) { + return cell.getDateCellValue(); + } else { + return cell.getNumericCellValue(); + } + case FORMULA: + return cell.getCellFormula(); + default: + //Cell must be empty at this branch + return EMPTY_CELL_STRING; + } + } + + /** + * Returns the name of the column that the cell currently lives in + * Cell Value: 6784022342 -> Header name: Phone Number + * + * @param cell current cell being read + * @param tableName current sheet name being read + * @return the name of the column the current cell lives in + */ + private String getColumnName(Cell cell, String tableName) { + Iterator sheetIter = workbook.getSheet(tableName).rowIterator(); + if(sheetIter.hasNext()) { + Row header = sheetIter.next(); + Cell columnHeaderCell = header.getCell(cell.getRowIndex()); + if(columnHeaderCell == null) { + return EMPTY_CELL_STRING; + } + Object columnHeaderValue = getCellValue(columnHeaderCell); + if(columnHeaderValue instanceof String) { + return (String) columnHeaderValue; + } else { + return columnHeaderValue.toString(); + } + } + //No header present + return EMPTY_CELL_STRING; + } + + /** + * Returns a map of sheet names to headers (header is in a comma-seperated string). + * + * @return A map of sheet names to header strings. + * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + */ + @Override + public Map getTableSchemas() throws FileReaderException { + Map tableSchemas = new HashMap<>(); + for(Sheet sheet : workbook) { + Iterator iterator = sheet.rowIterator(); + if(iterator.hasNext()) { + //Consume header + Row header = iterator.next(); + String headerStringFormat = StringUtils.join(header.cellIterator(), ", "); + tableSchemas.put(sheet.getSheetName(), headerStringFormat); + } + } + + return tableSchemas; + } + + @Override + public void close() { + try { + workbook.close(); + } catch (IOException ex) { + //Non-essential exception, user has no need for the connection + //object at this stage so closing details are not important + logger.log(Level.WARNING, "Could not close excel file input stream", ex); + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java similarity index 80% rename from Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java rename to Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java index af9d657277..8b2e678ec3 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java @@ -16,9 +16,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.sqlitereader; +package org.sleuthkit.autopsy.tabulardatareader; -import org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException; +import org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException; import org.sleuthkit.datamodel.AbstractFile; /** @@ -27,9 +27,12 @@ import org.sleuthkit.datamodel.AbstractFile; public final class FileReaderFactory { /** + * Instantiates the appropriate reader given the mimeType argument. Currently + * supports SQLite files and Excel files (.xls and .xlsx). BIFF5 format of .xls + * is not supported. * - * @param mimeType - * @param file + * @param mimeType mimeType passed in from the ingest module + * @param file * @param localDiskPath * @return * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException diff --git a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java similarity index 99% rename from Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java rename to Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index dff4d21269..1a2ea4dc06 100755 --- a/Core/src/org/sleuthkit/autopsy/sqlitereader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.sqlitereader; +package org.sleuthkit.autopsy.tabulardatareader; import java.io.File; import java.io.IOException; From abb32cb11f6ccdd57bc2c6980d4dd14b1d66c2c8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 15:12:26 -0400 Subject: [PATCH 19/33] Added some more comments and fixed old exception comments --- .../tabulardatareader/AbstractReader.java | 34 +++++++++++- .../tabulardatareader/ExcelReader.java | 52 ++++++++++--------- .../tabulardatareader/FileReaderFactory.java | 8 +-- .../tabulardatareader/SQLiteReader.java | 10 ++-- 4 files changed, 69 insertions(+), 35 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java index 981552ce27..fddf02c3d2 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java @@ -27,7 +27,8 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.TskCoreException; /** - * An abstract reader interface to + * An abstract reader interface for retrieving contents from files via a common + * API. */ public abstract class AbstractReader implements AutoCloseable { @@ -39,6 +40,7 @@ public abstract class AbstractReader implements AutoCloseable { /** * Copies the data source file contents to local drive for processing. + * This function is common to all readers. * * @param file AbstractFile from the data source * @param localDiskPath Local drive path to copy AbstractFile contents @@ -59,15 +61,41 @@ public abstract class AbstractReader implements AutoCloseable { } } + /** + * Return the a mapping of table names to table schemas (may be in the form of + * headers or create table statements for databases). + * + * @return + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + */ public abstract Map getTableSchemas() throws FileReaderException; + /** + * Returns the row count fo the given table name. + * + * @param tableName + * @return + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + */ public abstract Integer getRowCountFromTable(String tableName) throws FileReaderException; + /** + * Returns a collection view of the rows in a table. + * + * @param tableName + * @return + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + */ public abstract List> getRowsFromTable(String tableName) throws FileReaderException; @Override public abstract void close(); + /** + * Checked exceptions are specific to a given implementation, so this custom + * exception allows for a common interface to accommodate all of them. Init + * exception allows for more flexibility in logging. + */ public static class FileReaderInitException extends Exception { public FileReaderInitException(String message, Throwable cause) { super(message, cause); @@ -82,6 +110,10 @@ public abstract class AbstractReader implements AutoCloseable { } } + /** + * Checked exceptions are specific to a given implementation, so this custom + * exception allows for a common interface to accommodate all of them. + */ public class FileReaderException extends Exception { public FileReaderException(String message, Throwable cause) { super(message, cause); diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index 3b5afe452c..e5430bd2ab 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -43,7 +43,8 @@ import org.apache.poi.hssf.OldExcelFormatException; /** - * + * Reads excel files and implements the abstract reader api for interfacing with the + * content. Supports .xls and .xlsx files. */ public class ExcelReader extends AbstractReader { /* Boilerplate code */ @@ -81,24 +82,25 @@ public class ExcelReader extends AbstractReader { */ private Workbook createWorkbook(String localDiskPath, String mimeType) throws IOException, FileReaderInitException { - if(mimeType.equals(XLSMimeType)) { - try { - //Apache POI only supports BIFF8 format, anything below is considered - //old excel format and is not a concern for us. - return new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); - } catch (OldExcelFormatException e) { - throw new FileReaderInitException(e); - } - } else if(mimeType.equals(XLSXMimeType)) { - InputStream is = new FileInputStream(new File(localDiskPath)); - //StreamingReader is part of the xlsx streamer dependency that creates - //a streaming version of XSSFWorkbook for reading (SXSSFWorkbook is only for writing - //large workbooks, not reading). This libary provides a workbook interface - //that is mostly identical to the poi workbook api, hence both the HSSFWorkbook - //and this can use the same functions below. - return StreamingReader.builder().rowCacheSize(500).bufferSize(4096).open(is); - } else { - throw new FileReaderInitException(String.format("Excel reader for mime " + switch (mimeType) { + case XLSMimeType: + try { + //Apache POI only supports BIFF8 format, anything below is considered + //old excel format and is not a concern for us. + return new HSSFWorkbook(new FileInputStream(new File(localDiskPath))); + } catch (OldExcelFormatException e) { + throw new FileReaderInitException(e); + } + case XLSXMimeType: + InputStream is = new FileInputStream(new File(localDiskPath)); + //StreamingReader is part of the xlsx streamer dependency that creates + //a streaming version of XSSFWorkbook for reading (SXSSFWorkbook is only for writing + //large workbooks, not reading). This libary provides a workbook interface + //that is mostly identical to the poi workbook api, hence both the HSSFWorkbook + //and this can use the same functions below. + return StreamingReader.builder().rowCacheSize(500).bufferSize(4096).open(is); + default: + throw new FileReaderInitException(String.format("Excel reader for mime " + "type [%s] is not supported", mimeType)); } } @@ -107,8 +109,8 @@ public class ExcelReader extends AbstractReader { * Returns the number of rows in a given excel table (aka sheet). * * @param tableName Name of table to count total rows from - * @return row count for requested table name - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + * @return row count for requested table name + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override public Integer getRowCountFromTable(String tableName) throws FileReaderException { @@ -119,8 +121,8 @@ public class ExcelReader extends AbstractReader { * Returns a collection of all the rows from a given table in an excel document. * * @param tableName Current sheet name being read - * @return A collection of row maps - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + * @return A collection of row maps + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override public List> getRowsFromTable(String tableName) throws FileReaderException { @@ -198,8 +200,8 @@ public class ExcelReader extends AbstractReader { /** * Returns a map of sheet names to headers (header is in a comma-seperated string). * - * @return A map of sheet names to header strings. - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + * @return A map of sheet names to header strings. + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override public Map getTableSchemas() throws FileReaderException { diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java index 8b2e678ec3..8e98e0447c 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java @@ -32,10 +32,10 @@ public final class FileReaderFactory { * is not supported. * * @param mimeType mimeType passed in from the ingest module - * @param file - * @param localDiskPath - * @return - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException + * @param file current file under inspection + * @param localDiskPath path for abstract file contents to be written + * @return The correct reader class needed to read the file contents + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException */ public static AbstractReader createReader(String mimeType, AbstractFile file, String localDiskPath) throws FileReaderInitException { diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index 1a2ea4dc06..c7bad9aade 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -62,7 +62,7 @@ public class SQLiteReader extends AbstractReader { * * @param sqliteDbFile Data source abstract file * @param localDiskPath Location for database contents to be copied to - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderInitException + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException */ public SQLiteReader(AbstractFile sqliteDbFile, String localDiskPath) throws FileReaderInitException { super(sqliteDbFile, localDiskPath); @@ -134,7 +134,7 @@ public class SQLiteReader extends AbstractReader { * CREATE TABLE statments). * * @return A map of table names to table schemas - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override public Map getTableSchemas() throws FileReaderException { @@ -165,7 +165,7 @@ public class SQLiteReader extends AbstractReader { * * @param tableName * @return Row count from tableName - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override public Integer getRowCountFromTable(String tableName) @@ -187,7 +187,7 @@ public class SQLiteReader extends AbstractReader { * @param tableName * @return List of rows, where each row is * represented as a column-value map. - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ @Override public List> getRowsFromTable(String tableName) @@ -213,7 +213,7 @@ public class SQLiteReader extends AbstractReader { * @param numRowsToRead Number of rows past the start index * @return List of rows, where each row is * represented as a column-value map. - * @throws org.sleuthkit.autopsy.sqlitereader.AbstractReader.FileReaderException + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ public List> getRowsFromTable(String tableName, int startRow, int numRowsToRead) throws FileReaderException{ From f98cb902d77b4d01abebecbe956b334cf5812f8c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 15:21:24 -0400 Subject: [PATCH 20/33] Changed some project files back to the original state --- Core/nbproject/project.properties | 10 +++++----- Core/nbproject/project.xml | 4 ++-- .../sleuthkit/autopsy/contentviewers/SQLiteViewer.java | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 2d65b137b8..cdfddfc078 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -1,9 +1,9 @@ file.reference.activemq-all-5.11.1.jar=release/modules/ext/activemq-all-5.11.1.jar file.reference.c3p0-0.9.5.jar=release/modules/ext/c3p0-0.9.5.jar file.reference.commons-compress-1.14.jar=release/modules/ext/commons-compress-1.14.jar -file.reference.commons-dbcp2-2.1.1.jar=release\\modules\\ext\\commons-dbcp2-2.1.1.jar -file.reference.commons-pool2-2.4.2.jar=release\\modules\\ext\\commons-pool2-2.4.2.jar -file.reference.commons-validator-1.6.jar=release\\modules\\ext\\commons-validator-1.6.jar +file.reference.commons-dbcp2-2.1.1.jar=release/modules/ext/commons-dbcp2-2.1.1.jar +file.reference.commons-pool2-2.4.2.jar=release/modules/ext/commons-pool2-2.4.2.jar +file.reference.commons-validator-1.6.jar=release/modules/ext/commons-validator-1.6.jar file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar file.reference.jdom-2.0.5-contrib.jar=release/modules/ext/jdom-2.0.5-contrib.jar file.reference.jdom-2.0.5.jar=release/modules/ext/jdom-2.0.5.jar @@ -16,7 +16,7 @@ file.reference.postgresql-9.4.1211.jre7.jar=release/modules/ext/postgresql-9.4.1 file.reference.Rejistry-1.0-SNAPSHOT.jar=release/modules/ext/Rejistry-1.0-SNAPSHOT.jar file.reference.sevenzipjbinding-AllPlatforms.jar=release/modules/ext/sevenzipjbinding-AllPlatforms.jar file.reference.sevenzipjbinding.jar=release/modules/ext/sevenzipjbinding.jar -file.reference.sqlite-jdbc-3.8.11.jar=release\\modules\\ext\\sqlite-jdbc-3.8.11.jar +file.reference.sqlite-jdbc-3.8.11.jar=release/modules/ext/sqlite-jdbc-3.8.11.jar file.reference.StixLib.jar=release/modules/ext/StixLib.jar file.reference.bcprov-jdk15on-1.54.jar=release/modules/ext/bcprov-jdk15on-1.54.jar file.reference.jackcess-2.1.8.jar=release/modules/ext/jackcess-2.1.8.jar @@ -36,7 +36,7 @@ file.reference.tika-parsers-1.17.jar=release/modules/ext/tika-parsers-1.17.jar file.reference.curator-client-2.8.0.jar=release/modules/ext/curator-client-2.8.0.jar file.reference.curator-framework-2.8.0.jar=release/modules/ext/curator-framework-2.8.0.jar file.reference.curator-recipes-2.8.0.jar=release/modules/ext/curator-recipes-2.8.0.jar -file.reference.xlsx-streamer-1.2.1.jar=release\\modules\\ext\\xlsx-streamer-1.2.1.jar +file.reference.xlsx-streamer-1.2.1.jar=release/modules/ext/xlsx-streamer-1.2.1.jar file.reference.xmpcore-5.1.3.jar=release/modules/ext/xmpcore-5.1.3.jar file.reference.xz-1.6.jar=release/modules/ext/xz-1.6.jar file.reference.zookeeper-3.4.6.jar=release/modules/ext/zookeeper-3.4.6.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index dbd89f2761..fcf968d5c5 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -446,7 +446,7 @@ ext/sqlite-jdbc-3.8.11.jar - release\modules\ext\sqlite-jdbc-3.8.11.jar + release/modules/ext/sqlite-jdbc-3.8.11.jar ext/activemq-all-5.11.1.jar @@ -498,7 +498,7 @@ ext/xlsx-streamer-1.2.1.jar - release\modules\ext\xlsx-streamer-1.2.1.jar + release/modules/ext/xlsx-streamer-1.2.1.jar ext/pdfbox-2.0.8.jar diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 23aa450c4c..0bee767b63 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -454,7 +454,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { * * @param file * @param tableName - * @param rowMap -- A list of rows in the table, where each row is represented as a column-value + * @param rowMap A list of rows in the table, where each row is represented as a column-value * map. * @throws FileNotFoundException * @throws IOException @@ -527,8 +527,8 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { * Returns a comma seperated header string from the keys of the column * row map. * - * @param row -- column header row map - * @return -- comma seperated header string + * @param row column header row map + * @return comma seperated header string */ private String createColumnHeader(Map row) { return row.entrySet() From 2e0ff95a8cf15467e190b9cd14709a1f8fba8a89 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 16:31:03 -0400 Subject: [PATCH 21/33] Added a new api call and refactored/commented some more code --- .../tabulardatareader/AbstractReader.java | 11 +++++++ .../tabulardatareader/ExcelReader.java | 31 ++++++++++++++++--- .../tabulardatareader/SQLiteReader.java | 7 +++-- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java index fddf02c3d2..c9c0bce055 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java @@ -88,6 +88,17 @@ public abstract class AbstractReader implements AutoCloseable { */ public abstract List> getRowsFromTable(String tableName) throws FileReaderException; + /** + * + * @param tableName + * @param offset + * @param numRowsToRead + * @return + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + */ + public abstract List> getRowsFromTable(String tableName, + int offset, int numRowsToRead) throws FileReaderException; + @Override public abstract void close(); diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index e5430bd2ab..a830e9e2b5 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -126,12 +126,35 @@ public class ExcelReader extends AbstractReader { */ @Override public List> getRowsFromTable(String tableName) throws FileReaderException { + return getRowsFromTable(tableName, 0, getRowCountFromTable(tableName)); + } + + /** + * Returns a window of rows starting at the offset and ending when the number of rows read + * equals the 'numRowsToRead' parameter or the iterator has nothing left to read. + * + * For instance: offset 1, numRowsToRead 5 would return 5 results (1-5). + * offset 0, numRowsToRead 5 would return 5 results (0-4). + * + * @param tableName Current name of sheet to be read + * @param offset start index to begin reading (documents are 0 indexed) + * @param numRowsToRead number of rows to read + * @return + * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException + */ + @Override + public List> getRowsFromTable(String tableName, + int offset, int numRowsToRead) throws FileReaderException { + Iterator sheetIter = workbook.getSheet(tableName).rowIterator(); List> rowList = new ArrayList<>(); - Iterator iterator = workbook.getSheet(tableName).rowIterator(); - while(iterator.hasNext()) { + int currRowCount = 0; + while(sheetIter.hasNext() && currRowCount < (offset + numRowsToRead)) { + Row currRow = sheetIter.next(); + if(currRowCount++ < offset) { + continue; + } Map row = new HashMap<>(); - Row currRow = iterator.next(); for(Cell cell : currRow) { String columnName = getColumnName(cell, tableName); Object value = getCellValue(cell); @@ -218,7 +241,7 @@ public class ExcelReader extends AbstractReader { return tableSchemas; } - + @Override public void close() { try { diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index c7bad9aade..b939ced24e 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -209,20 +209,21 @@ public class SQLiteReader extends AbstractReader { * Retrieves a subset of the rows from a given table in the SQLite database. * * @param tableName - * @param startRow Desired start index (rows begin at 1) + * @param offset Desired start index (rows begin at 1) * @param numRowsToRead Number of rows past the start index * @return List of rows, where each row is * represented as a column-value map. * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ + @Override public List> getRowsFromTable(String tableName, - int startRow, int numRowsToRead) throws FileReaderException{ + int offset, int numRowsToRead) throws FileReaderException{ tableName = wrapTableNameStringWithQuotes(tableName); try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( "SELECT * FROM " + tableName //NON-NLS + " LIMIT " + Integer.toString(numRowsToRead) //NON-NLS - + " OFFSET " + Integer.toString(startRow - 1))) { //NON-NLS + + " OFFSET " + Integer.toString(offset - 1))) { //NON-NLS return resultSetToList(resultSet); } catch (SQLException ex) { throw new FileReaderException(ex); From 0539b53da8d151c49921e909b5397f94ecded2d8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 16:33:31 -0400 Subject: [PATCH 22/33] Changed content viewer to use abstract reader interface --- .../sleuthkit/autopsy/contentviewers/SQLiteViewer.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 0bee767b63..47a33dc578 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -25,7 +25,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -38,19 +37,17 @@ import javax.swing.JFileChooser; import javax.swing.JOptionPane; import javax.swing.filechooser.FileNameExtensionFilter; import org.apache.commons.io.FilenameUtils; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import org.sleuthkit.autopsy.tabulardatareader.AbstractReader; import org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException; import org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException; import org.sleuthkit.autopsy.tabulardatareader.FileReaderFactory; -import org.sleuthkit.autopsy.tabulardatareader.SQLiteReader; /** * A file content viewer for SQLite database files. @@ -65,7 +62,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { private final SQLiteTableView selectedTableView = new SQLiteTableView(); private AbstractFile sqliteDbFile; private File tmpDbFile; - private SQLiteReader sqliteReader; + private AbstractReader sqliteReader; private int numRows; // num of rows in the selected table private int currPage = 0; // curr page of rows being displayed @@ -367,7 +364,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { String localDiskPath = Case.getCurrentCaseThrows().getTempDirectory() + File.separator + sqliteDbFile.getName(); - sqliteReader = (SQLiteReader) FileReaderFactory.createReader(SUPPORTED_MIMETYPES[0], sqliteDbFile, localDiskPath); + sqliteReader = FileReaderFactory.createReader(SUPPORTED_MIMETYPES[0], sqliteDbFile, localDiskPath); Map dbTablesMap = sqliteReader.getTableSchemas(); From 4e401c52f81c71f0333140681477d02e0b20c6f9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 17:17:56 -0400 Subject: [PATCH 23/33] Final comment additions --- .../tabulardatareader/AbstractReader.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java index c9c0bce055..2e0c24d1db 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java @@ -65,7 +65,7 @@ public abstract class AbstractReader implements AutoCloseable { * Return the a mapping of table names to table schemas (may be in the form of * headers or create table statements for databases). * - * @return + * @return Mapping of table names to schemas * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ public abstract Map getTableSchemas() throws FileReaderException; @@ -74,7 +74,7 @@ public abstract class AbstractReader implements AutoCloseable { * Returns the row count fo the given table name. * * @param tableName - * @return + * @return number of rows in the current table * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ public abstract Integer getRowCountFromTable(String tableName) throws FileReaderException; @@ -83,17 +83,19 @@ public abstract class AbstractReader implements AutoCloseable { * Returns a collection view of the rows in a table. * * @param tableName - * @return + * @return List view of the rows in the table * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ public abstract List> getRowsFromTable(String tableName) throws FileReaderException; /** + * Returns a window of rows starting at the offset and ending when the number of rows read + * equals the 'numRowsToRead' parameter or there is nothing left to read. * - * @param tableName - * @param offset - * @param numRowsToRead - * @return + * @param tableName table name to be read from + * @param offset start index to begin reading + * @param numRowsToRead number of rows to read past offset + * @return List view of the rows in the table * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException */ public abstract List> getRowsFromTable(String tableName, From 17d49ef11f1e132f3c5ec7e901248dfd6d3e7e08 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 17:25:55 -0400 Subject: [PATCH 24/33] Removed commons validator dependency from autopsy and moved it --- Core/nbproject/project.properties | 5 ++--- Core/nbproject/project.xml | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index cdfddfc078..08bc1496f9 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -1,9 +1,8 @@ file.reference.activemq-all-5.11.1.jar=release/modules/ext/activemq-all-5.11.1.jar file.reference.c3p0-0.9.5.jar=release/modules/ext/c3p0-0.9.5.jar file.reference.commons-compress-1.14.jar=release/modules/ext/commons-compress-1.14.jar -file.reference.commons-dbcp2-2.1.1.jar=release/modules/ext/commons-dbcp2-2.1.1.jar -file.reference.commons-pool2-2.4.2.jar=release/modules/ext/commons-pool2-2.4.2.jar -file.reference.commons-validator-1.6.jar=release/modules/ext/commons-validator-1.6.jar +file.reference.commons-dbcp2-2.1.1.jar=release\\modules\\ext\\commons-dbcp2-2.1.1.jar +file.reference.commons-pool2-2.4.2.jar=release\\modules\\ext\\commons-pool2-2.4.2.jar file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar file.reference.jdom-2.0.5-contrib.jar=release/modules/ext/jdom-2.0.5-contrib.jar file.reference.jdom-2.0.5.jar=release/modules/ext/jdom-2.0.5.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index fcf968d5c5..f07138c40f 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -356,10 +356,6 @@ ext/cxf-rt-transports-http-3.0.16.jar release/modules/ext/cxf-rt-transports-http-3.0.16.jar - - ext/commons-validator-1.6.jar - release\modules\ext\commons-validator-1.6.jar - ext/curator-framework-2.8.0.jar release/modules/ext/curator-framework-2.8.0.jar From ee9f961bdd73655b79bc77c93103bcedac135fb0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 17:26:58 -0400 Subject: [PATCH 25/33] Ivy change --- Core/ivy.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/ivy.xml b/Core/ivy.xml index 5124aa2fd8..1ed71d69cf 100644 --- a/Core/ivy.xml +++ b/Core/ivy.xml @@ -29,7 +29,6 @@ - From 934fd0198cdc83250940326d792f7dfdaa413362 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 17 Aug 2018 13:39:52 -0400 Subject: [PATCH 26/33] Fixed the iterator error introduced from the xlsx streamer approach. --- .../tabulardatareader/ExcelReader.java | 62 ++++++++++++------- .../tabulardatareader/SQLiteReader.java | 2 +- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index a830e9e2b5..607fc6de98 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.tabulardatareader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -55,12 +54,14 @@ public class ExcelReader extends AbstractReader { private final String XLSXMimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; private final String XLSMimeType = "application/vnd.ms-excel"; private final String EMPTY_CELL_STRING = ""; + private HashMap headerCache; public ExcelReader(AbstractFile file, String localDiskPath, String mimeType) throws FileReaderInitException { super(file, localDiskPath); try { this.workbook = createWorkbook(localDiskPath, mimeType); + headerCache = new HashMap<>(); } catch (IOException ex) { throw new FileReaderInitException(ex); } @@ -92,13 +93,12 @@ public class ExcelReader extends AbstractReader { throw new FileReaderInitException(e); } case XLSXMimeType: - InputStream is = new FileInputStream(new File(localDiskPath)); //StreamingReader is part of the xlsx streamer dependency that creates //a streaming version of XSSFWorkbook for reading (SXSSFWorkbook is only for writing //large workbooks, not reading). This libary provides a workbook interface //that is mostly identical to the poi workbook api, hence both the HSSFWorkbook //and this can use the same functions below. - return StreamingReader.builder().rowCacheSize(500).bufferSize(4096).open(is); + return StreamingReader.builder().rowCacheSize(500).open(new File(localDiskPath)); default: throw new FileReaderInitException(String.format("Excel reader for mime " + "type [%s] is not supported", mimeType)); @@ -126,7 +126,12 @@ public class ExcelReader extends AbstractReader { */ @Override public List> getRowsFromTable(String tableName) throws FileReaderException { - return getRowsFromTable(tableName, 0, getRowCountFromTable(tableName)); + //Pad with + 1 because rows are zero index, thus a LastRowNum() (in getRowCountFromTable()) of 1 + //indicates that there are records in 0 and 1 and so a total row count of + //2. This also implies there is no way to determine if a workbook is empty, + //since a last row num of 0 doesnt differentiate between a record in 0 or + //nothing in the workbook. Such a HSSF. + return getRowsFromTable(tableName, 0, getRowCountFromTable(tableName) + 1); } /** @@ -145,27 +150,45 @@ public class ExcelReader extends AbstractReader { @Override public List> getRowsFromTable(String tableName, int offset, int numRowsToRead) throws FileReaderException { + //StreamingReader maintains the same pointer to a sheet rowIterator, so this + //call returns an iterator that could have already been iterated on instead + //of a fresh copy. We must cache the header value from the call to + //getTableSchemas as important information in the first row could have been + //missed. Iterator sheetIter = workbook.getSheet(tableName).rowIterator(); List> rowList = new ArrayList<>(); int currRowCount = 0; + + //Read the header value as the header may be a row of data in the + //excel sheet + if(headerCache.containsKey(tableName)) { + Row header = headerCache.get(tableName); + if(currRowCount++ >= offset) { + rowList.add(getRowMap(tableName, header)); + } + } + while(sheetIter.hasNext() && currRowCount < (offset + numRowsToRead)) { Row currRow = sheetIter.next(); - if(currRowCount++ < offset) { - continue; + if(currRowCount++ >= offset) { + rowList.add(getRowMap(tableName, currRow)); } - Map row = new HashMap<>(); - for(Cell cell : currRow) { - String columnName = getColumnName(cell, tableName); - Object value = getCellValue(cell); - row.put(columnName, value); - } - rowList.add(row); } return rowList; } + private Map getRowMap(String tableName, Row row) { + Map rowMap = new HashMap<>(); + for(Cell cell : row) { + String columnName = getColumnName(cell, tableName); + Object value = getCellValue(cell); + rowMap.put(columnName, value); + } + return rowMap; + } + /** * Returns the value of a given cell. The correct value function must be * called on a cell depending on its type, hence the switch. @@ -202,19 +225,14 @@ public class ExcelReader extends AbstractReader { * @return the name of the column the current cell lives in */ private String getColumnName(Cell cell, String tableName) { - Iterator sheetIter = workbook.getSheet(tableName).rowIterator(); - if(sheetIter.hasNext()) { - Row header = sheetIter.next(); + if(headerCache.containsKey(tableName)) { + Row header = headerCache.get(tableName); Cell columnHeaderCell = header.getCell(cell.getRowIndex()); if(columnHeaderCell == null) { return EMPTY_CELL_STRING; } Object columnHeaderValue = getCellValue(columnHeaderCell); - if(columnHeaderValue instanceof String) { - return (String) columnHeaderValue; - } else { - return columnHeaderValue.toString(); - } + return columnHeaderValue.toString(); } //No header present return EMPTY_CELL_STRING; @@ -222,6 +240,7 @@ public class ExcelReader extends AbstractReader { /** * Returns a map of sheet names to headers (header is in a comma-seperated string). + * Warning: Only call this ONCE per excel file. * * @return A map of sheet names to header strings. * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderException @@ -234,6 +253,7 @@ public class ExcelReader extends AbstractReader { if(iterator.hasNext()) { //Consume header Row header = iterator.next(); + headerCache.put(sheet.getSheetName(), header); String headerStringFormat = StringUtils.join(header.cellIterator(), ", "); tableSchemas.put(sheet.getSheetName(), headerStringFormat); } diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index b939ced24e..91eba0c5c6 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -82,7 +82,7 @@ public class SQLiteReader extends AbstractReader { * Searches for a meta file associated with the give SQLite database. If found, * copies the file to the local disk folder * - * @param file file being processed + * @param sqliteFile file being processed * @param metaFileName name of meta file to look for * @throws NoCurrentCaseException Case has been closed. * @throws TskCoreException fileManager cannot find AbstractFile files. From 43fb8c8eba31b3a1432ae623b8a4bfbf5424c477 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 17 Aug 2018 14:26:24 -0400 Subject: [PATCH 27/33] Fixed the bugs in the getRowsFromTable method, everything should be working as intended. --- .../tabulardatareader/ExcelReader.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index 607fc6de98..71c3b2e8a7 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -131,7 +131,7 @@ public class ExcelReader extends AbstractReader { //2. This also implies there is no way to determine if a workbook is empty, //since a last row num of 0 doesnt differentiate between a record in 0 or //nothing in the workbook. Such a HSSF. - return getRowsFromTable(tableName, 0, getRowCountFromTable(tableName) + 1); + return getRowsFromTable(tableName, 0, getRowCountFromTable(tableName)); } /** @@ -158,22 +158,29 @@ public class ExcelReader extends AbstractReader { Iterator sheetIter = workbook.getSheet(tableName).rowIterator(); List> rowList = new ArrayList<>(); - int currRowCount = 0; - //Read the header value as the header may be a row of data in the //excel sheet if(headerCache.containsKey(tableName)) { Row header = headerCache.get(tableName); - if(currRowCount++ >= offset) { + if(header.getRowNum() >= offset + && header.getRowNum() < (offset + numRowsToRead)) { rowList.add(getRowMap(tableName, header)); } } - while(sheetIter.hasNext() && currRowCount < (offset + numRowsToRead)) { + while(sheetIter.hasNext()) { Row currRow = sheetIter.next(); - if(currRowCount++ >= offset) { + //If the current row number is within the window of our row capture + if(currRow.getRowNum() >= offset + && currRow.getRowNum() < (offset + numRowsToRead)) { rowList.add(getRowMap(tableName, currRow)); } + + //if current row number is equal to our upper bound + //of rows requested to be read. + if(currRow.getRowNum() >= (offset + numRowsToRead)) { + break; + } } return rowList; From a0ca35d6ce73bdf6bfc4058a194c48b82fdb1632 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 16 Aug 2018 14:40:42 -0400 Subject: [PATCH 28/33] Fixed the out of memory issue with excel xml document reading, caught the old excel format exception, and refactored the code --- Core/nbproject/project.properties | 2 +- Core/nbproject/project.xml | 2 +- .../sleuthkit/autopsy/tabulardatareader/AbstractReader.java | 2 +- .../sleuthkit/autopsy/tabulardatareader/ExcelReader.java | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 08bc1496f9..90ce31dab9 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -15,7 +15,7 @@ file.reference.postgresql-9.4.1211.jre7.jar=release/modules/ext/postgresql-9.4.1 file.reference.Rejistry-1.0-SNAPSHOT.jar=release/modules/ext/Rejistry-1.0-SNAPSHOT.jar file.reference.sevenzipjbinding-AllPlatforms.jar=release/modules/ext/sevenzipjbinding-AllPlatforms.jar file.reference.sevenzipjbinding.jar=release/modules/ext/sevenzipjbinding.jar -file.reference.sqlite-jdbc-3.8.11.jar=release/modules/ext/sqlite-jdbc-3.8.11.jar +file.reference.sqlite-jdbc-3.8.11.jar=release\\modules\\ext\\sqlite-jdbc-3.8.11.jar file.reference.StixLib.jar=release/modules/ext/StixLib.jar file.reference.bcprov-jdk15on-1.54.jar=release/modules/ext/bcprov-jdk15on-1.54.jar file.reference.jackcess-2.1.8.jar=release/modules/ext/jackcess-2.1.8.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index f07138c40f..303d14422c 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -442,7 +442,7 @@ ext/sqlite-jdbc-3.8.11.jar - release/modules/ext/sqlite-jdbc-3.8.11.jar + release\modules\ext\sqlite-jdbc-3.8.11.jar ext/activemq-all-5.11.1.jar diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java index 2e0c24d1db..cb74819142 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java @@ -28,7 +28,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * An abstract reader interface for retrieving contents from files via a common - * API. + * API. */ public abstract class AbstractReader implements AutoCloseable { diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index 71c3b2e8a7..2d4bc7f9be 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -100,8 +100,8 @@ public class ExcelReader extends AbstractReader { //and this can use the same functions below. return StreamingReader.builder().rowCacheSize(500).open(new File(localDiskPath)); default: - throw new FileReaderInitException(String.format("Excel reader for mime " - + "type [%s] is not supported", mimeType)); + throw new FileReaderInitException(String.format("Excel reader for mime " + + "type [%s] is not supported", mimeType)); } } @@ -268,7 +268,7 @@ public class ExcelReader extends AbstractReader { return tableSchemas; } - + @Override public void close() { try { From 77bb15ec358bf9d5d995f8efc857727bdd22db65 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 20 Aug 2018 13:58:03 -0400 Subject: [PATCH 29/33] Patched bug with not catching pois run time exceptions --- .../autopsy/tabulardatareader/FileReaderFactory.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java index 8e98e0447c..5ce902a234 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java @@ -44,7 +44,15 @@ public final class FileReaderFactory { return new SQLiteReader(file, localDiskPath); case "application/vnd.ms-excel": case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": - return new ExcelReader(file, localDiskPath, mimeType); + try { + return new ExcelReader(file, localDiskPath, mimeType); + //Catches runtime exceptions being emitted from Apache + //POI (such as EncryptedDocumentException) as wraps them + //into FileReaderInitException to be caught and logged + //in the ingest module. + } catch(Exception poiInitException) { + throw new FileReaderInitException(poiInitException); + } default: throw new FileReaderInitException(String.format("Reader for mime " + "type [%s] is not supported", mimeType)); From ede32f6fe1ce533cb68b33d0a0ac8e61d74d6443 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 20 Aug 2018 14:26:48 -0400 Subject: [PATCH 30/33] Type fix in factory comment' --- .../autopsy/tabulardatareader/FileReaderFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java index 5ce902a234..5295bb924e 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java @@ -32,7 +32,7 @@ public final class FileReaderFactory { * is not supported. * * @param mimeType mimeType passed in from the ingest module - * @param file current file under inspection +g * @param file current file under inspection * @param localDiskPath path for abstract file contents to be written * @return The correct reader class needed to read the file contents * @throws org.sleuthkit.autopsy.tabulardatareader.AbstractReader.FileReaderInitException @@ -47,7 +47,7 @@ public final class FileReaderFactory { try { return new ExcelReader(file, localDiskPath, mimeType); //Catches runtime exceptions being emitted from Apache - //POI (such as EncryptedDocumentException) as wraps them + //POI (such as EncryptedDocumentException) and wraps them //into FileReaderInitException to be caught and logged //in the ingest module. } catch(Exception poiInitException) { From 60402ba2d20368fe32b09257d8dc4dc2b48ecaef Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 20 Aug 2018 15:12:28 -0400 Subject: [PATCH 31/33] Added final keyword to the readers --- .../org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java | 2 +- .../org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index 2d4bc7f9be..6352741d22 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -45,7 +45,7 @@ import org.apache.poi.hssf.OldExcelFormatException; * Reads excel files and implements the abstract reader api for interfacing with the * content. Supports .xls and .xlsx files. */ -public class ExcelReader extends AbstractReader { +public final class ExcelReader extends AbstractReader { /* Boilerplate code */ private final IngestServices services = IngestServices.getInstance(); private final Logger logger = services.getLogger(ExcelReader.class.getName()); diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index 91eba0c5c6..328f1ac628 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -50,7 +50,7 @@ import org.sleuthkit.datamodel.TskCoreException; @NbBundle.Messages({ "SQLiteReader.ReadSQLiteFiles.moduleName=SQLiteReader" }) -public class SQLiteReader extends AbstractReader { +public final class SQLiteReader extends AbstractReader { private final Connection connection; private final IngestServices services = IngestServices.getInstance(); From e277931d58bb03f9196c971171e621782041ea91 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 22 Aug 2018 14:05:58 -0400 Subject: [PATCH 32/33] Fixed Codacy issues --- .../autopsy/tabulardatareader/ExcelReader.java | 12 ++++++------ .../autopsy/tabulardatareader/FileReaderFactory.java | 2 ++ .../autopsy/tabulardatareader/SQLiteReader.java | 12 ++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index 6352741d22..3b341bb892 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -51,10 +51,10 @@ public final class ExcelReader extends AbstractReader { private final Logger logger = services.getLogger(ExcelReader.class.getName()); private Workbook workbook; - private final String XLSXMimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; - private final String XLSMimeType = "application/vnd.ms-excel"; - private final String EMPTY_CELL_STRING = ""; - private HashMap headerCache; + private final static String XLSX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private final static String XLS_MIME_TYPE = "application/vnd.ms-excel"; + private final static String EMPTY_CELL_STRING = ""; + private Map headerCache; public ExcelReader(AbstractFile file, String localDiskPath, String mimeType) throws FileReaderInitException { @@ -84,7 +84,7 @@ public final class ExcelReader extends AbstractReader { private Workbook createWorkbook(String localDiskPath, String mimeType) throws IOException, FileReaderInitException { switch (mimeType) { - case XLSMimeType: + case XLS_MIME_TYPE: try { //Apache POI only supports BIFF8 format, anything below is considered //old excel format and is not a concern for us. @@ -92,7 +92,7 @@ public final class ExcelReader extends AbstractReader { } catch (OldExcelFormatException e) { throw new FileReaderInitException(e); } - case XLSXMimeType: + case XLSX_MIME_TYPE: //StreamingReader is part of the xlsx streamer dependency that creates //a streaming version of XSSFWorkbook for reading (SXSSFWorkbook is only for writing //large workbooks, not reading). This libary provides a workbook interface diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java index 5295bb924e..e6af1673b2 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/FileReaderFactory.java @@ -26,6 +26,8 @@ import org.sleuthkit.datamodel.AbstractFile; */ public final class FileReaderFactory { + private FileReaderFactory() { + } /** * Instantiates the appropriate reader given the mimeType argument. Currently * supports SQLite files and Excel files (.xls and .xlsx). BIFF5 format of .xls diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index 328f1ac628..2a152f2ffa 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -170,10 +170,10 @@ public final class SQLiteReader extends AbstractReader { @Override public Integer getRowCountFromTable(String tableName) throws FileReaderException { - tableName = wrapTableNameStringWithQuotes(tableName); + String quotedTableName = wrapTableNameStringWithQuotes(tableName); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( - "SELECT count (*) as count FROM " + tableName)){ //NON-NLS + "SELECT count (*) as count FROM " + quotedTableName)){ //NON-NLS return resultSet.getInt("count"); //NON-NLS } catch (SQLException ex) { throw new FileReaderException(ex); @@ -195,10 +195,10 @@ public final class SQLiteReader extends AbstractReader { //This method does not directly call its overloaded counterpart //since the second parameter would need to be retreived from a call to //getTableRowCount(). - tableName = wrapTableNameStringWithQuotes(tableName); + String quotedTableName = wrapTableNameStringWithQuotes(tableName); try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( - "SELECT * FROM " + tableName)) { //NON-NLS + "SELECT * FROM " + quotedTableName)) { //NON-NLS return resultSetToList(resultSet); } catch (SQLException ex) { throw new FileReaderException(ex); @@ -218,10 +218,10 @@ public final class SQLiteReader extends AbstractReader { @Override public List> getRowsFromTable(String tableName, int offset, int numRowsToRead) throws FileReaderException{ - tableName = wrapTableNameStringWithQuotes(tableName); + String quotedTableName = wrapTableNameStringWithQuotes(tableName); try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery( - "SELECT * FROM " + tableName //NON-NLS + "SELECT * FROM " + quotedTableName //NON-NLS + " LIMIT " + Integer.toString(numRowsToRead) //NON-NLS + " OFFSET " + Integer.toString(offset - 1))) { //NON-NLS return resultSetToList(resultSet); From 3b80c792be2610904768ce163dfe640fb386cf13 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 22 Aug 2018 15:13:43 -0400 Subject: [PATCH 33/33] final codacy suggestions...... --- .../tabulardatareader/ExcelReader.java | 4 +-- .../tabulardatareader/SQLiteReader.java | 7 ++--- .../autopsy/testutils/SuiteUtils.java | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/testutils/SuiteUtils.java diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java index 3b341bb892..2fcc79094c 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/ExcelReader.java @@ -47,8 +47,8 @@ import org.apache.poi.hssf.OldExcelFormatException; */ public final class ExcelReader extends AbstractReader { /* Boilerplate code */ - private final IngestServices services = IngestServices.getInstance(); - private final Logger logger = services.getLogger(ExcelReader.class.getName()); + private final static IngestServices services = IngestServices.getInstance(); + private final static Logger logger = services.getLogger(ExcelReader.class.getName()); private Workbook workbook; private final static String XLSX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java index 2a152f2ffa..c08f571280 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/SQLiteReader.java @@ -47,14 +47,11 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Reads sqlite databases and returns results in a list collection. */ -@NbBundle.Messages({ - "SQLiteReader.ReadSQLiteFiles.moduleName=SQLiteReader" -}) public final class SQLiteReader extends AbstractReader { private final Connection connection; - private final IngestServices services = IngestServices.getInstance(); - private final Logger logger = services.getLogger(Bundle.SQLiteReader_ReadSQLiteFiles_moduleName()); + private final static IngestServices services = IngestServices.getInstance(); + private final static Logger logger = services.getLogger(SQLiteReader.class.getName()); /** * Writes data source file contents to local disk and opens a sqlite JDBC diff --git a/Core/src/org/sleuthkit/autopsy/testutils/SuiteUtils.java b/Core/src/org/sleuthkit/autopsy/testutils/SuiteUtils.java new file mode 100755 index 0000000000..ebb0a6452a --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/testutils/SuiteUtils.java @@ -0,0 +1,26 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.testutils; + + +//import junit.framework.Test; +//import org.netbeans.junit.NbModuleSuite; + +/** + * + * @author dsmyda + */ +public final class SuiteUtils { + + /* + public static Test createSuite(Class cls) { + NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(cls). + clusters(".*"). + enableModules(".*"); + return conf.suite(); + } + */ +}