Merge pull request #3990 from raman-bt/1001-ig-multiuser

1001 ig multiuser
This commit is contained in:
Richard Cordovano 2018-08-01 13:16:27 -04:00 committed by GitHub
commit 35303f66c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 312 additions and 44 deletions

View File

@ -329,6 +329,7 @@
<package>org.sleuthkit.autopsy.guiutils</package>
<package>org.sleuthkit.autopsy.healthmonitor</package>
<package>org.sleuthkit.autopsy.ingest</package>
<package>org.sleuthkit.autopsy.ingest.events</package>
<package>org.sleuthkit.autopsy.keywordsearchservice</package>
<package>org.sleuthkit.autopsy.menuactions</package>
<package>org.sleuthkit.autopsy.modules.encryptiondetection</package>

View File

@ -23,8 +23,10 @@ import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
@ -53,11 +55,13 @@ import javafx.scene.layout.Region;
import javafx.scene.layout.StackPane;
import javafx.scene.paint.Color;
import javax.annotation.Nullable;
import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.Case.CaseType;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent;
import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent;
@ -66,6 +70,7 @@ import org.sleuthkit.autopsy.coreutils.History;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.ThreadConfined;
import org.sleuthkit.autopsy.events.AutopsyEvent;
import org.sleuthkit.autopsy.imagegallery.actions.UndoRedoManager;
import org.sleuthkit.autopsy.imagegallery.datamodel.CategoryManager;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
@ -77,9 +82,11 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState;
import org.sleuthkit.autopsy.imagegallery.gui.NoGroupsDialog;
import org.sleuthkit.autopsy.imagegallery.gui.Toolbar;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
@ -213,11 +220,15 @@ public final class ImageGalleryController {
listeningEnabled.addListener((observable, oldValue, newValue) -> {
try {
//if we just turned on listening and a case is open and that case is not up to date
if (newValue && !oldValue && ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows())) {
// if we just turned on listening and a single-user case is open and that case is not up to date, then rebuild it
// For multiuser cases, we defer DB rebuild till the user actually opens Image Gallery
if ( newValue && !oldValue &&
ImageGalleryModule.isDrawableDBStale(Case.getCurrentCaseThrows()) &&
(Case.getCurrentCaseThrows().getCaseType() == CaseType.SINGLE_USER_CASE) ) {
//populate the db
queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase));
this.rebuildDB();
}
} catch (NoCurrentCaseException ex) {
LOGGER.log(Level.WARNING, "Exception while getting open case.", ex);
}
@ -386,6 +397,14 @@ public final class ImageGalleryController {
}
}
/**
* Rebuilds the DrawableDB database.
*
*/
public void rebuildDB() {
queueDBTask(new CopyAnalyzedFiles(instance, db, sleuthKitCase));
}
/**
* reset the state of the controller (eg if the case is closed)
*/
@ -411,6 +430,56 @@ public final class ImageGalleryController {
db = null;
}
/**
* Checks if the datasources table in drawable DB is stale.
*
* @return true if datasources table is stale
*/
boolean isDataSourcesTableStale() {
// no current case open to check
if ((null == getDatabase()) || (null == getSleuthKitCase())) {
return false;
}
try {
Set<Long> knownDataSourceIds= getDatabase().getDataSourceIds();
List<DataSource> dataSources = getSleuthKitCase().getDataSources();
Set<Long> caseDataSourceIds = new HashSet<>();
dataSources.forEach((dataSource) -> {
caseDataSourceIds.add(dataSource.getId());
});
return !(knownDataSourceIds.containsAll(caseDataSourceIds) && caseDataSourceIds.containsAll(knownDataSourceIds));
}
catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Image Gallery failed to check if datasources table is stale.", ex);
return false;
}
}
/**
* Update the datasources table in drawable DB.
*
*/
private void updateDataSourcesTable() {
// no current case open to update
if ((null == getDatabase()) || (null == getSleuthKitCase())) {
return;
}
try {
List<DataSource> dataSources = getSleuthKitCase().getDataSources();
dataSources.forEach((dataSource) -> {
getDatabase().insertDataSource(dataSource.getId());
});
}
catch (TskCoreException ex) {
LOGGER.log(Level.SEVERE, "Image Gallery failed to update data_sources table.", ex);
}
}
synchronized private void shutDownDBExecutor() {
if (dbExecutor != null) {
dbExecutor.shutdownNow();
@ -483,8 +552,8 @@ public final class ImageGalleryController {
void onStart() {
Platform.setImplicitExit(false);
LOGGER.info("setting up ImageGallery listeners"); //NON-NLS
//TODO can we do anything usefull in an InjestJobEventListener?
//IngestManager.getInstance().addIngestJobEventListener((PropertyChangeEvent evt) -> {});
IngestManager.getInstance().addIngestJobEventListener( new IngestJobEventListener());
IngestManager.getInstance().addIngestModuleEventListener(new IngestModuleEventListener());
Case.addPropertyChangeListener(new CaseEventListener());
}
@ -661,9 +730,9 @@ public final class ImageGalleryController {
abstract static private class BulkTransferTask extends BackgroundTask {
static private final String FILE_EXTENSION_CLAUSE =
"(name LIKE '%." //NON-NLS
+ String.join("' OR name LIKE '%.", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS
+ "')";
"(extension LIKE '" //NON-NLS
+ String.join("' OR extension LIKE '", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS
+ "') ";
static private final String MIMETYPE_CLAUSE =
"(mime_type LIKE '" //NON-NLS
@ -684,6 +753,8 @@ public final class ImageGalleryController {
ProgressHandle progressHandle;
private boolean taskCompletionStatus;
BulkTransferTask(ImageGalleryController controller, DrawableDB taskDB, SleuthkitCase tskCase) {
this.controller = controller;
this.taskDB = taskDB;
@ -713,16 +784,22 @@ public final class ImageGalleryController {
updateProgress(0.0);
taskCompletionStatus = true;
//do in transaction
DrawableDB.DrawableTransaction tr = taskDB.beginTransaction();
int workDone = 0;
for (final AbstractFile f : files) {
if (isCancelled() || Thread.interrupted()) {
if (isCancelled()) {
LOGGER.log(Level.WARNING, "Task cancelled: not all contents may be transfered to drawable database."); //NON-NLS
progressHandle.finish();
break;
}
if (Thread.interrupted()) {
LOGGER.log(Level.WARNING, "BulkTransferTask interrupted. Ignoring it to update the contents of drawable database."); //NON-NLS
}
processFile(f, tr);
workDone++;
@ -750,10 +827,14 @@ public final class ImageGalleryController {
updateMessage("");
updateProgress(-1.0);
}
cleanup(true);
cleanup(taskCompletionStatus);
}
abstract ProgressHandle getInitialProgressHandle();
protected void setTaskCompletionStatus(boolean status) {
taskCompletionStatus = status;
}
}
/**
@ -774,6 +855,7 @@ public final class ImageGalleryController {
@Override
protected void cleanup(boolean success) {
controller.updateDataSourcesTable();
controller.setStale(!success);
}
@ -783,7 +865,7 @@ public final class ImageGalleryController {
}
@Override
void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr) {
void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr) throws TskCoreException {
final boolean known = f.getKnown() == TskData.FileKnown.KNOWN;
if (known) {
@ -791,13 +873,20 @@ public final class ImageGalleryController {
} else {
try {
if (FileTypeUtils.hasDrawableMIMEType(f)) { //supported mimetype => analyzed
//supported mimetype => analyzed
if ( null != f.getMIMEType() && FileTypeUtils.hasDrawableMIMEType(f)) {
taskDB.updateFile(DrawableFile.create(f, true, false), tr);
} else { //unsupported mimtype => analyzed but shouldn't include
}
else { //unsupported mimtype => analyzed but shouldn't include
// if mimetype of the file hasn't been ascertained, ingest might not have completed yet.
if (null == f.getMIMEType()) {
this.setTaskCompletionStatus(false);
}
taskDB.removeFile(f.getId(), tr);
}
} catch (FileTypeDetector.FileTypeDetectorInitException ex) {
throw new RuntimeException(ex);
throw new TskCoreException("Failed to initialize FileTypeDetector.", ex);
}
}
}
@ -890,6 +979,9 @@ public final class ImageGalleryController {
AbstractFile file = (AbstractFile) evt.getNewValue();
// only process individual files in realtime on the node that is running the ingest
// on a remote node, image files are processed enblock when ingest is complete
if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) {
if (isListeningEnabled()) {
if (file.isFile()) {
try {
@ -913,6 +1005,7 @@ public final class ImageGalleryController {
} else { //TODO: keep track of what we missed for later
setStale(true);
}
}
break;
}
}
@ -943,13 +1036,15 @@ public final class ImageGalleryController {
}
break;
case DATA_SOURCE_ADDED:
//copy all file data to drawable databse
//For a data source added on the local node, prepopulate all file data to drawable database
if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) {
Content newDataSource = (Content) evt.getNewValue();
if (isListeningEnabled()) {
queueDBTask(new PrePopulateDataSourceFiles(newDataSource, ImageGalleryController.this, getDatabase(), getSleuthKitCase()));
} else {//TODO: keep track of what we missed for later
setStale(true);
}
}
break;
case CONTENT_TAG_ADDED:
final ContentTagAddedEvent tagAddedEvent = (ContentTagAddedEvent) evt;
@ -966,4 +1061,62 @@ public final class ImageGalleryController {
}
}
}
/**
* Listener for Ingest Job events.
*/
private class IngestJobEventListener implements PropertyChangeListener {
@NbBundle.Messages({
"ImageGalleryController.dataSourceAnalyzed.confDlg.msg= A new data source was added and finished ingest.\n" +
"The image / video database may be out of date. " +
"Do you want to update the database with ingest results?\n",
"ImageGalleryController.dataSourceAnalyzed.confDlg.title=Image Gallery"
})
@Override
public void propertyChange(PropertyChangeEvent evt) {
String eventName = evt.getPropertyName();
if ( eventName.equals(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED.toString())) {
if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) {
// A remote node added a new data source and just finished ingest on it.
//drawable db is stale, and if ImageGallery is open, ask user what to do
setStale(true);
SwingUtilities.invokeLater(() -> {
if (isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) {
int answer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(),
Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(),
Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(),
JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE);
switch (answer) {
case JOptionPane.YES_OPTION:
rebuildDB();
break;
case JOptionPane.NO_OPTION:
case JOptionPane.CANCEL_OPTION:
default:
break; //do nothing
}
}
});
} else {
// received event from local node
// add the datasource to drawable db
long dsObjId = 0;
DataSourceAnalysisCompletedEvent event = (DataSourceAnalysisCompletedEvent)evt;
if(event.getDataSource() != null) {
dsObjId = event.getDataSource().getId();
db.insertDataSource(dsObjId);
// All files for the data source have been analyzed.
setStale(false);
} else {
LOGGER.log(Level.SEVERE, "DataSourceAnalysisCompletedEvent does not contain a dataSource object"); //NON-NLS
}
}
}
}
}
}

View File

@ -86,7 +86,9 @@ public class ImageGalleryModule {
public static boolean isDrawableDBStale(Case c) {
if (c != null) {
String stale = new PerCaseProperties(c).getConfigSetting(ImageGalleryModule.MODULE_NAME, PerCaseProperties.STALE);
return StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : true;
return ( ImageGalleryController.getDefault().isDataSourcesTableStale() ||
(StringUtils.isNotBlank(stale) ? Boolean.valueOf(stale) : true) );
} else {
return false;
}

View File

@ -39,6 +39,7 @@ import org.openide.windows.RetainLocation;
import org.openide.windows.TopComponent;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ThreadConfined;
import org.sleuthkit.autopsy.imagegallery.gui.StatusBar;
import org.sleuthkit.autopsy.imagegallery.gui.SummaryTablePane;
import org.sleuthkit.autopsy.imagegallery.gui.Toolbar;
@ -88,6 +89,29 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl
private VBox leftPane;
private Scene myScene;
/**
* Returns whether the ImageGallery window is open or not.
*
* @return true, if Image gallery is opened, false otherwise
*/
public static boolean isImageGalleryOpen() {
final TopComponent topComponent = WindowManager.getDefault().findTopComponent(PREFERRED_ID);
if (topComponent != null) {
return topComponent.isOpened();
}
return false;
}
/**
* Returns the top component window.
*
* @return Image gallery top component window, null if it's not open
*/
public static TopComponent getTopComponent() {
return WindowManager.getDefault().findTopComponent(PREFERRED_ID);
}
public static void openTopComponent() {
//TODO:eventually move to this model, throwing away everything and rebuilding controller groupmanager etc for each case.
// synchronized (OpenTimelineAction.class) {

View File

@ -172,7 +172,18 @@ public final class OpenAction extends CallableSystemAction {
switch (answer) {
case JOptionPane.YES_OPTION:
// For a single-user case, we favor user experience, and rebuild the database
// as soon as Image Gallery is enabled for the case.
// For a multi-user case, we favor overall performance and user experience, not every user may want to review images,
// so we rebuild the database only when a user launches Image Gallery
if (currentCase.getCaseType() == Case.CaseType.SINGLE_USER_CASE) {
ImageGalleryController.getDefault().setListeningEnabled(true);
}
else {
ImageGalleryController.getDefault().rebuildDB();
}
//fall through
case JOptionPane.NO_OPTION:
ImageGalleryTopComponent.openTopComponent();

View File

@ -47,7 +47,6 @@ import javax.annotation.Nonnull;
import javax.annotation.concurrent.GuardedBy;
import javax.swing.SortOrder;
import org.apache.commons.lang3.StringUtils;
import org.openide.util.Exceptions;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.imagegallery.FileTypeUtils;
@ -102,6 +101,8 @@ public final class DrawableDB {
private final PreparedStatement insertHashHitStmt;
private final PreparedStatement insertDataSourceStmt;
private final PreparedStatement updateFileStmt;
private final PreparedStatement insertFileStmt;
@ -208,6 +209,10 @@ public final class DrawableDB {
"INSERT OR IGNORE INTO drawable_files (obj_id , path, name, created_time, modified_time, make, model, analyzed) " //NON-NLS
+ "VALUES (?,?,?,?,?,?,?,?)"); //NON-NLS
insertDataSourceStmt = prepareStatement(
"INSERT OR IGNORE INTO datasources (ds_obj_id) " //NON-NLS
+ "VALUES (?)"); //NON-NLS
removeFileStmt = prepareStatement("DELETE FROM drawable_files WHERE obj_id = ?"); //NON-NLS
pathGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE path = ? ", DrawableAttribute.PATH); //NON-NLS
@ -348,6 +353,17 @@ public final class DrawableDB {
LOGGER.log(Level.SEVERE, "problem accessing database", ex); //NON-NLS
return false;
}
try (Statement stmt = con.createStatement()) {
String sql = "CREATE TABLE if not exists datasources " //NON-NLS
+ "( id INTEGER PRIMARY KEY, " //NON-NLS
+ " ds_obj_id integer UNIQUE NOT NULL)"; //NON-NLS
stmt.execute(sql);
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "problem creating datasources table", ex); //NON-NLS
return false;
}
try (Statement stmt = con.createStatement()) {
String sql = "CREATE TABLE if not exists drawable_files " //NON-NLS
+ "( obj_id INTEGER PRIMARY KEY, " //NON-NLS
@ -688,6 +704,67 @@ public final class DrawableDB {
}
}
/**
* Gets all data source object ids from datasources table
*
* @return list of known data source object ids
*/
public Set<Long> getDataSourceIds() throws TskCoreException {
Statement statement = null;
ResultSet rs = null;
Set<Long> ret = new HashSet<>();
dbReadLock();
try {
statement = con.createStatement();
rs = statement.executeQuery("SELECT ds_obj_id FROM datasources "); //NON-NLS
while (rs.next()) {
ret.add(rs.getLong(1));
}
} catch (SQLException e) {
throw new TskCoreException("SQLException while getting data source object ids", e);
} finally {
if (rs != null) {
try {
rs.close();
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "Error closing resultset", ex); //NON-NLS
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException ex) {
LOGGER.log(Level.SEVERE, "Error closing statement ", ex); //NON-NLS
}
}
dbReadUnlock();
}
return ret;
}
/**
* Insert given data source object id into datasources table
*
* If the object id exists in the table already, it does nothing.
*
* @param dsObjectId data source object id to insert
*/
public void insertDataSource(long dsObjectId) {
dbWriteLock();
try {
// "INSERT OR IGNORE/ INTO datasources (ds_obj_id)"
insertDataSourceStmt.setLong(1,dsObjectId);
insertDataSourceStmt.executeUpdate();
} catch (SQLException | NullPointerException ex) {
LOGGER.log(Level.SEVERE, "failed to insert/update datasources table", ex); //NON-NLS
} finally {
dbWriteUnlock();
}
}
public DrawableTransaction beginTransaction() {
return new DrawableTransaction();
}