mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6944 from kellykelly3/7286-ig-rebuild-feedback

Browse Source 
7286 ig rebuild feedback
This commit is contained in:
Richard Cordovano 2021-05-06 09:19:39 -04:00 committed by GitHub
commit 8cef408da5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 144 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/AddDrawableFilesTask.java
View File

@ -1,78 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.imagegallery;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableFile;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
* A task that queries the case database for all files with supported
* image/video mime types or extensions and adds them to the drawables database.
*/
class AddDrawableFilesTask extends BulkDrawableFilesTask {
private final ImageGalleryController controller;
private final DrawableDB taskDB;
AddDrawableFilesTask(long dataSourceObjId, ImageGalleryController controller) {
super(dataSourceObjId, controller);
this.controller = controller;
this.taskDB = controller.getDrawablesDatabase();
taskDB.buildFileMetaDataCache();
}
@Override
protected void cleanup() {
taskDB.freeFileMetaDataCache();
// at the end of the task, set the stale status based on the
// cumulative status of all data sources
controller.setModelIsStale(controller.isDataSourcesTableStale());
}
@Override
void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr, SleuthkitCase.CaseDbTransaction caseDbTransaction) throws TskCoreException {
final boolean known = f.getKnown() == TskData.FileKnown.KNOWN;
if (known) {
taskDB.removeFile(f.getId(), tr); //remove known files
} else {
// NOTE: Files are being processed because they have the right MIME type,
// so we do not need to worry about this calculating them
if (FileTypeUtils.hasDrawableMIMEType(f)) {
taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction);
} //unsupported mimtype => analyzed but shouldn't include
else {
taskDB.removeFile(f.getId(), tr);
}
}
}
@Override
@NbBundle.Messages({
"AddDrawableFilesTask.populatingDb.status=populating analyzed image/video database"
})
ProgressHandle getInitialProgressHandle() {
return ProgressHandle.createHandle(Bundle.AddDrawableFilesTask_populatingDb_status(), this);
}
}

9
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/Bundle.properties-MERGED
View File

@ -1,12 +1,11 @@
AddDrawableFilesTask.populatingDb.status=populating analyzed image/video database
BulkDrawableFilesTask.committingDb.status=committing image/video database
BulkDrawableFilesTask.errPopulating.errMsg=There was an error populating Image Gallery database.
BulkDrawableFilesTask.populatingDb.status=populating analyzed image/video database
BulkDrawableFilesTask.stopCopy.status=Stopping copy to drawable db task.
CTL_ImageGalleryAction=Image/Video Gallery
CTL_ImageGalleryTopComponent=Image/Video Gallery
DrawableDbTask.InnerTask.message.name=status
DrawableDbTask.InnerTask.progress.name=progress
DrawableFileUpdateTask_committingDb.status=committing image/video database
DrawableFileUpdateTask_errPopulating_errMsg=There was an error populating Image Gallery database.
DrawableFileUpdateTask_populatingDb_status=populating analyzed image/video database
DrawableFileUpdateTask_stopCopy_status=Stopping copy to drawable db task.
ImageGallery.dialogTitle=Image Gallery
ImageGallery.showTooManyFiles.contentText=There are too many files in the selected datasource(s) to ensure reasonable performance.
ImageGallery.showTooManyFiles.headerText=

190
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/BulkDrawableFilesTask.java → ImageGallery/src/org/sleuthkit/autopsy/imagegallery/DrawableFileUpdateTask.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015-2019 Basis Technology Corp.
* Copyright 2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -22,100 +22,159 @@ import java.sql.SQLException;
import java.util.List;
import java.util.logging.Level;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableFile;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
* An abstract base class for tasks that add or modify the drawables database
* records for multiple drawable files.
* A bulk update task for adding images to the image gallery.
*/
@NbBundle.Messages({
"BulkDrawableFilesTask.committingDb.status=committing image/video database",
"BulkDrawableFilesTask.stopCopy.status=Stopping copy to drawable db task.",
"BulkDrawableFilesTask.errPopulating.errMsg=There was an error populating Image Gallery database."
})
abstract class BulkDrawableFilesTask extends DrawableDbTask {
final class DrawableFileUpdateTask extends DrawableDbTask {
private static final Logger logger = Logger.getLogger(DrawableFileUpdateTask.class.getName());
private static final Logger logger = Logger.getLogger(BulkDrawableFilesTask.class.getName());
private static final String MIMETYPE_CLAUSE = "(mime_type LIKE '" //NON-NLS
+ String.join("' OR mime_type LIKE '", FileTypeUtils.getAllSupportedMimeTypes()) //NON-NLS
+ "') ";
private final String drawableQuery;
private final ImageGalleryController controller;
private final DrawableDB taskDB;
private final SleuthkitCase tskCase;
private final long dataSourceObjId;
//NON-NLS
BulkDrawableFilesTask(long dataSourceObjId, ImageGalleryController controller) {
private final ImageGalleryController controller;
/**
* Construct a new task.
*
* @param controller A handle to the IG controller.
*/
DrawableFileUpdateTask(ImageGalleryController controller) {
this.controller = controller;
this.taskDB = controller.getDrawablesDatabase();
this.tskCase = controller.getCaseDatabase();
this.dataSourceObjId = dataSourceObjId;
drawableQuery = " (data_source_obj_id = " + dataSourceObjId + ") "
}
@Override
public void run() {
for (Long dataSourceObjId : controller.getStaleDataSourceIds()) {
updateFileForDataSource(dataSourceObjId);
}
}
/**
* Gets the drawables database that is part of the model for the controller.
*
* @return The the drawable db object.
*/
private DrawableDB getDrawableDB() {
return controller.getDrawablesDatabase();
}
/**
* Return the sleuthkit case object for the open case.
*
* @return The case db object.
*/
private SleuthkitCase getCaseDB() {
return controller.getCaseDatabase();
}
/**
* Returns a list of files to be processed by the task for the given
* datasource.
*
* @param dataSourceObjId
* @return
* @throws TskCoreException
*/
private List<AbstractFile> getFilesForDataSource(long dataSourceObjId) throws TskCoreException {
List<AbstractFile> list = getCaseDB().findAllFilesWhere(getDrawableQuery(dataSourceObjId));
return list;
}
/**
* Process a single file for the IG drawable db.
*
* @param file The file to process.
* @param tr A valid DrawableTransaction object.
* @param caseDbTransaction A valid caseDBTransaction object.
*
* @throws TskCoreException
*/
void processFile(AbstractFile file, DrawableDB.DrawableTransaction tr, SleuthkitCase.CaseDbTransaction caseDbTransaction) throws TskCoreException {
final boolean known = file.getKnown() == TskData.FileKnown.KNOWN;
if (known) {
getDrawableDB().removeFile(file.getId(), tr); //remove known files
} else {
// NOTE: Files are being processed because they have the right MIME type,
// so we do not need to worry about this calculating them
if (FileTypeUtils.hasDrawableMIMEType(file)) {
getDrawableDB().updateFile(DrawableFile.create(file, true, false), tr, caseDbTransaction);
} //unsupported mimtype => analyzed but shouldn't include
else {
getDrawableDB().removeFile(file.getId(), tr);
}
}
}
/**
* Returns the image query for the given data source.
*
* @param dataSourceObjId
*
* @return SQL query for given data source.
*/
private String getDrawableQuery(long dataSourceObjId) {
return " (data_source_obj_id = " + dataSourceObjId + ") "
+ " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + " AND ( " + MIMETYPE_CLAUSE //NON-NLS
+ " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )" //NON-NLS
+ " ORDER BY parent_path ";
}
/**
* Do any cleanup for this task.
*/
abstract void cleanup();
abstract void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, SleuthkitCase.CaseDbTransaction caseDBTransaction) throws TskCoreException;
/**
* Gets a list of files to process.
*
* @return list of files to process
*
* @throws TskCoreException
*/
List<AbstractFile> getFiles() throws TskCoreException {
return tskCase.findAllFilesWhere(drawableQuery);
}
@Override
@NbBundle.Messages({
"BulkDrawableFilesTask.populatingDb.status=populating analyzed image/video database"
@Messages({
"DrawableFileUpdateTask_populatingDb_status=populating analyzed image/video database",
"DrawableFileUpdateTask_committingDb.status=committing image/video database",
"DrawableFileUpdateTask_stopCopy_status=Stopping copy to drawable db task.",
"DrawableFileUpdateTask_errPopulating_errMsg=There was an error populating Image Gallery database."
})
public void run() {
private void updateFileForDataSource(long dataSourceObjId) {
ProgressHandle progressHandle = getInitialProgressHandle();
progressHandle.start();
updateMessage(Bundle.BulkDrawableFilesTask_populatingDb_status() + " (Data Source " + dataSourceObjId + ")");
updateMessage(Bundle.DrawableFileUpdateTask_populatingDb_status() + " (Data Source " + dataSourceObjId + ")");
DrawableDB.DrawableTransaction drawableDbTransaction = null;
SleuthkitCase.CaseDbTransaction caseDbTransaction = null;
boolean hasFilesWithNoMime = true;
boolean endedEarly = false;
try {
getDrawableDB().buildFileMetaDataCache();
// See if there are any files in the DS w/out a MIME TYPE
hasFilesWithNoMime = controller.hasFilesWithNoMimeType(dataSourceObjId);
//grab all files with detected mime types
final List<AbstractFile> files = getFiles();
final List<AbstractFile> files = getFilesForDataSource(dataSourceObjId);
progressHandle.switchToDeterminate(files.size());
taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS);
getDrawableDB().insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS);
updateProgress(0.0);
int workDone = 0;
// Cycle through all of the files returned and call processFile on each
//do in transaction
drawableDbTransaction = taskDB.beginTransaction();
drawableDbTransaction = getDrawableDB().beginTransaction();
/*
* We are going to periodically commit the CaseDB transaction and
* sleep so that the user can have Autopsy do other stuff while
* these bulk tasks are ongoing.
*/
int caseDbCounter = 0;
for (final AbstractFile f : files) {
updateMessage(f.getName());
if (caseDbTransaction == null) {
caseDbTransaction = tskCase.beginTransaction();
caseDbTransaction = getCaseDB().beginTransaction();
}
if (isCancelled() || Thread.interrupted()) {
logger.log(Level.WARNING, "Task cancelled or interrupted: not all contents may be transfered to drawable database."); //NON-NLS
endedEarly = true;
@ -132,20 +191,19 @@ abstract class BulkDrawableFilesTask extends DrawableDbTask {
if ((++caseDbCounter % 200) == 0) {
caseDbTransaction.commit();
caseDbTransaction = null;
Thread.sleep(500); // 1/2 second
Thread.sleep(500); // 1/2 millisecond
}
}
progressHandle.finish();
progressHandle = ProgressHandle.createHandle(Bundle.BulkDrawableFilesTask_committingDb_status());
updateMessage(Bundle.BulkDrawableFilesTask_committingDb_status() + " (Data Source " + dataSourceObjId + ")");
progressHandle = ProgressHandle.createHandle(Bundle.DrawableFileUpdateTask_committingDb_status());
updateMessage(Bundle.DrawableFileUpdateTask_committingDb_status() + " (Data Source " + dataSourceObjId + ")");
updateProgress(1.0);
progressHandle.start();
if (caseDbTransaction != null) {
caseDbTransaction.commit();
caseDbTransaction = null;
}
// pass true so that groupmanager is notified of the changes
taskDB.commitTransaction(drawableDbTransaction, true);
getDrawableDB().commitTransaction(drawableDbTransaction, true);
drawableDbTransaction = null;
} catch (TskCoreException | SQLException | InterruptedException ex) {
if (null != caseDbTransaction) {
@ -157,14 +215,14 @@ abstract class BulkDrawableFilesTask extends DrawableDbTask {
}
if (null != drawableDbTransaction) {
try {
taskDB.rollbackTransaction(drawableDbTransaction);
getDrawableDB().rollbackTransaction(drawableDbTransaction);
} catch (SQLException ex2) {
logger.log(Level.SEVERE, String.format("Failed to roll back drawables db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS
}
}
progressHandle.progress(Bundle.BulkDrawableFilesTask_stopCopy_status());
progressHandle.progress(Bundle.DrawableFileUpdateTask_stopCopy_status());
logger.log(Level.WARNING, "Stopping copy to drawable db task. Failed to transfer all database contents", ex); //NON-NLS
MessageNotifyUtil.Notify.warn(Bundle.BulkDrawableFilesTask_errPopulating_errMsg(), ex.getMessage());
MessageNotifyUtil.Notify.warn(Bundle.DrawableFileUpdateTask_errPopulating_errMsg(), ex.getMessage());
endedEarly = true;
} finally {
progressHandle.finish();
@ -172,15 +230,27 @@ abstract class BulkDrawableFilesTask extends DrawableDbTask {
// if there was cancellation or errors
DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = ((hasFilesWithNoMime == true) || (endedEarly == true)) ? DrawableDB.DrawableDbBuildStatusEnum.REBUILT_STALE : DrawableDB.DrawableDbBuildStatusEnum.COMPLETE;
try {
taskDB.insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus);
getDrawableDB().insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus);
} catch (SQLException ex) {
logger.log(Level.SEVERE, String.format("Error updating datasources table (data source object ID = %d, status = %s)", dataSourceObjId, datasourceDrawableDBStatus.toString(), ex)); //NON-NLS
}
updateMessage("");
updateProgress(-1.0);
getDrawableDB().freeFileMetaDataCache();
// at the end of the task, set the stale status based on the
// cumulative status of all data sources
controller.setModelIsStale(controller.isDataSourcesTableStale());
}
cleanup();
}
abstract ProgressHandle getInitialProgressHandle();
/**
* Returns a ProgressHandle.
*
* @return A new ProgressHandle.
*/
private ProgressHandle getInitialProgressHandle() {
return ProgressHandle.createHandle(Bundle.DrawableFileUpdateTask_populatingDb_status(), this);
}
}

5
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java
View File

@ -486,8 +486,7 @@ public final class ImageGalleryController {
*
*/
public void rebuildDrawablesDb() {
// queue a rebuild task for each stale data source
getStaleDataSourceIds().forEach(dataSourceObjId -> queueDBTask(new AddDrawableFilesTask(dataSourceObjId, this)));
queueDBTask(new DrawableFileUpdateTask(this));
}
/**
@ -670,7 +669,7 @@ public final class ImageGalleryController {
*
* @param bgTask
*/
public synchronized void queueDBTask(DrawableDbTask bgTask) {
public synchronized void queueDBTask(Runnable bgTask) {
if (!dbExecutor.isShutdown()) {
incrementQueueSize();
dbExecutor.submit(bgTask).addListener(this::decrementQueueSize, MoreExecutors.directExecutor());

2
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/Bundle.properties-MERGED
View File

@ -37,7 +37,7 @@ OpenAction.noControllerDialog.text=An initialization error ocurred.\nPlease see
OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet.\nPlease run FileType and EXIF ingest modules.
OpenAction.openTopComponent.error.message=An error occurred while attempting to open Image Gallery.
OpenAction.openTopComponent.error.title=Failed to open Image Gallery
OpenAction.stale.confDlg.msg=The image / video database may be out of date. Do you want to update and listen for further ingest results?\nChoosing 'yes' will update the database and enable listening to future ingests.
OpenAction.stale.confDlg.msg=The image / video database may be out of date. Do you want to update and listen for further ingest results?\nChoosing 'yes' will update the database and enable listening to future ingests.\n\nDatabase update status will appear in the lower right corner of the application window.
OpenAction.stale.confDlg.title=Image Gallery
OpenExternalViewerAction.displayName=External Viewer
RedoAction.name=Redo

27
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java
View File

@ -40,6 +40,7 @@ import org.openide.awt.ActionID;
import org.openide.awt.ActionReference;
import org.openide.awt.ActionReferences;
import org.openide.awt.ActionRegistration;
import org.openide.util.Exceptions;
import org.openide.util.HelpCtx;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
@ -69,7 +70,8 @@ import org.sleuthkit.datamodel.TskCoreException;
@Messages({"CTL_OpenAction=Images/Videos",
"OpenAction.stale.confDlg.msg=The image / video database may be out of date. "
+ "Do you want to update and listen for further ingest results?\n"
+ "Choosing 'yes' will update the database and enable listening to future ingests.",
+ "Choosing 'yes' will update the database and enable listening to future ingests.\n\n"
+ "Database update status will appear in the lower right corner of the application window.",
"OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet.\n"
+ "Please run FileType and EXIF ingest modules.",
"OpenAction.stale.confDlg.title=Image Gallery"})
@ -254,28 +256,7 @@ public final class OpenAction extends CallableSystemAction {
// They don't want to rebuild. Just open the UI as is.
// NOTE: There could be no data....
} else if (answer == ButtonType.YES) {
if (controller.getCase().getCaseType() == Case.CaseType.SINGLE_USER_CASE) {
/*
* For a single-user case, we favor user
* experience, and rebuild the database as soon
* as Image Gallery is enabled for the case.
*
* Turning listening off is necessary in order
* to invoke the listener that will call
* controller.rebuildDB();
*/
controller.setListeningEnabled(false);
controller.setListeningEnabled(true);
} else {
/*
* For a multi-user case, we favor overall
* performance and user experience, not every
* user may want to review images, so we rebuild
* the database only when a user launches Image
* Gallery.
*/
controller.rebuildDrawablesDb();
}
controller.rebuildDrawablesDb();
}
openTopComponent();
return;

5
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java
View File

@ -66,6 +66,7 @@ import org.sleuthkit.autopsy.imagegallery.ImageGalleryController;
import org.sleuthkit.autopsy.imagegallery.actions.CategorizeGroupAction;
import org.sleuthkit.autopsy.imagegallery.actions.TagGroupAction;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.DrawableGroup;
import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupSortBy;
import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState;
@ -236,8 +237,8 @@ public class Toolbar extends ToolBar {
}
private void initDataSourceComboBox() {
dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus()));
dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus()));
dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable, new HashMap<>()));
dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable, new HashMap<>()));
dataSourceComboBox.setConverter(new StringConverter<Optional<DataSource>>() {
@Override
public String toString(Optional<DataSource> object) {