mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed IG issue

Browse Source
This commit is contained in:
Kelly Kelly 2021-04-28 16:07:15 -04:00
parent 385fd31b3b
commit cb0f0fd104
6 changed files with 148 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/AddDrawableFilesTask.java
View File

@ -1,78 +0,0 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015-2019 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.imagegallery;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableFile;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
* A task that queries the case database for all files with supported
* image/video mime types or extensions and adds them to the drawables database.
*/
class AddDrawableFilesTask extends BulkDrawableFilesTask {
private final ImageGalleryController controller;
private final DrawableDB taskDB;
AddDrawableFilesTask(long dataSourceObjId, ImageGalleryController controller) {
super(dataSourceObjId, controller);
this.controller = controller;
this.taskDB = controller.getDrawablesDatabase();
taskDB.buildFileMetaDataCache();
}
@Override
protected void cleanup() {
taskDB.freeFileMetaDataCache();
// at the end of the task, set the stale status based on the
// cumulative status of all data sources
controller.setModelIsStale(controller.isDataSourcesTableStale());
}
@Override
void processFile(AbstractFile f, DrawableDB.DrawableTransaction tr, SleuthkitCase.CaseDbTransaction caseDbTransaction) throws TskCoreException {
final boolean known = f.getKnown() == TskData.FileKnown.KNOWN;
if (known) {
taskDB.removeFile(f.getId(), tr); //remove known files
} else {
// NOTE: Files are being processed because they have the right MIME type,
// so we do not need to worry about this calculating them
if (FileTypeUtils.hasDrawableMIMEType(f)) {
taskDB.updateFile(DrawableFile.create(f, true, false), tr, caseDbTransaction);
} //unsupported mimtype => analyzed but shouldn't include
else {
taskDB.removeFile(f.getId(), tr);
}
}
}
@Override
@NbBundle.Messages({
"AddDrawableFilesTask.populatingDb.status=populating analyzed image/video database"
})
ProgressHandle getInitialProgressHandle() {
return ProgressHandle.createHandle(Bundle.AddDrawableFilesTask_populatingDb_status(), this);
}
}

9
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/Bundle.properties-MERGED
View File

@ -1,12 +1,11 @@
AddDrawableFilesTask.populatingDb.status=populating analyzed image/video database
BulkDrawableFilesTask.committingDb.status=committing image/video database
BulkDrawableFilesTask.errPopulating.errMsg=There was an error populating Image Gallery database.
BulkDrawableFilesTask.populatingDb.status=populating analyzed image/video database
BulkDrawableFilesTask.stopCopy.status=Stopping copy to drawable db task.
CTL_ImageGalleryAction=Image/Video Gallery
CTL_ImageGalleryTopComponent=Image/Video Gallery
DrawableDbTask.InnerTask.message.name=status
DrawableDbTask.InnerTask.progress.name=progress
DrawableFileUpdateTask_committingDb.status=committing image/video database
DrawableFileUpdateTask_errPopulating_errMsg=There was an error populating Image Gallery database.
DrawableFileUpdateTask_populatingDb_status=populating analyzed image/video database
DrawableFileUpdateTask_stopCopy_status=Stopping copy to drawable db task.
ImageGallery.dialogTitle=Image Gallery
ImageGallery.showTooManyFiles.contentText=There are too many files in the selected datasource(s) to ensure reasonable performance.
ImageGallery.showTooManyFiles.headerText=

196
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/BulkDrawableFilesTask.java → ImageGallery/src/org/sleuthkit/autopsy/imagegallery/DrawableFileUpdateTask.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015-2019 Basis Technology Corp.
* Copyright 2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -22,100 +22,163 @@ import java.sql.SQLException;
import java.util.List;
import java.util.logging.Level;
import org.netbeans.api.progress.ProgressHandle;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableFile;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
/**
* An abstract base class for tasks that add or modify the drawables database
* records for multiple drawable files.
* A bulk update task for adding images to the image gallery.
*/
@NbBundle.Messages({
"BulkDrawableFilesTask.committingDb.status=committing image/video database",
"BulkDrawableFilesTask.stopCopy.status=Stopping copy to drawable db task.",
"BulkDrawableFilesTask.errPopulating.errMsg=There was an error populating Image Gallery database."
})
abstract class BulkDrawableFilesTask extends DrawableDbTask {
final class DrawableFileUpdateTask extends DrawableDbTask {
private static final Logger logger = Logger.getLogger(DrawableFileUpdateTask.class.getName());
private static final Logger logger = Logger.getLogger(BulkDrawableFilesTask.class.getName());
private static final String MIMETYPE_CLAUSE = "(mime_type LIKE '" //NON-NLS
+ String.join("' OR mime_type LIKE '", FileTypeUtils.getAllSupportedMimeTypes()) //NON-NLS
+ "') ";
private final String drawableQuery;
private final ImageGalleryController controller;
private final DrawableDB taskDB;
private final SleuthkitCase tskCase;
private final long dataSourceObjId;
//NON-NLS
BulkDrawableFilesTask(long dataSourceObjId, ImageGalleryController controller) {
private final ImageGalleryController controller;
/**
* Construct a new task.
*
* @param controller A handle to the IG controller.
*/
DrawableFileUpdateTask(ImageGalleryController controller) {
this.controller = controller;
this.taskDB = controller.getDrawablesDatabase();
this.tskCase = controller.getCaseDatabase();
this.dataSourceObjId = dataSourceObjId;
drawableQuery = " (data_source_obj_id = " + dataSourceObjId + ") "
}
@Override
public void run() {
for (Long dataSourceObjId : controller.getStaleDataSourceIds()) {
updateFileForDataSource(dataSourceObjId);
}
}
/**
* Gets the drawables database that is part of the model for the controller.
*
* @return The the drawable db object.
*/
private DrawableDB getDrawableDB() {
return controller.getDrawablesDatabase();
}
/**
* Return the sleuthkit case object for the open case.
*
* @return The case db object.
*/
private SleuthkitCase getCaseDB() {
return controller.getCaseDatabase();
}
/**
* Returns a list of files to be processed by the task for the given
* datasource.
*
* @param dataSourceObjId
* @return
* @throws TskCoreException
*/
private List<AbstractFile> getFilesForDataSource(long dataSourceObjId) throws TskCoreException {
List<AbstractFile> list = getCaseDB().findAllFilesWhere(getDrawableQuery(dataSourceObjId));
return list;
}
/**
* Process a single file for the IG drawable db.
*
* @param file The file to process.
* @param tr A valid DrawableTransaction object.
* @param caseDbTransaction A valid caseDBTransaction object.
*
* @throws TskCoreException
*/
void processFile(AbstractFile file, DrawableDB.DrawableTransaction tr, SleuthkitCase.CaseDbTransaction caseDbTransaction) throws TskCoreException {
final boolean known = file.getKnown() == TskData.FileKnown.KNOWN;
if (known) {
getDrawableDB().removeFile(file.getId(), tr); //remove known files
} else {
// NOTE: Files are being processed because they have the right MIME type,
// so we do not need to worry about this calculating them
if (FileTypeUtils.hasDrawableMIMEType(file)) {
getDrawableDB().updateFile(DrawableFile.create(file, true, false), tr, caseDbTransaction);
} //unsupported mimtype => analyzed but shouldn't include
else {
getDrawableDB().removeFile(file.getId(), tr);
}
}
}
/**
* Returns the image query for the given data source.
*
* @param dataSourceObjId
*
* @return SQL query for given data source.
*/
private String getDrawableQuery(long dataSourceObjId) {
return " (data_source_obj_id = " + dataSourceObjId + ") "
+ " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + " AND ( " + MIMETYPE_CLAUSE //NON-NLS
+ " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )" //NON-NLS
+ " ORDER BY parent_path ";
}
/**
* Do any cleanup for this task.
*/
abstract void cleanup();
abstract void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, SleuthkitCase.CaseDbTransaction caseDBTransaction) throws TskCoreException;
/**
* Gets a list of files to process.
*
* @return list of files to process
*
* @throws TskCoreException
*/
List<AbstractFile> getFiles() throws TskCoreException {
return tskCase.findAllFilesWhere(drawableQuery);
}
@Override
@NbBundle.Messages({
"BulkDrawableFilesTask.populatingDb.status=populating analyzed image/video database"
@Messages({
"DrawableFileUpdateTask_populatingDb_status=populating analyzed image/video database",
"DrawableFileUpdateTask_committingDb.status=committing image/video database",
"DrawableFileUpdateTask_stopCopy_status=Stopping copy to drawable db task.",
"DrawableFileUpdateTask_errPopulating_errMsg=There was an error populating Image Gallery database."
})
public void run() {
private void updateFileForDataSource(long dataSourceObjId) {
ProgressHandle progressHandle = getInitialProgressHandle();
progressHandle.start();
updateMessage(Bundle.BulkDrawableFilesTask_populatingDb_status() + " (Data Source " + dataSourceObjId + ")");
updateMessage(Bundle.DrawableFileUpdateTask_populatingDb_status() + " (Data Source " + dataSourceObjId + ")");
DrawableDB.DrawableTransaction drawableDbTransaction = null;
SleuthkitCase.CaseDbTransaction caseDbTransaction = null;
boolean hasFilesWithNoMime = true;
boolean endedEarly = false;
try {
getDrawableDB().buildFileMetaDataCache();
// See if there are any files in the DS w/out a MIME TYPE
hasFilesWithNoMime = controller.hasFilesWithNoMimeType(dataSourceObjId);
//grab all files with detected mime types
final List<AbstractFile> files = getFiles();
final List<AbstractFile> files = getFilesForDataSource(dataSourceObjId);
progressHandle.switchToDeterminate(files.size());
taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS);
getDrawableDB().insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS);
updateProgress(0.0);
int workDone = 0;
// Cycle through all of the files returned and call processFile on each
//do in transaction
drawableDbTransaction = taskDB.beginTransaction();
/*
* We are going to periodically commit the CaseDB transaction and
* sleep so that the user can have Autopsy do other stuff while
* these bulk tasks are ongoing.
*/
int caseDbCounter = 0;
for (final AbstractFile f : files) {
updateMessage(f.getName());
if (caseDbTransaction == null) {
caseDbTransaction = tskCase.beginTransaction();
caseDbTransaction = getCaseDB().beginTransaction();
}
if (drawableDbTransaction == null) {
drawableDbTransaction = getDrawableDB().beginTransaction();
}
if (isCancelled() || Thread.interrupted()) {
logger.log(Level.WARNING, "Task cancelled or interrupted: not all contents may be transfered to drawable database."); //NON-NLS
endedEarly = true;
@ -132,20 +195,23 @@ abstract class BulkDrawableFilesTask extends DrawableDbTask {
if ((++caseDbCounter % 200) == 0) {
caseDbTransaction.commit();
caseDbTransaction = null;
getDrawableDB().commitTransaction(drawableDbTransaction, true);
drawableDbTransaction = null;
Thread.sleep(500); // 1/2 second
}
}
progressHandle.finish();
progressHandle = ProgressHandle.createHandle(Bundle.BulkDrawableFilesTask_committingDb_status());
updateMessage(Bundle.BulkDrawableFilesTask_committingDb_status() + " (Data Source " + dataSourceObjId + ")");
progressHandle = ProgressHandle.createHandle(Bundle.DrawableFileUpdateTask_committingDb_status());
updateMessage(Bundle.DrawableFileUpdateTask_committingDb_status() + " (Data Source " + dataSourceObjId + ")");
updateProgress(1.0);
progressHandle.start();
if (caseDbTransaction != null) {
caseDbTransaction.commit();
caseDbTransaction = null;
}
// pass true so that groupmanager is notified of the changes
taskDB.commitTransaction(drawableDbTransaction, true);
getDrawableDB().commitTransaction(drawableDbTransaction, true);
drawableDbTransaction = null;
} catch (TskCoreException | SQLException | InterruptedException ex) {
if (null != caseDbTransaction) {
@ -157,14 +223,14 @@ abstract class BulkDrawableFilesTask extends DrawableDbTask {
}
if (null != drawableDbTransaction) {
try {
taskDB.rollbackTransaction(drawableDbTransaction);
getDrawableDB().rollbackTransaction(drawableDbTransaction);
} catch (SQLException ex2) {
logger.log(Level.SEVERE, String.format("Failed to roll back drawables db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS
}
}
progressHandle.progress(Bundle.BulkDrawableFilesTask_stopCopy_status());
progressHandle.progress(Bundle.DrawableFileUpdateTask_stopCopy_status());
logger.log(Level.WARNING, "Stopping copy to drawable db task. Failed to transfer all database contents", ex); //NON-NLS
MessageNotifyUtil.Notify.warn(Bundle.BulkDrawableFilesTask_errPopulating_errMsg(), ex.getMessage());
MessageNotifyUtil.Notify.warn(Bundle.DrawableFileUpdateTask_errPopulating_errMsg(), ex.getMessage());
endedEarly = true;
} finally {
progressHandle.finish();
@ -172,15 +238,27 @@ abstract class BulkDrawableFilesTask extends DrawableDbTask {
// if there was cancellation or errors
DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = ((hasFilesWithNoMime == true) || (endedEarly == true)) ? DrawableDB.DrawableDbBuildStatusEnum.REBUILT_STALE : DrawableDB.DrawableDbBuildStatusEnum.COMPLETE;
try {
taskDB.insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus);
getDrawableDB().insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus);
} catch (SQLException ex) {
logger.log(Level.SEVERE, String.format("Error updating datasources table (data source object ID = %d, status = %s)", dataSourceObjId, datasourceDrawableDBStatus.toString(), ex)); //NON-NLS
}
updateMessage("");
updateProgress(-1.0);
getDrawableDB().freeFileMetaDataCache();
// at the end of the task, set the stale status based on the
// cumulative status of all data sources
controller.setModelIsStale(controller.isDataSourcesTableStale());
}
cleanup();
}
abstract ProgressHandle getInitialProgressHandle();
/**
* Returns a ProgressHandle.
*
* @return A new ProgressHandle.
*/
private ProgressHandle getInitialProgressHandle() {
return ProgressHandle.createHandle(Bundle.DrawableFileUpdateTask_populatingDb_status(), this);
}
}

5
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java
View File

@ -486,8 +486,7 @@ public final class ImageGalleryController {
*
*/
public void rebuildDrawablesDb() {
// queue a rebuild task for each stale data source
getStaleDataSourceIds().forEach(dataSourceObjId -> queueDBTask(new AddDrawableFilesTask(dataSourceObjId, this)));
queueDBTask(new DrawableFileUpdateTask(this));
}
/**
@ -670,7 +669,7 @@ public final class ImageGalleryController {
*
* @param bgTask
*/
public synchronized void queueDBTask(DrawableDbTask bgTask) {
public synchronized void queueDBTask(Runnable bgTask) {
if (!dbExecutor.isShutdown()) {
incrementQueueSize();
dbExecutor.submit(bgTask).addListener(this::decrementQueueSize, MoreExecutors.directExecutor());

24
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java
View File

@ -40,6 +40,7 @@ import org.openide.awt.ActionID;
import org.openide.awt.ActionReference;
import org.openide.awt.ActionReferences;
import org.openide.awt.ActionRegistration;
import org.openide.util.Exceptions;
import org.openide.util.HelpCtx;
import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
@ -255,28 +256,7 @@ public final class OpenAction extends CallableSystemAction {
// They don't want to rebuild. Just open the UI as is.
// NOTE: There could be no data....
} else if (answer == ButtonType.YES) {
if (controller.getCase().getCaseType() == Case.CaseType.SINGLE_USER_CASE) {
/*
* For a single-user case, we favor user
* experience, and rebuild the database as soon
* as Image Gallery is enabled for the case.
*
* Turning listening off is necessary in order
* to invoke the listener that will call
* controller.rebuildDB();
*/
controller.setListeningEnabled(false);
controller.setListeningEnabled(true);
} else {
/*
* For a multi-user case, we favor overall
* performance and user experience, not every
* user may want to review images, so we rebuild
* the database only when a user launches Image
* Gallery.
*/
controller.rebuildDrawablesDb();
}
controller.rebuildDrawablesDb();
}
openTopComponent();
return;

5
ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java
View File

@ -66,6 +66,7 @@ import org.sleuthkit.autopsy.imagegallery.ImageGalleryController;
import org.sleuthkit.autopsy.imagegallery.actions.CategorizeGroupAction;
import org.sleuthkit.autopsy.imagegallery.actions.TagGroupAction;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute;
import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB;
import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.DrawableGroup;
import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupSortBy;
import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState;
@ -236,8 +237,8 @@ public class Toolbar extends ToolBar {
}
private void initDataSourceComboBox() {
dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus()));
dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus()));
dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable, new HashMap<>()));
dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable, new HashMap<>()));
dataSourceComboBox.setConverter(new StringConverter<Optional<DataSource>>() {
@Override
public String toString(Optional<DataSource> object) {