diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java index 17fbb40e1b..a2895b4d55 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceBlackboardArtifactTagAction.java @@ -64,14 +64,14 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction() { @Override @@ -91,7 +91,7 @@ public final class ReplaceBlackboardArtifactTagAction extends ReplaceTagAction "# {1} - content obj id", "ReplaceContentTagAction.replaceTag.alert=Unable to replace tag {0} for {1}."}) @Override - protected void replaceTag(ContentTag oldTag, TagName newTagName, String comment) { + protected void replaceTag(ContentTag oldTag, TagName newTagName, String newComment) { new SwingWorker() { @Override @@ -84,7 +84,7 @@ public final class ReplaceContentTagAction extends ReplaceTagAction logger.log(Level.INFO, "Replacing tag {0} with tag {1} for artifact {2}", new Object[]{oldTag.getName().getDisplayName(), newTagName.getDisplayName(), oldTag.getContent().getName()}); //NON-NLS tagsManager.deleteContentTag(oldTag); - tagsManager.addContentTag(oldTag.getContent(), newTagName, comment); + tagsManager.addContentTag(oldTag.getContent(), newTagName, newComment); } catch (TskCoreException tskCoreException) { logger.log(Level.SEVERE, "Error replacing artifact tag", tskCoreException); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java index 03d48f4012..79ca4f4739 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/ReplaceTagAction.java @@ -141,7 +141,7 @@ abstract class ReplaceTagAction extends AbstractAction implements // Add action to replace the tag tagNameItem.addActionListener((ActionEvent event) -> { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, entry.getValue(), ""); + replaceTag(oldtag, entry.getValue(), oldtag.getComment()); }); }); @@ -178,7 +178,7 @@ abstract class ReplaceTagAction extends AbstractAction implements TagName newTagName = GetTagNameDialog.doDialog(); if (null != newTagName) { selectedTags.forEach((oldtag) -> { - replaceTag(oldtag, newTagName, ""); + replaceTag(oldtag, newTagName, oldtag.getComment()); }); } }); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index 44bf52503f..9019ae162d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -129,6 +129,7 @@ public class Case { private static final String EXPORT_FOLDER = "Export"; //NON-NLS private static final String LOG_FOLDER = "Log"; //NON-NLS private static final String REPORTS_FOLDER = "Reports"; //NON-NLS + private static final String CONFIG_FOLDER = "Config"; // NON-NLS private static final String TEMP_FOLDER = "Temp"; //NON-NLS private static final String MODULE_FOLDER = "ModuleOutput"; //NON-NLS private static final String CASE_ACTION_THREAD_NAME = "%s-case-action"; @@ -291,9 +292,10 @@ public class Case { */ ADDING_DATA_SOURCE_FAILED, /** - * A new data source has been added to the current case. The old value - * of the PropertyChangeEvent is null, the new value is the newly-added - * data source (type: Content). Cast the PropertyChangeEvent to + * A new data source or series of data sources have been added to the + * current case. The old value of the PropertyChangeEvent is null, the + * new value is the newly-added data source (type: Content). Cast the + * PropertyChangeEvent to * org.sleuthkit.autopsy.casemodule.events.DataSourceAddedEvent to * access additional event data. */ @@ -1098,6 +1100,10 @@ public class Case { /* * Open the top components (windows within the main application * window). + * + * Note: If the core windows are not opened here, they will be + * opened via the DirectoryTreeTopComponent 'propertyChange()' + * method on a DATA_SOURCE_ADDED event. */ if (newCurrentCase.hasData()) { CoreComponentControl.openCoreWindows(); @@ -1351,6 +1357,16 @@ public class Case { return getOrCreateSubdirectory(REPORTS_FOLDER); } + /** + * Gets the full path to the config directory for this case, creating it if + * it does not exist. + * + * @return The config directory path. + */ + public String getConfigDirectory() { + return getOrCreateSubdirectory(CONFIG_FOLDER); + } + /** * Gets the full path to the module output directory for this case, creating * it if it does not exist. diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java index cb1ec5ddfd..c5c8fb32fb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskPanel.java @@ -257,6 +257,7 @@ final class LocalDiskPanel extends JPanel { pathTextField.setEnabled(copyImageCheckbox.isSelected()); browseButton.setEnabled(copyImageCheckbox.isSelected()); changeDatabasePathCheckbox.setEnabled(copyImageCheckbox.isSelected()); + changeDatabasePathCheckbox.setSelected(copyImageCheckbox.isSelected()); fireUpdateEvent(); }//GEN-LAST:event_copyImageCheckboxActionPerformed diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties index e9c1b5307c..266db3a64c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/Bundle.properties @@ -1,6 +1,5 @@ OptionsCategory_Name_TagNamesOptions=Tags OptionsCategory_TagNames=TagNames -Blackboard.unableToIndexArtifact.error.msg=Unable to index blackboard artifact {0} TagNameDialog.title.text=New Tag TagNameDialog.JOptionPane.tagNameIllegalCharacters.message=Tag name may not contain any of the following symbols\: \\ \: * ? " < > | , ; TagNameDialog.JOptionPane.tagNameIllegalCharacters.title=Invalid character in tag name diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java index fb02e7d2d4..33e8f29237 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java @@ -54,7 +54,7 @@ final class TagNameDefinition implements Comparable { private static final List STANDARD_TAG_DISPLAY_NAMES = Arrays.asList(Bundle.TagNameDefinition_predefTagNames_bookmark_text(), Bundle.TagNameDefinition_predefTagNames_followUp_text(), Bundle.TagNameDefinition_predefTagNames_notableItem_text(), DhsImageCategory.ONE.getDisplayName(), DhsImageCategory.TWO.getDisplayName(), DhsImageCategory.THREE.getDisplayName(), - DhsImageCategory.FOUR.getDisplayName(), DhsImageCategory.FIVE.getDisplayName()); + DhsImageCategory.FOUR.getDisplayName(), DhsImageCategory.FIVE.getDisplayName(), DhsImageCategory.ZERO.getDisplayName()); private final String displayName; private final String description; private final TagName.HTML_COLOR color; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java index dfeb4fbdcd..654ee7161e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/AddEditCentralRepoCommentAction.java @@ -105,7 +105,7 @@ public final class AddEditCentralRepoCommentAction extends AbstractAction { dbManager.updateAttributeInstanceComment(correlationAttribute); } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error adding comment", ex); } } return centralRepoCommentDialog.getComment(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 767e86ebcc..f535d58ccd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -58,6 +58,9 @@ abstract class AbstractSqlEamDb implements EamDb { protected int bulkArtifactsThreshold; private final Map> bulkArtifacts; + // Maximum length for the value column in the instance tables + static final int MAX_VALUE_LENGTH = 256; + // number of instances to keep in bulk queue before doing an insert. // Update Test code if this changes. It's hard coded there. static final int DEFAULT_BULK_THRESHHOLD = 1000; @@ -432,7 +435,8 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamDataSource.getCaseID() == -1) { throw new EamDbException("Case ID is -1"); } else if (eamDataSource.getID() != -1) { - throw new EamDbException("Database ID is already set in object"); + // This data source is already in the central repo + return; } Connection conn = connect(); @@ -550,6 +554,13 @@ abstract class AbstractSqlEamDb implements EamDb { if (eamArtifact.getCorrelationValue() == null) { throw new EamDbException("Correlation value is null"); } + if (eamArtifact.getCorrelationValue().length() >= MAX_VALUE_LENGTH) { + throw new EamDbException("Artifact value too long for central repository." + + "\nCorrelationArtifact ID: " + eamArtifact.getID() + + "\nCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\nCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); + + } Connection conn = connect(); @@ -975,27 +986,50 @@ abstract class AbstractSqlEamDb implements EamDb { if (!eamArtifact.getCorrelationValue().isEmpty()) { if (eamInstance.getCorrelationCase() == null) { - throw new EamDbException("CorrelationAttributeInstance case is null"); + throw new EamDbException("CorrelationAttributeInstance case is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getCorrelationDataSource() == null) { - throw new EamDbException("CorrelationAttributeInstance data source is null"); + throw new EamDbException("CorrelationAttributeInstance data source is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()); } if (eamInstance.getKnownStatus() == null) { - throw new EamDbException("CorrelationAttributeInstance known status is null"); + throw new EamDbException("CorrelationAttributeInstance known status is null for: " + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue() + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID()); } - bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); - bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); - bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); - bulkPs.setString(4, eamArtifact.getCorrelationValue()); - bulkPs.setString(5, eamInstance.getFilePath()); - bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); - if ("".equals(eamInstance.getComment())) { - bulkPs.setNull(7, Types.INTEGER); + if (eamArtifact.getCorrelationValue().length() < MAX_VALUE_LENGTH) { + bulkPs.setString(1, eamInstance.getCorrelationCase().getCaseUUID()); + bulkPs.setString(2, eamInstance.getCorrelationDataSource().getDeviceID()); + bulkPs.setInt(3, eamInstance.getCorrelationDataSource().getCaseID()); + bulkPs.setString(4, eamArtifact.getCorrelationValue()); + bulkPs.setString(5, eamInstance.getFilePath()); + bulkPs.setByte(6, eamInstance.getKnownStatus().getFileKnownValue()); + if ("".equals(eamInstance.getComment())) { + bulkPs.setNull(7, Types.INTEGER); + } else { + bulkPs.setString(7, eamInstance.getComment()); + } + bulkPs.addBatch(); } else { - bulkPs.setString(7, eamInstance.getComment()); + logger.log(Level.WARNING, ("Artifact value too long for central repository." + + "\n\tCorrelationArtifact ID: " + eamArtifact.getID() + + "\n\tCorrelationArtifact Type: " + eamArtifact.getCorrelationType().getDisplayName() + + "\n\tCorrelationArtifact Value: " + eamArtifact.getCorrelationValue()) + + "\n\tEam Instance: " + + "\n\t\tCaseId: " + eamInstance.getCorrelationDataSource().getCaseID() + + "\n\t\tDeviceID: " + eamInstance.getCorrelationDataSource().getDeviceID() + + "\n\t\tFilePath: " + eamInstance.getFilePath()); } - bulkPs.addBatch(); } } } @@ -1740,11 +1774,13 @@ abstract class AbstractSqlEamDb implements EamDb { return 0 < badInstances; } + /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance + * * @throws EamDbException */ @Override diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java index 8cf07a42e9..84dce834a1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationDataSource.java @@ -67,6 +67,7 @@ public class CorrelationDataSource implements Serializable { /** * Create a CorrelationDataSource object from a TSK Content object. + * This will add it to the central repository. * * @param correlationCase the current CorrelationCase used for ensuring * uniqueness of DataSource diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 55958287d5..a3d49d6ba3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -450,10 +450,10 @@ final class CaseEventListener implements PropertyChangeListener { correlationCase = dbManager.newCase(openCase); } if (null == dbManager.getDataSource(correlationCase, deviceId)) { - dbManager.newDataSource(CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource)); + CorrelationDataSource.fromTSKDataSource(correlationCase, newDataSource); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS + LOGGER.log(Level.SEVERE, "Error adding new data source to the central repository", ex); //NON-NLS } catch (TskCoreException | TskDataException ex) { LOGGER.log(Level.SEVERE, "Error getting data source from DATA_SOURCE_ADDED event content.", ex); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 46ebe75541..e3cfad9fe8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -276,12 +276,12 @@ public class IngestEventsListener { } } if (FALSE == eamArtifacts.isEmpty()) { - try { - for (CorrelationAttribute eamArtifact : eamArtifacts) { + for (CorrelationAttribute eamArtifact : eamArtifacts) { + try { dbManager.addArtifact(eamArtifact); + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS } - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); //NON-NLS } } // DATA_ADDED } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java index 356056a1bc..0e94e561b2 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesPanel.java @@ -264,7 +264,6 @@ public final class CommonFilesPanel extends javax.swing.JPanel { viewers.add(table); progress.setDisplayName(Bundle.CommonFilesPanel_search_done_searchProgressDisplay()); DataResultTopComponent.createInstance(tabTitle, pathText, tableFilterWithDescendantsNode, metadata.size(), viewers); - progress.finish(); } catch (InterruptedException ex) { LOGGER.log(Level.SEVERE, "Interrupted while loading Common Files", ex); MessageNotifyUtil.Message.error(Bundle.CommonFilesPanel_search_done_interupted()); @@ -285,6 +284,8 @@ public final class CommonFilesPanel extends javax.swing.JPanel { errorMessage = Bundle.CommonFilesPanel_search_done_exception(); } MessageNotifyUtil.Message.error(errorMessage); + } finally { + progress.finish(); } } }.execute(); diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java index 1727c9322b..5a3f887564 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchAction.java @@ -24,7 +24,6 @@ import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.actions.CallableSystemAction; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.core.Installer; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.Logger; diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java index 71ffe1a21e..cacbdd83f9 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonFilesSearchResultsViewerTable.java @@ -23,23 +23,29 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; /** - * DataResultViewerTable which overrides the default column - * header width calculations. The CommonFilesSearchResultsViewerTable - * presents multiple tiers of data which are not always present and it may not - * make sense to try to calculate the column widths for such tables by sampling - * rows and looking for wide cells. Rather, we just pick some reasonable values. + * DataResultViewerTable which overrides the default column header + * width calculations. The CommonFilesSearchResultsViewerTable + * presents multiple tiers of data which are not always present and it may not + * make sense to try to calculate the column widths for such tables by sampling + * rows and looking for wide cells. Rather, we just pick some reasonable values. */ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { - + private static final Map COLUMN_WIDTHS; private static final long serialVersionUID = 1L; + + private static final Logger LOGGER = Logger.getLogger(CommonFilesSearchResultsViewerTable.class.getName()); + private static final int DEFAULT_WIDTH = 100; + static { Map map = new HashMap<>(); map.put(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), 260); @@ -49,10 +55,10 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { map.put(Bundle.CommonFilesSearchResultsViewerTable_hashsetHitsColLbl(), 100); map.put(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), 130); map.put(Bundle.CommonFilesSearchResultsViewerTable_tagsColLbl1(), 300); - + COLUMN_WIDTHS = Collections.unmodifiableMap(map); } - + @NbBundle.Messages({ "CommonFilesSearchResultsViewerTable.filesColLbl=Files", "CommonFilesSearchResultsViewerTable.instancesColLbl=Instances", @@ -63,18 +69,24 @@ public class CommonFilesSearchResultsViewerTable extends DataResultViewerTable { "CommonFilesSearchResultsViewerTable.tagsColLbl1=Tags" }) @Override - protected void setColumnWidths(){ + protected void setColumnWidths() { TableColumnModel model = this.getColumnModel(); - + Enumeration columnsEnumerator = model.getColumns(); - while(columnsEnumerator.hasMoreElements()){ - + while (columnsEnumerator.hasMoreElements()) { + TableColumn column = columnsEnumerator.nextElement(); - + final String headerValue = column.getHeaderValue().toString(); - final Integer get = COLUMN_WIDTHS.get(headerValue); - - column.setPreferredWidth(get); + + final Integer defaultWidth = COLUMN_WIDTHS.get(headerValue); + + if(defaultWidth == null){ + column.setPreferredWidth(DEFAULT_WIDTH); + LOGGER.log(Level.SEVERE, String.format("Tried to set width on a column not supported by the CommonFilesSearchResultsViewerTable: %s", headerValue)); + } else { + column.setPreferredWidth(defaultWidth); + } } } } diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index b7a5498bf8..40a667e3d7 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -56,7 +56,6 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; -import org.sleuthkit.autopsy.coreutils.TimeStampUtils; /** * A file content viewer for SQLite database files. @@ -289,28 +288,22 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { fileChooser.setAcceptAllFileFilterUsed(true); fileChooser.setFileFilter(csvFilter); fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); + String defaultFileName = (String) this.tablesDropdownList.getSelectedItem(); + fileChooser.setSelectedFile(new File(defaultFileName)); int choice = fileChooser.showSaveDialog((Component) evt.getSource()); //TODO if (JFileChooser.APPROVE_OPTION == choice) { - boolean overwrite = false; File file = fileChooser.getSelectedFile(); - if (file == null) { - JOptionPane.showMessageDialog(this, - Bundle.SQLiteViewer_csvExport_fileName_empty(), - Bundle.SQLiteViewer_csvExport_title(), - JOptionPane.WARNING_MESSAGE); - return; - } else if (file.exists() && FilenameUtils.getExtension(file.getName()).equalsIgnoreCase("csv")) { + if (file.exists() && FilenameUtils.getExtension(file.getName()).equalsIgnoreCase("csv")) { if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(this, Bundle.SQLiteViewer_csvExport_confirm_msg(), Bundle.SQLiteViewer_csvExport_title(), JOptionPane.YES_NO_OPTION)) { - overwrite = true; } else { return; } } - exportTableToCsv(file, overwrite); + exportTableToCsv(file); } }//GEN-LAST:event_exportCsvButtonActionPerformed @@ -567,9 +560,8 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { "SQLiteViewer.exportTableToCsv.FileName=File name: ", "SQLiteViewer.exportTableToCsv.TableName=Table name: " }) - private void exportTableToCsv(File file, boolean overwrite) { + private void exportTableToCsv(File file) { String tableName = (String) this.tablesDropdownList.getSelectedItem(); - String csvFileSuffix = "_" + tableName + "_" + TimeStampUtils.createTimeStamp() + ".csv"; try ( Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery("SELECT * FROM " + tableName)) { @@ -578,42 +570,42 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { if (Objects.isNull(currentTableRows) || currentTableRows.isEmpty()) { logger.log(Level.INFO, String.format("The table %s is empty. (objId=%d)", tableName, sqliteDbFile.getId())); //NON-NLS } else { - String fileName = file.getName(); File csvFile; - if (overwrite) { + String fileName = file.getName(); + if (FilenameUtils.getExtension(fileName).equalsIgnoreCase("csv")) { csvFile = file; - } else if (FilenameUtils.getExtension(fileName).equalsIgnoreCase("csv")) { - csvFile = new File(file.getParentFile(), FilenameUtils.removeExtension(fileName) + csvFileSuffix); } else { - csvFile = new File(file.toString() + csvFileSuffix); + csvFile = new File(file.toString() + ".csv"); } - FileOutputStream out = new FileOutputStream(csvFile, false); - out.write((Bundle.SQLiteViewer_exportTableToCsv_FileName() + csvFile.getName() + "\n").getBytes()); - out.write((Bundle.SQLiteViewer_exportTableToCsv_TableName() + tableName + "\n").getBytes()); - // Set up the column names - Map row = currentTableRows.get(0); - StringBuffer header = new StringBuffer(); - for (Map.Entry col : row.entrySet()) { - String colName = col.getKey(); - if (header.length() > 0) { - header.append(',').append(colName); - } else { - header.append(colName); - } - } - out.write(header.append('\n').toString().getBytes()); + try (FileOutputStream out = new FileOutputStream(csvFile, false)) { - for (Map maps : currentTableRows) { - StringBuffer valueLine = new StringBuffer(); - maps.values().forEach((value) -> { - if (valueLine.length() > 0) { - valueLine.append(',').append(value.toString()); + out.write((Bundle.SQLiteViewer_exportTableToCsv_FileName() + csvFile.getName() + "\n").getBytes()); + out.write((Bundle.SQLiteViewer_exportTableToCsv_TableName() + tableName + "\n").getBytes()); + // Set up the column names + Map row = currentTableRows.get(0); + StringBuffer header = new StringBuffer(); + for (Map.Entry col : row.entrySet()) { + String colName = col.getKey(); + if (header.length() > 0) { + header.append(',').append(colName); } else { - valueLine.append(value.toString()); + header.append(colName); } - }); - out.write(valueLine.append('\n').toString().getBytes()); + } + out.write(header.append('\n').toString().getBytes()); + + for (Map maps : currentTableRows) { + StringBuffer valueLine = new StringBuffer(); + maps.values().forEach((value) -> { + if (valueLine.length() > 0) { + valueLine.append(',').append(value.toString()); + } else { + valueLine.append(value.toString()); + } + }); + out.write(valueLine.append('\n').toString().getBytes()); + } } } } catch (SQLException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java index bf26dc8295..06ad27962a 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/CoreComponentControl.java @@ -88,7 +88,21 @@ final public class CoreComponentControl { TopComponent directoryTree = null; TopComponent favorites = null; final WindowManager windowManager = WindowManager.getDefault(); + + // Set the UI selections to null before closing the top components. + // Otherwise it may experience errors trying to load data for the closed case. for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { + if(tc instanceof DataContent) { + ((DataContent) tc).setNode(null); + } else if(tc instanceof DataResult) { + ((DataResult) tc).setNode(null); + } + } + } + + for (Mode mode : windowManager.getModes()) { + for (TopComponent tc : windowManager.getOpenedTopComponents(mode)) { String tcName = tc.getName(); @@ -105,7 +119,7 @@ final public class CoreComponentControl { } } } - + if (directoryTree != null) { directoryTree.close(); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 235f00cb1b..d6fb9afc04 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -312,7 +312,11 @@ public class BlackboardArtifactNode extends AbstractContentNode(NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.fileSize.name"), + NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.fileSize.displayName"), + NO_DESCR, + size)); + sheetSet.put(new NodeProperty<>( + NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.path.name"), + NbBundle.getMessage(BlackboardArtifactNode.class, "BlackboardArtifactNode.createSheet.path.displayName"), + NO_DESCR, + path)); + } addTagProperty(sheetSet); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java b/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java deleted file mode 100644 index 0c83a12f61..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datamodel/CommonFileChildNodeLoading.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * - * Autopsy Forensic Browser - * - * Copyright 2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datamodel; - -import java.util.LinkedHashMap; -import java.util.Map; -import org.openide.nodes.Children; -import org.openide.nodes.Sheet; -import org.openide.util.NbBundle; - -/** - * A dummy node used by the child factory to display while children are being - * generated - */ -public class CommonFileChildNodeLoading extends DisplayableItemNode { - - public CommonFileChildNodeLoading(Children children) { - super(children); - } - - @Override - public T accept(DisplayableItemNodeVisitor visitor) { - return visitor.visit(this); - } - - @Override - public boolean isLeafTypeNode() { - return true; - } - - @Override - public String getItemType() { - return getClass().getName(); - } - - @Override - protected Sheet createSheet() { - Sheet sheet = new Sheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - - Map map = new LinkedHashMap<>(); - map.put(CommonFileChildLoadingPropertyType.File.toString(), "Loading..."); - - final String NO_DESCR = Bundle.AbstractFsContentNode_noDesc_text(); - for (CommonFileChildLoadingPropertyType propType : CommonFileChildLoadingPropertyType.values()) { - final String propString = propType.toString(); - sheetSet.put(new NodeProperty<>(propString, propString, NO_DESCR, map.get(propString))); - } - - return sheet; - } - - /** - * Represents the sole column for the 'dummy' loading node. - */ - @NbBundle.Messages({ - "CommonFileChildLoadingPropertyType.fileColLbl=File" - }) - public enum CommonFileChildLoadingPropertyType { - - File(Bundle.CommonFileChildLoadingPropertyType_fileColLbl()); - - final private String displayString; - - private CommonFileChildLoadingPropertyType(String displayString) { - this.displayString = displayString; - } - - @Override - public String toString() { - return displayString; - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java b/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java index 0e9bd1fcba..9cd13a2f4a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java @@ -120,8 +120,6 @@ public interface DisplayableItemNodeVisitor { T visit(CommonFilesNode cfn); T visit(FileInstanceNode fin); - - T visit(CommonFileChildNodeLoading cfcnl); T visit(InstanceCountNode icn); @@ -213,11 +211,6 @@ public interface DisplayableItemNodeVisitor { return defaultVisit(icn); } - @Override - public T visit(CommonFileChildNodeLoading cfcnl) { - return defaultVisit(cfcnl); - } - @Override public T visit(DirectoryNode dn) { return defaultVisit(dn); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index 6a304d7b86..aa1d0a04ec 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -249,7 +249,7 @@ public final class FileTypes implements AutopsyVisitableItem { if (typesRoot.showCounts) { //only show "(counting...)" the first time, otherwise it is distracting. setDisplayName(getDisplayNameBase() + ((childCount < 0) ? Bundle.FileTypes_bgCounting_placeholder() - : ("(" + childCount + ")"))); //NON-NLS + : (" (" + childCount + ")"))); //NON-NLS new SwingWorker() { @Override protected Long doInBackground() throws Exception { diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties index 0e7f3cf315..926c4b4626 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/directorytree/Bundle.properties @@ -120,3 +120,8 @@ AddExternalViewerRulePanel.exePathTextField.text= AddExternalViewerRulePanel.exePathLabel.text=Path of the program to use for files with this type or extension AddExternalViewerRulePanel.extRadioButton.text=Extension DirectoryTreeTopComponent.groupByDatasourceCheckBox.text=Group by Data Source +GroupDataSourcesDialog.dataSourceCountLabel.text=jLabel1 +GroupDataSourcesDialog.queryLabel.text=Would you like to group by data source for faster loading? +GroupDataSourcesDialog.yesButton.text=Yes +GroupDataSourcesDialog.noButton.text=No +GroupDataSourcesDialog.title=Group by Data Source? \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 7db201f00b..95ba2f110c 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -24,6 +24,11 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.beans.PropertyVetoException; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -35,6 +40,7 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.prefs.PreferenceChangeEvent; import java.util.prefs.PreferenceChangeListener; +import java.util.Properties; import javax.swing.Action; import javax.swing.SwingUtilities; import javax.swing.SwingWorker; @@ -62,6 +68,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataExplorer; import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent; import org.sleuthkit.autopsy.corecomponents.TableFilterNode; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.datamodel.ArtifactNodeSelectionInfo; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; import org.sleuthkit.autopsy.datamodel.CreditCards; @@ -103,6 +110,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private static final Logger LOGGER = Logger.getLogger(DirectoryTreeTopComponent.class.getName()); private AutopsyTreeChildrenFactory autopsyTreeChildrenFactory; private Children autopsyTreeChildren; + private static final long DEFAULT_DATASOURCE_GROUPING_THRESHOLD = 5; // Threshold for prompting the user about grouping by data source + private static final String GROUPING_THRESHOLD_NAME = "GroupDataSourceThreshold"; + private static final String SETTINGS_FILE = "CasePreferences.properties"; //NON-NLS /** * the constructor @@ -125,7 +135,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat this.forwardList = new LinkedList<>(); backButton.setEnabled(false); forwardButton.setEnabled(false); - + groupByDatasourceCheckBox.setSelected(UserPreferences.groupItemsInTreeByDatasource()); } @@ -149,7 +159,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } }); - + Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.DATA_SOURCE_ADDED), this); this.em.addPropertyChangeListener(this); IngestManager.getInstance().addIngestJobEventListener(this); @@ -370,6 +380,51 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return TopComponent.PERSISTENCE_NEVER; } + /** + * Ask the user if they want to group by data source when opening a large + * case. + * + * @param currentCase + * @param dataSourceCount + */ + private void promptForDataSourceGrouping(Case currentCase, int dataSourceCount) { + Path settingsFile = Paths.get(currentCase.getConfigDirectory(), SETTINGS_FILE); //NON-NLS + if (settingsFile.toFile().exists()) { + // Read the setting + try (InputStream inputStream = Files.newInputStream(settingsFile)) { + Properties props = new Properties(); + props.load(inputStream); + if (props.getProperty("groupByDataSource", "false").equals("true")) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error reading settings file", ex); + } + } else { + GroupDataSourcesDialog dialog = new GroupDataSourcesDialog(dataSourceCount); + dialog.display(); + if (dialog.groupByDataSourceSelected()) { + UserPreferences.setGroupItemsInTreeByDatasource(true); + groupByDatasourceCheckBox.setSelected(true); + } + + // Save the response + Properties props = new Properties(); + if (dialog.groupByDataSourceSelected()) { + props.setProperty("groupByDataSource", "true"); + } else { + props.setProperty("groupByDataSource", "false"); + } + + try (OutputStream fos = Files.newOutputStream(settingsFile)) { + props.store(fos, ""); //NON-NLS + } catch (IOException ex) { + LOGGER.log(Level.SEVERE, "Error writing settings file", ex); + } + } + } + /** * Called only when top component was closed on all workspaces before and * now is opened for the first time on some workspace. The intent is to @@ -377,6 +432,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * existing workspaces. Subclasses will usually perform initializing tasks * here. */ + @NbBundle.Messages({"# {0} - dataSourceCount", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.text=This case contains {0} data sources. Would you like to group by data source for faster loading?", + "DirectoryTreeTopComponent.componentOpened.groupDataSources.title=Group by data source?"}) @Override public void componentOpened() { // change the cursor to "waiting cursor" for this operation @@ -392,9 +450,34 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat if (null == currentCase || currentCase.hasData() == false) { getTree().setRootVisible(false); // hide the root } else { + // If the case contains a lot of data sources, and they aren't already grouping + // by data source, give the user the option to do so before loading the tree. + if (RuntimeProperties.runningWithGUI()) { + long threshold = DEFAULT_DATASOURCE_GROUPING_THRESHOLD; + if (ModuleSettings.settingExists(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)) { + try { + threshold = Long.parseLong(ModuleSettings.getConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME)); + } catch (NumberFormatException ex) { + LOGGER.log(Level.SEVERE, "Group data sources threshold is not a number", ex); + } + } else { + ModuleSettings.setConfigSetting(ModuleSettings.MAIN_SETTINGS, GROUPING_THRESHOLD_NAME, String.valueOf(threshold)); + } + + try { + int dataSourceCount = currentCase.getDataSources().size(); + if (!UserPreferences.groupItemsInTreeByDatasource() + && dataSourceCount > threshold) { + promptForDataSourceGrouping(currentCase, dataSourceCount); + } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error loading data sources", ex); + } + } + // if there's at least one image, load the image and open the top componen autopsyTreeChildrenFactory = new AutopsyTreeChildrenFactory(); - autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); + autopsyTreeChildren = Children.create(autopsyTreeChildrenFactory, true); Node root = new AbstractNode(autopsyTreeChildren) { //JIRA-2807: What is the point of these overrides? /** @@ -453,9 +536,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat tree.collapseNode(views); } /* - * JIRA-2806: What is this supposed to do? Right now it selects - * the data sources node, but the comment seems to indicate - * it is supposed to select the first datasource. + * JIRA-2806: What is this supposed to do? Right now it + * selects the data sources node, but the comment seems to + * indicate it is supposed to select the first datasource. */ // select the first image node, if there is one // (this has to happen after dataResult is opened, because the event @@ -484,7 +567,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // dataResult active) try { Node[] selections = get(); - if (selections != null && selections.length > 0){ + if (selections != null && selections.length > 0) { em.setSelectedNodes(selections); } } catch (PropertyVetoException ex) { @@ -592,15 +675,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } /** - * The "listener" that listens to any changes made in the Case.java class. - * It will do something based on the changes in the Case.java class. + * The "listener" that monitors changes made in the Case class. This serves + * the purpose of keeping the UI in sync with the data as it changes. * - * @param evt the property change event + * @param event The property change event. */ @Override - public void propertyChange(PropertyChangeEvent evt) { + public void propertyChange(PropertyChangeEvent event) { if (RuntimeProperties.runningWithGUI()) { - String changed = evt.getPropertyName(); + String changed = event.getPropertyName(); if (changed.equals(Case.Events.CURRENT_CASE.toString())) { // changed current case // When a case is closed, the old value of this property is the // closed Case object and the new value is null. When a case is @@ -610,15 +693,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // opened events instead of property change events would be a better // solution. Either way, more probably needs to be done to clean up // data model objects when a case is closed. - if (evt.getOldValue() != null && evt.getNewValue() == null) { + if (event.getOldValue() != null && event.getNewValue() == null) { // The current case has been closed. Reset the ExplorerManager. SwingUtilities.invokeLater(() -> { Node emptyNode = new AbstractNode(Children.LEAF); em.setRootContext(emptyNode); }); - } else if (evt.getNewValue() != null) { + } else if (event.getNewValue() != null) { // A new case has been opened. Reset the ExplorerManager. - Case newCase = (Case) evt.getNewValue(); + Case newCase = (Case) event.getNewValue(); final String newCaseName = newCase.getName(); SwingUtilities.invokeLater(() -> { em.getRootContext().setName(newCaseName); @@ -642,20 +725,27 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat * already closed. */ try { - Case currentCase = Case.getCurrentCaseThrows(); - // We only need to trigger openCoreWindows() when the - // first data source is added. - if (currentCase.getDataSources().size() == 1) { + Case.getCurrentCaseThrows(); + /* + * In case the Case 'updateGUIForCaseOpened()' method hasn't + * already done so, open the tree and all other core + * windows. + * + * TODO: (JIRA-4053) DirectoryTreeTopComponent should not be + * responsible for opening core windows. Consider moving + * this elsewhere. + */ + if (!this.isOpened()) { SwingUtilities.invokeLater(CoreComponentControl::openCoreWindows); } - } catch (NoCurrentCaseException | TskCoreException notUsed) { + } catch (NoCurrentCaseException notUsed) { /** * Case is closed, do nothing. */ } } // change in node selection else if (changed.equals(ExplorerManager.PROP_SELECTED_NODES)) { - respondSelection((Node[]) evt.getOldValue(), (Node[]) evt.getNewValue()); + respondSelection((Node[]) event.getOldValue(), (Node[]) event.getNewValue()); } else if (changed.equals(IngestManager.IngestModuleEvent.DATA_ADDED.toString())) { // nothing to do here. // all nodes should be listening for these events and update accordingly. @@ -800,7 +890,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat /** * Rebuilds the autopsy tree. - * + * * Does nothing if there is no open case. */ private void rebuildTree() { @@ -813,9 +903,9 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return; } if (null == currentCase || currentCase.hasData() == false) { - return; + return; } - + // refresh all children of the root. autopsyTreeChildrenFactory.refreshChildren(); @@ -842,14 +932,14 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } }.execute(); } - + /** * Selects the first node in the tree. - * + * */ - private void selectFirstChildNode () { + private void selectFirstChildNode() { Children rootChildren = em.getRootContext().getChildren(); - + if (rootChildren.getNodesCount() > 0) { Node firstNode = rootChildren.getNodeAt(0); if (firstNode != null) { @@ -858,6 +948,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } } } + /** * Set the selected node using a path to a previously selected node. * diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form new file mode 100644 index 0000000000..e31c97d4ba --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.form @@ -0,0 +1,101 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java new file mode 100644 index 0000000000..7641655d57 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/GroupDataSourcesDialog.java @@ -0,0 +1,147 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.directorytree; + +import javax.swing.JFrame; +import org.openide.util.NbBundle; +import org.openide.windows.WindowManager; + +/** + * + */ +@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives +final class GroupDataSourcesDialog extends javax.swing.JDialog { + + boolean shouldGroupByDataSource = false; + + /** + * Creates new form GroupDataSourcesDialog + */ + @NbBundle.Messages({"# {0} - dataSourceCount", + "GroupDataSourcesDialog.groupDataSources.text=This case contains {0} data sources."}) + GroupDataSourcesDialog(int dataSourceCount) { + super((JFrame) WindowManager.getDefault().getMainWindow()); + initComponents(); + dataSourceCountLabel.setText(Bundle.GroupDataSourcesDialog_groupDataSources_text(dataSourceCount)); + } + + /** + * Display the dialog. + */ + void display() { + setModal(true); + setSize(getPreferredSize()); + setLocationRelativeTo(this.getParent()); + setAlwaysOnTop(false); + pack(); + setVisible(true); + } + + boolean groupByDataSourceSelected() { + return shouldGroupByDataSource; + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + dataSourceCountLabel = new javax.swing.JLabel(); + queryLabel = new javax.swing.JLabel(); + yesButton = new javax.swing.JButton(); + noButton = new javax.swing.JButton(); + + setDefaultCloseOperation(javax.swing.WindowConstants.DISPOSE_ON_CLOSE); + setTitle(org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.title")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(dataSourceCountLabel, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.dataSourceCountLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(queryLabel, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.queryLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(yesButton, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.yesButton.text")); // NOI18N + yesButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + yesButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(noButton, org.openide.util.NbBundle.getMessage(GroupDataSourcesDialog.class, "GroupDataSourcesDialog.noButton.text")); // NOI18N + noButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + noButtonActionPerformed(evt); + } + }); + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); + getContentPane().setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(queryLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createSequentialGroup() + .addComponent(dataSourceCountLabel) + .addGap(0, 0, Short.MAX_VALUE)) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addGap(0, 0, Short.MAX_VALUE) + .addComponent(yesButton, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(noButton, javax.swing.GroupLayout.PREFERRED_SIZE, 76, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addContainerGap()) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(dataSourceCountLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(queryLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(yesButton) + .addComponent(noButton)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + ); + + pack(); + }// //GEN-END:initComponents + + private void yesButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_yesButtonActionPerformed + shouldGroupByDataSource = true; + dispose(); + }//GEN-LAST:event_yesButtonActionPerformed + + private void noButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_noButtonActionPerformed + shouldGroupByDataSource = false; + dispose(); + }//GEN-LAST:event_noButtonActionPerformed + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JLabel dataSourceCountLabel; + private javax.swing.JButton noButton; + private javax.swing.JLabel queryLabel; + private javax.swing.JButton yesButton; + // End of variables declaration//GEN-END:variables +} diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java index 96d7973d8a..1ed54df58d 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/DataSourcePanel.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.filesearch; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.io.File; import java.util.ArrayList; import java.util.Collections; @@ -29,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.event.ListSelectionEvent; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -47,38 +44,28 @@ public class DataSourcePanel extends javax.swing.JPanel { private static final Logger logger = Logger.getLogger(DataSourcePanel.class.getName()); private static final long serialVersionUID = 1L; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); /** * Creates new form DataSourcePanel */ public DataSourcePanel() { initComponents(); - this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { - firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); - }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - if (evt.getSource() instanceof JList) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(toolTipList.get(index)); - } - } - } - }); + if (this.dataSourceList.getModel().getSize() > 1) { + this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + }); + } else { + /* + * Disable data source filtering since there aren't multiple data + * sources to choose from. + */ + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceList.setEnabled(false); + } } /** - * Get dataSourceMap with object id and data source display name. Add the data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ @@ -93,8 +80,7 @@ public class DataSourcePanel extends javax.swing.JPanel { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); - dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); + dataSourceMap.put(ds.getId(), displayName); dsList.add(displayName); } } catch (NoCurrentCaseException ex) { @@ -107,7 +93,8 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Get a set of data source object ids that are selected. - * @return A set of selected object ids. + * + * @return A set of selected object ids. */ Set getDataSourcesSelected() { Set dataSourceObjIdSet = new HashSet<>(); @@ -124,6 +111,7 @@ public class DataSourcePanel extends javax.swing.JPanel { /** * Is dataSourceCheckBox selected + * * @return true if the dataSoureCheckBox is selected */ boolean isSelected() { @@ -131,7 +119,8 @@ public class DataSourcePanel extends javax.swing.JPanel { } /** - * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is checked. + * Enable the dsList and dataSourceNoteLable if the dataSourceCheckBox is + * checked. */ final void setComponentsEnabled() { boolean enabled = this.isSelected(); diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index b69d84b629..50dfe318c3 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -556,17 +556,29 @@ class ReportHTML implements TableReportModule { } /** - * Add a row to the current table. + * Add a row to the current table, escaping the text to be contained in the + * row. * * @param row values for each cell in the row */ @Override public void addRow(List row) { + addRow(row, true); + } + + /** + * Add a row to the current table. + * + * @param row values for each cell in the row + * @param escapeText whether or not the text of the row should be escaped, + * true for escaped, false for not escaped + */ + private void addRow(List row, boolean escapeText) { StringBuilder builder = new StringBuilder(); builder.append("\t\n"); //NON-NLS for (String cell : row) { - String escapeHTMLCell = EscapeUtil.escapeHtml(cell); - builder.append("\t\t").append(escapeHTMLCell).append("\n"); //NON-NLS + String cellText = escapeText ? EscapeUtil.escapeHtml(cell) : cell; + builder.append("\t\t").append(cellText).append("\n"); //NON-NLS } builder.append("\t\n"); //NON-NLS rowCount++; @@ -593,7 +605,7 @@ class ReportHTML implements TableReportModule { public void addRowWithTaggedContentHyperlink(List row, ContentTag contentTag) { Content content = contentTag.getContent(); if (content instanceof AbstractFile == false) { - addRow(row); + addRow(row, true); return; } AbstractFile file = (AbstractFile) content; @@ -647,7 +659,7 @@ class ReportHTML implements TableReportModule { int pages = 1; for (Content content : images) { if (currentRow.size() == THUMBNAIL_COLUMNS) { - addRow(currentRow); + addRow(currentRow, false); currentRow.clear(); } @@ -727,7 +739,7 @@ class ReportHTML implements TableReportModule { // Finish out the row. currentRow.add(""); } - addRow(currentRow); + addRow(currentRow, false); } // manually set rowCount to be the total number of images. @@ -1094,8 +1106,8 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS summary.append(writeSummaryCaseDetails()); summary.append(writeSummaryImageInfo()); - summary.append(writeSummarySoftwareInfo(skCase,ingestJobs)); - summary.append(writeSummaryIngestHistoryInfo(skCase,ingestJobs)); + summary.append(writeSummarySoftwareInfo(skCase, ingestJobs)); + summary.append(writeSummaryIngestHistoryInfo(skCase, ingestJobs)); if (generatorLogoSet) { summary.append("
\n"); //NON-NLS summary.append("\n"); //NON-NLS @@ -1126,14 +1138,13 @@ class ReportHTML implements TableReportModule { } } } - + /** * Write the case details section of the summary for this report. - * + * * @return StringBuilder updated html report with case details */ - - private StringBuilder writeSummaryCaseDetails(){ + private StringBuilder writeSummaryCaseDetails() { StringBuilder summary = new StringBuilder(); String caseName = currentCase.getDisplayName(); String caseNumber = currentCase.getNumber(); @@ -1146,40 +1157,39 @@ class ReportHTML implements TableReportModule { imagecount = 0; } summary.append("
\n"); //NON-NLS - if (agencyLogoSet) { - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - } - final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("\n"); //NON-NLS - summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS - .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS - .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS - .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS - .append(!examiner.isEmpty() ? examiner : NbBundle - .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) - .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS - .append("").append(imagecount).append("
\n"); //NON-NLS + if (agencyLogoSet) { + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - summary.append("
\n"); //NON-NLS - return summary; + } + final String align = agencyLogoSet ? "right" : "left"; //NON-NLS NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("\n"); //NON-NLS + summary.append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseName")) //NON-NLS + .append("").append(caseName).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.caseNum")) //NON-NLS + .append("").append(!caseNumber.isEmpty() ? caseNumber : NbBundle //NON-NLS + .getMessage(this.getClass(), "ReportHTML.writeSum.noCaseNum")).append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.examiner")).append("") //NON-NLS + .append(!examiner.isEmpty() ? examiner : NbBundle + .getMessage(this.getClass(), "ReportHTML.writeSum.noExaminer")) + .append("
").append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.numImages")) //NON-NLS + .append("").append(imagecount).append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + summary.append("
\n"); //NON-NLS + return summary; } - + /** * Write the Image Information section of the summary for this report. - * + * * @return StringBuilder updated html report with Image Information */ - private StringBuilder writeSummaryImageInfo() { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.imageInfoHeading")); @@ -1208,13 +1218,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the software information section of the summary for this report. - * + * * @return StringBuilder updated html report with software information */ - private StringBuilder writeSummarySoftwareInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); summary.append(NbBundle.getMessage(this.getClass(), "ReportHTML.writeSum.softwareInfoHeading")); @@ -1244,13 +1253,12 @@ class ReportHTML implements TableReportModule { summary.append("
\n"); //NON-NLS return summary; } - + /** * Write the Ingest History section of the summary for this report. - * + * * @return StringBuilder updated html report with ingest history */ - private StringBuilder writeSummaryIngestHistoryInfo(SleuthkitCase skCase, List ingestJobs) { StringBuilder summary = new StringBuilder(); try { @@ -1304,4 +1312,4 @@ class ReportHTML implements TableReportModule { + thumbFile.getName(); } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java index 9d5d41c836..f87851fe87 100644 --- a/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java +++ b/Core/src/org/sleuthkit/autopsy/report/TableReportGenerator.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.ContentUtils; +import static org.sleuthkit.autopsy.casemodule.services.TagsManager.getNotableTagLabel; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -53,6 +54,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.Type; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TagName; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -538,6 +540,65 @@ class TableReportGenerator { logger.log(Level.SEVERE, "Exception while getting open case: ", ex); //NON-NLS return; } + + // Get a list of all selected tag IDs + String tagIDList = ""; + if( ! tagNamesFilter.isEmpty()) { + try { + Map tagNamesMap = Case.getCurrentCaseThrows().getServices().getTagsManager().getDisplayNamesToTagNamesMap(); + for(String tagDisplayName : tagNamesFilter) { + if(tagNamesMap.containsKey(tagDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(tagDisplayName).getId(); + } else { + // If the tag name ends with "(Notable)", try stripping that off + if(tagDisplayName.endsWith(getNotableTagLabel())) { + String editedDisplayName = tagDisplayName.substring(0, tagDisplayName.length() - getNotableTagLabel().length()); + if(tagNamesMap.containsKey(editedDisplayName)) { + if (! tagIDList.isEmpty()) { + tagIDList += ","; + } + tagIDList += tagNamesMap.get(editedDisplayName).getId(); + } + } + } + } + } catch (NoCurrentCaseException | TskCoreException ex) { + logger.log(Level.SEVERE, "Exception while getting tag info - proceeding without tag filter: ", ex); //NON-NLS + tagIDList = ""; + } + } + + // Check if there are any ad-hoc results + String adHocCountQuery = "SELECT COUNT(*) FROM " + //NON-NLS + "(SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 ";//NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + adHocCountQuery += "WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + if (!tagIDList.isEmpty()) { + adHocCountQuery += " AND (art.artifact_id = tag.artifact_id) AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + adHocCountQuery += "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ")) "; //NON-NLS + + int adHocCount = 0; + try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(adHocCountQuery)) { + ResultSet adHocCountResultSet = dbQuery.getResultSet(); + if (adHocCountResultSet.next()) { + adHocCount = adHocCountResultSet.getInt(1); //NON-NLS + } else { + throw new TskCoreException("Error counting ad hoc keywords"); + } + } catch (TskCoreException | SQLException ex) { + errorList.add(NbBundle.getMessage(this.getClass(), "ReportGenerator.errList.failedQueryKWLists")); + logger.log(Level.SEVERE, "Failed to count ad hoc searches with query " + adHocCountQuery, ex); //NON-NLS + return; + } + + // Create the query to get the keyword list names if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST"; //NON-NLS } else { @@ -546,16 +607,25 @@ class TableReportGenerator { String keywordListQuery = "SELECT att.value_text AS list " + //NON-NLS - "FROM blackboard_attributes AS att, blackboard_artifacts AS art " - + //NON-NLS - "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + "FROM blackboard_attributes AS att, blackboard_artifacts AS art "; // NON-NLS + if(! tagIDList.isEmpty()) { + keywordListQuery += ", blackboard_artifact_tags as tag "; //NON-NLS + } + keywordListQuery += "WHERE att.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + " " + //NON-NLS "AND art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + " " + //NON-NLS - "AND att.artifact_id = art.artifact_id " - + //NON-NLS - "GROUP BY list " + orderByClause; //NON-NLS + "AND att.artifact_id = art.artifact_id "; + if (! tagIDList.isEmpty()) { + keywordListQuery += "AND (art.artifact_id = tag.artifact_id) " + //NON-NLS + "AND (tag.tag_name_id IN (" + tagIDList + ")) "; //NON-NLS + } + if (adHocCount > 0) { + keywordListQuery += " UNION SELECT \"\" AS list "; + } + keywordListQuery += "GROUP BY list " + orderByClause; //NON-NLS + // Make the table of contents links for each list type try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordListQuery)) { ResultSet listsRs = dbQuery.getResultSet(); List lists = new ArrayList<>(); @@ -579,6 +649,7 @@ class TableReportGenerator { return; } + // Query for keywords, grouped by list if (openCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { orderByClause = "ORDER BY convert_to(att3.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS + "convert_to(att1.value_text, 'SQL_ASCII') ASC NULLS FIRST, " //NON-NLS @@ -588,9 +659,10 @@ class TableReportGenerator { } else { orderByClause = "ORDER BY list ASC, keyword ASC, parent_path ASC, name ASC, preview ASC"; //NON-NLS } - // Query for keywords, grouped by list - String keywordsQuery - = "SELECT art.artifact_id, art.obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + + // Query for keywords that are part of a list + String keywordListsQuery + = "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, att3.value_text AS list, f.name AS name, f.parent_path AS parent_path " + //NON-NLS "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, blackboard_attributes AS att3, tsk_files AS f " + //NON-NLS @@ -608,9 +680,24 @@ class TableReportGenerator { + //NON-NLS "AND (att3.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + ") " + //NON-NLS - "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " - + //NON-NLS - orderByClause; //NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; + + // Query for keywords that are not part of a list + String keywordAdHocQuery = + "SELECT art.artifact_id AS artifact_id, art.obj_id AS obj_id, att1.value_text AS keyword, att2.value_text AS preview, \"\" AS list, f.name AS name, f.parent_path AS parent_path " + // NON-NLS + "FROM blackboard_artifacts AS art, blackboard_attributes AS att1, blackboard_attributes AS att2, tsk_files AS f " + // NON-NLS + "WHERE " + // NON-NLS + " (art.artifact_id IN (SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") " + // NON-NLS + "EXCEPT " + // NON-NLS + "SELECT art.artifact_id FROM blackboard_artifacts AS art, blackboard_attributes AS att1 WHERE (att1.artifact_id = art.artifact_id) AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() + "))) " + //NON-NLS + "AND (att1.artifact_id = art.artifact_id) " + //NON-NLS + "AND (att2.artifact_id = art.artifact_id) " + //NON-NLS + "AND (f.obj_id = art.obj_id) " + //NON-NLS + "AND (att1.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID() + ") " + // NON-NLS + "AND (att2.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID() + ") " + // NON-NLS + "AND (art.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID() + ") "; // NON-NLS + + String keywordsQuery = keywordListsQuery + " UNION " + keywordAdHocQuery + orderByClause; try (SleuthkitCase.CaseDbQuery dbQuery = openCase.getSleuthkitCase().executeQuery(keywordsQuery)) { ResultSet resultSet = dbQuery.getResultSet(); @@ -1623,14 +1710,15 @@ class TableReportGenerator { private HashSet getUniqueTagNames(long artifactId) throws TskCoreException { HashSet uniqueTagNames = new HashSet<>(); - String query = "SELECT display_name, artifact_id FROM tag_names AS tn, blackboard_artifact_tags AS bat " + String query = "SELECT display_name, artifact_id, knownStatus FROM tag_names AS tn, blackboard_artifact_tags AS bat " + //NON-NLS "WHERE tn.tag_name_id = bat.tag_name_id AND bat.artifact_id = " + artifactId; //NON-NLS try (SleuthkitCase.CaseDbQuery dbQuery = Case.getCurrentCaseThrows().getSleuthkitCase().executeQuery(query)) { ResultSet tagNameRows = dbQuery.getResultSet(); while (tagNameRows.next()) { - uniqueTagNames.add(tagNameRows.getString("display_name")); //NON-NLS + String notableString = tagNameRows.getInt("knownStatus") == TskData.FileKnown.BAD.ordinal() ? getNotableTagLabel() : ""; + uniqueTagNames.add(tagNameRows.getString("display_name") + notableString); //NON-NLS } } catch (TskCoreException | SQLException | NoCurrentCaseException ex) { throw new TskCoreException("Error getting tag names for artifact: ", ex); diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java index 9fddc6b5e1..f81e561b38 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IntraCaseUtils.java @@ -104,16 +104,36 @@ class IntraCaseUtils { } void setUp() { - CaseUtils.createAsCurrentCase(this.caseName); + this.createAsCurrentCase(); final ImageDSProcessor imageDSProcessor = new ImageDSProcessor(); - IngestUtils.addDataSource(imageDSProcessor, imagePath1); - IngestUtils.addDataSource(imageDSProcessor, imagePath2); - IngestUtils.addDataSource(imageDSProcessor, imagePath3); + this.addImageOne(imageDSProcessor); + this.addImageTwo(imageDSProcessor); + this.addImageThree(imageDSProcessor); + this.addImageFour(imageDSProcessor); + } + + void addImageFour(final ImageDSProcessor imageDSProcessor) { IngestUtils.addDataSource(imageDSProcessor, imagePath4); } + void addImageThree(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath3); + } + + void addImageTwo(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath2); + } + + void addImageOne(final ImageDSProcessor imageDSProcessor) { + IngestUtils.addDataSource(imageDSProcessor, imagePath1); + } + + void createAsCurrentCase() { + CaseUtils.createAsCurrentCase(this.caseName); + } + Map getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException { return this.dataSourceLoader.getDataSourceMap(); } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java new file mode 100644 index 0000000000..ba178a59f6 --- /dev/null +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/MatchesInAtLeastTwoSources.java @@ -0,0 +1,128 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.commonfilessearch; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import junit.framework.Test; +import org.netbeans.junit.NbModuleSuite; +import org.netbeans.junit.NbTestCase; +import org.openide.util.Exceptions; +import org.python.icu.impl.Assert; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.ImageDSProcessor; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.commonfilesearch.AllDataSourcesCommonFilesAlgorithm; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadata; +import org.sleuthkit.autopsy.commonfilesearch.CommonFilesMetadataBuilder; +import static org.sleuthkit.autopsy.commonfilessearch.IntraCaseUtils.*; +import org.sleuthkit.autopsy.ingest.IngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleTemplate; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory; +import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory; +import org.sleuthkit.autopsy.testutils.IngestUtils; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Ensures that matches only are found for files which appear in at least two + * data sources. + * + * The two datasources used here have no common files. One of the data sources + * has two identical files within it. This should not count as a match. + * + * None of the test files should be found in the results of this test. + */ +public class MatchesInAtLeastTwoSources extends NbTestCase { + + public static Test suite() { + NbModuleSuite.Configuration conf = NbModuleSuite.createConfiguration(MatchesInAtLeastTwoSources.class). + clusters(".*"). + enableModules(".*"); + return conf.suite(); + } + + private final IntraCaseUtils utils; + + public MatchesInAtLeastTwoSources(String name) { + super(name); + + this.utils = new IntraCaseUtils(this, "MatchesInAtLeastTwoSources"); + } + + @Override + public void setUp() { + this.utils.createAsCurrentCase(); + + final ImageDSProcessor imageDSProcessor = new ImageDSProcessor(); + + this.utils.addImageOne(imageDSProcessor); + this.utils.addImageFour(imageDSProcessor); + + IngestModuleTemplate hashLookupTemplate = IngestUtils.getIngestModuleTemplate(new HashLookupModuleFactory()); + IngestModuleTemplate mimeTypeLookupTemplate = IngestUtils.getIngestModuleTemplate(new FileTypeIdModuleFactory()); + + ArrayList templates = new ArrayList<>(); + templates.add(hashLookupTemplate); + templates.add(mimeTypeLookupTemplate); + + IngestJobSettings ingestJobSettings = new IngestJobSettings(IngestedWithHashAndFileType.class.getCanonicalName(), IngestJobSettings.IngestType.FILES_ONLY, templates); + + try { + IngestUtils.runIngestJob(Case.getCurrentCaseThrows().getDataSources(), ingestJobSettings); + } catch (NoCurrentCaseException | TskCoreException ex) { + Exceptions.printStackTrace(ex); + Assert.fail(ex); + } + } + + @Override + public void tearDown() { + this.utils.tearDown(); + } + + public void testOne() { + try { + Map dataSources = this.utils.getDataSourceMap(); + + CommonFilesMetadataBuilder allSourcesBuilder = new AllDataSourcesCommonFilesAlgorithm(dataSources, false, false); + CommonFilesMetadata metadata = allSourcesBuilder.findCommonFiles(); + + Map objectIdToDataSource = IntraCaseUtils.mapFileInstancesToDataSources(metadata); + + List files = IntraCaseUtils.getFiles(objectIdToDataSource.keySet()); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, IMG, SET4, 0)); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, DOC, SET4, 0)); + + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET1, 0)); + assertTrue(IntraCaseUtils.verifyFileExistanceAndCount(files, dataSources, EMPTY, SET4, 0)); + + } catch (NoCurrentCaseException | TskCoreException | SQLException ex) { + Exceptions.printStackTrace(ex); + Assert.fail(ex); + } + } +} diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 9830cecbe9..e62c58343c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -33,7 +33,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.LocalDiskDSProcessor; import org.sleuthkit.autopsy.casemodule.LocalFilesDSProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import static org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java index 1cc10fab76..d744abe8b4 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusPanel.java @@ -125,18 +125,6 @@ final class AinStatusPanel extends javax.swing.JPanel implements ExplorerManager setLayout(new java.awt.BorderLayout()); }// //GEN-END:initComponents - /** - * Get the AutoIngestJob for the currently selected node of this panel. - * - * @return AutoIngestJob which is currently selected in this panel - */ - AutoIngestJob getSelectedAutoIngestJob() { - Node[] selectedRows = explorerManager.getSelectedNodes(); - if (selectedRows.length == 1) { - return ((JobNode) selectedRows[0]).getAutoIngestJob(); - } - return null; - } // Variables declaration - do not modify//GEN-BEGIN:variables // End of variables declaration//GEN-END:variables diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java index 87c805fa12..1e1b82d8e7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java @@ -338,7 +338,7 @@ final class AutoIngestAdminActions { dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); AutoIngestManager.CaseDeletionResult result = dashboard.getMonitor().deleteCase(job); - dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor().getJobsSnapshot())); + dashboard.getCompletedJobsPanel().refresh(new AutoIngestNodeRefreshEvents.RefreshChildrenEvent(dashboard.getMonitor())); dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); if (AutoIngestManager.CaseDeletionResult.FAILED == result) { JOptionPane.showMessageDialog(dashboard, diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java index 49e9cdbd17..031aa64daf 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java @@ -38,7 +38,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.core.ServicesMonitor; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeRefreshEvents.RefreshChildrenEvent; /** @@ -257,7 +256,7 @@ final class AutoIngestDashboard extends JPanel implements Observer { @Override public void update(Observable observable, Object arg) { - if (arg instanceof JobsSnapshot) { + if (arg == null ) { EventQueue.invokeLater(() -> { refreshTables(); }); @@ -271,9 +270,9 @@ final class AutoIngestDashboard extends JPanel implements Observer { * @param nodeStateSnapshot The jobs snapshot. */ void refreshTables() { - pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); - completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor.getJobsSnapshot())); + pendingJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + runningJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); + completedJobsPanel.refresh(new RefreshChildrenEvent(autoIngestMonitor)); } /** diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java index a7b2ad6116..d5bfbd13d8 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Objects; import org.openide.nodes.AbstractNode; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; @@ -33,9 +34,10 @@ import org.openide.nodes.Node; import org.openide.nodes.Sheet; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.datamodel.NodeProperty; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; +import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.Stage; import org.sleuthkit.autopsy.guiutils.DurationCellRenderer; import org.sleuthkit.autopsy.guiutils.StatusIconCellRenderer; +import org.sleuthkit.autopsy.ingest.DataSourceIngestJob; /** * A node which represents all AutoIngestJobs of a given AutoIngestJobStatus. @@ -61,13 +63,13 @@ final class AutoIngestJobsNode extends AbstractNode { /** * Construct a new AutoIngestJobsNode. * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which will be used to send and receive * refresh events */ - AutoIngestJobsNode(JobsSnapshot jobsSnapshot, AutoIngestJobStatus status, EventBus eventBus) { - super(Children.create(new AutoIngestNodeChildren(jobsSnapshot, status, eventBus), false)); + AutoIngestJobsNode(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + super(Children.create(new AutoIngestNodeChildren(monitor, status, eventBus), false)); refreshChildrenEventBus = eventBus; } @@ -78,13 +80,106 @@ final class AutoIngestJobsNode extends AbstractNode { refreshChildrenEventBus.post(refreshEvent); } + /** + * The AutoIngestJob class considers auto ingest jobs to be equal if they + * have the same manifest path. This is not sufficient for the purposes of + * determining when the state of a job has changed. This class is used to + * distinguish between different auto ingest jobs based on the manifest + * path, the processing stage, the job snapshot and priority. + */ + private static final class AutoIngestJobWrapper implements Comparable { + + private final AutoIngestJob autoIngestJob; + + /** + * We keep our own references to the following job attributes because + * they can be changed by events in other threads which + */ + private final Stage jobStage; + private final List jobSnapshot; + private final Integer jobPriority; + + AutoIngestJobWrapper(AutoIngestJob job) { + autoIngestJob = job; + jobStage = job.getProcessingStage(); + jobSnapshot = job.getIngestJobSnapshots(); + jobPriority = job.getPriority(); + } + + AutoIngestJob getJob() { + return autoIngestJob; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof AutoIngestJobWrapper)) { + return false; + } + + if (this == other) { + return true; + } + + AutoIngestJob thisJob = this.autoIngestJob; + AutoIngestJob otherJob = ((AutoIngestJobWrapper) other).autoIngestJob; + + // Only equal if the manifest paths and processing stage details are the same. + return thisJob.getManifest().getFilePath().equals(otherJob.getManifest().getFilePath()) + && jobStage.equals(((AutoIngestJobWrapper) other).jobStage) + && jobSnapshot.equals(((AutoIngestJobWrapper) other).jobSnapshot) + && jobPriority.equals(((AutoIngestJobWrapper) other).jobPriority); + } + + @Override + public int hashCode() { + int hash = 7; + hash = 23 * hash + Objects.hashCode(this.autoIngestJob.getManifest().getFilePath()); + hash = 23 * hash + Objects.hashCode(this.jobStage); + hash = 23 * hash + Objects.hashCode(this.jobSnapshot); + hash = 23 * hash + Objects.hashCode(this.jobPriority); + return hash; + } + + @Override + public int compareTo(AutoIngestJobWrapper o) { + return autoIngestJob.compareTo(o.autoIngestJob); + } + + /** + * The remaining methods simply delegate to the wrapped job. + */ + Manifest getManifest() { + return autoIngestJob.getManifest(); + } + + boolean getErrorsOccurred() { + return autoIngestJob.getErrorsOccurred(); + } + + Date getCompletedDate() { + return autoIngestJob.getCompletedDate(); + } + + AutoIngestJob.StageDetails getProcessingStageDetails() { + return autoIngestJob.getProcessingStageDetails(); + } + + String getProcessingHostName() { + return autoIngestJob.getProcessingHostName(); + } + + Integer getPriority() { + return autoIngestJob.getPriority(); + } + } + /** * A ChildFactory for generating JobNodes. */ - static final class AutoIngestNodeChildren extends ChildFactory { + static final class AutoIngestNodeChildren extends ChildFactory { private final AutoIngestJobStatus autoIngestJobStatus; - private JobsSnapshot jobsSnapshot; + private AutoIngestMonitor monitor; private final RefreshChildrenSubscriber refreshChildrenSubscriber = new RefreshChildrenSubscriber(); private final EventBus refreshEventBus; @@ -92,43 +187,45 @@ final class AutoIngestJobsNode extends AbstractNode { * Create children nodes for the AutoIngestJobsNode which will each * represent a single AutoIngestJob * - * @param snapshot the snapshot which contains the AutoIngestJobs + * @param monitor the monitor which gives access to the AutoIngestJobs * @param status the status of the jobs being displayed * @param eventBus the event bus which the class registers to for * refresh events */ - AutoIngestNodeChildren(JobsSnapshot snapshot, AutoIngestJobStatus status, EventBus eventBus) { - jobsSnapshot = snapshot; + AutoIngestNodeChildren(AutoIngestMonitor monitor, AutoIngestJobStatus status, EventBus eventBus) { + this.monitor = monitor; autoIngestJobStatus = status; refreshEventBus = eventBus; refreshChildrenSubscriber.register(refreshEventBus); } @Override - protected boolean createKeys(List list) { + protected boolean createKeys(List list) { List jobs; switch (autoIngestJobStatus) { case PENDING_JOB: - jobs = jobsSnapshot.getPendingJobs(); + jobs = monitor.getPendingJobs(); Collections.sort(jobs); break; case RUNNING_JOB: - jobs = jobsSnapshot.getRunningJobs(); + jobs = monitor.getRunningJobs(); break; case COMPLETED_JOB: - jobs = jobsSnapshot.getCompletedJobs(); + jobs = monitor.getCompletedJobs(); break; default: jobs = new ArrayList<>(); } if (jobs != null && jobs.size() > 0) { - list.addAll(jobs); + jobs.forEach(j -> { + list.add(new AutoIngestJobWrapper(j)); + }); } return true; } @Override - protected Node createNodeForKey(AutoIngestJob key) { + protected Node createNodeForKey(AutoIngestJobWrapper key) { return new JobNode(key, autoIngestJobStatus, refreshEventBus); } @@ -167,7 +264,7 @@ final class AutoIngestJobsNode extends AbstractNode { //Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus //RefreshChildrenEvents can change which children are present however //RefreshJobEvents and RefreshCaseEvents can still change the order we want to display them in - jobsSnapshot = refreshEvent.getJobsSnapshot(); + monitor = refreshEvent.getMonitor(); refresh(true); } @@ -180,7 +277,7 @@ final class AutoIngestJobsNode extends AbstractNode { */ static final class JobNode extends AbstractNode { - private final AutoIngestJob autoIngestJob; + private final AutoIngestJobWrapper jobWrapper; private final AutoIngestJobStatus jobStatus; private final RefreshNodeSubscriber refreshNodeSubscriber = new RefreshNodeSubscriber(); @@ -191,12 +288,12 @@ final class AutoIngestJobsNode extends AbstractNode { * @param status - the current status of the AutoIngestJob being * represented */ - JobNode(AutoIngestJob job, AutoIngestJobStatus status, EventBus eventBus) { + JobNode(AutoIngestJobWrapper job, AutoIngestJobStatus status, EventBus eventBus) { super(Children.LEAF); jobStatus = status; - autoIngestJob = job; - setName(autoIngestJob.toString()); //alows job to be uniquely found by name since it will involve a hash of the AutoIngestJob - setDisplayName(autoIngestJob.getManifest().getCaseName()); //displays user friendly case name as name + jobWrapper = job; + setName(jobWrapper.toString()); //alows job to be uniquely found by name since it will involve a hash of the AutoIngestJob + setDisplayName(jobWrapper.getManifest().getCaseName()); //displays user friendly case name as name refreshNodeSubscriber.register(eventBus); } @@ -206,7 +303,7 @@ final class AutoIngestJobsNode extends AbstractNode { * @return autoIngestJob */ AutoIngestJob getAutoIngestJob() { - return autoIngestJob; + return jobWrapper.getJob(); } @Override @@ -221,20 +318,20 @@ final class AutoIngestJobsNode extends AbstractNode { s.put(ss); } ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_caseName_text(), Bundle.AutoIngestJobsNode_caseName_text(), Bundle.AutoIngestJobsNode_caseName_text(), - autoIngestJob.getManifest().getCaseName())); + jobWrapper.getManifest().getCaseName())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_dataSource_text(), Bundle.AutoIngestJobsNode_dataSource_text(), Bundle.AutoIngestJobsNode_dataSource_text(), - autoIngestJob.getManifest().getDataSourcePath().getFileName().toString())); + jobWrapper.getManifest().getDataSourcePath().getFileName().toString())); switch (jobStatus) { case PENDING_JOB: ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), - autoIngestJob.getManifest().getDateFileCreated())); + jobWrapper.getManifest().getDateFileCreated())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_priority_text(), Bundle.AutoIngestJobsNode_priority_text(), Bundle.AutoIngestJobsNode_priority_text(), - autoIngestJob.getPriority() > 0 ? Bundle.AutoIngestJobsNode_prioritized_true() : Bundle.AutoIngestJobsNode_prioritized_false())); + jobWrapper.getPriority() > 0 ? Bundle.AutoIngestJobsNode_prioritized_true() : Bundle.AutoIngestJobsNode_prioritized_false())); break; case RUNNING_JOB: - AutoIngestJob.StageDetails status = autoIngestJob.getProcessingStageDetails(); + AutoIngestJob.StageDetails status = jobWrapper.getProcessingStageDetails(); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_hostName_text(), Bundle.AutoIngestJobsNode_hostName_text(), Bundle.AutoIngestJobsNode_hostName_text(), - autoIngestJob.getProcessingHostName())); + jobWrapper.getProcessingHostName())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_stage_text(), Bundle.AutoIngestJobsNode_stage_text(), Bundle.AutoIngestJobsNode_stage_text(), status.getDescription())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_stageTime_text(), Bundle.AutoIngestJobsNode_stageTime_text(), Bundle.AutoIngestJobsNode_stageTime_text(), @@ -242,11 +339,11 @@ final class AutoIngestJobsNode extends AbstractNode { break; case COMPLETED_JOB: ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), Bundle.AutoIngestJobsNode_jobCreated_text(), - autoIngestJob.getManifest().getDateFileCreated())); + jobWrapper.getManifest().getDateFileCreated())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_jobCompleted_text(), Bundle.AutoIngestJobsNode_jobCompleted_text(), Bundle.AutoIngestJobsNode_jobCompleted_text(), - autoIngestJob.getCompletedDate())); + jobWrapper.getCompletedDate())); ss.put(new NodeProperty<>(Bundle.AutoIngestJobsNode_status_text(), Bundle.AutoIngestJobsNode_status_text(), Bundle.AutoIngestJobsNode_status_text(), - autoIngestJob.getErrorsOccurred() ? StatusIconCellRenderer.Status.WARNING : StatusIconCellRenderer.Status.OK)); + jobWrapper.getErrorsOccurred() ? StatusIconCellRenderer.Status.WARNING : StatusIconCellRenderer.Status.OK)); break; default: } @@ -259,24 +356,24 @@ final class AutoIngestJobsNode extends AbstractNode { if (AutoIngestDashboard.isAdminAutoIngestDashboard()) { switch (jobStatus) { case PENDING_JOB: - actions.add(new PrioritizationAction.PrioritizeJobAction(autoIngestJob)); - actions.add(new PrioritizationAction.PrioritizeCaseAction(autoIngestJob)); - PrioritizationAction.DeprioritizeJobAction deprioritizeJobAction = new PrioritizationAction.DeprioritizeJobAction(autoIngestJob); - deprioritizeJobAction.setEnabled(autoIngestJob.getPriority() > 0); + actions.add(new PrioritizationAction.PrioritizeJobAction(jobWrapper.getJob())); + actions.add(new PrioritizationAction.PrioritizeCaseAction(jobWrapper.getJob())); + PrioritizationAction.DeprioritizeJobAction deprioritizeJobAction = new PrioritizationAction.DeprioritizeJobAction(jobWrapper.getJob()); + deprioritizeJobAction.setEnabled(jobWrapper.getPriority() > 0); actions.add(deprioritizeJobAction); - PrioritizationAction.DeprioritizeCaseAction deprioritizeCaseAction = new PrioritizationAction.DeprioritizeCaseAction(autoIngestJob); - deprioritizeCaseAction.setEnabled(autoIngestJob.getPriority() > 0); + PrioritizationAction.DeprioritizeCaseAction deprioritizeCaseAction = new PrioritizationAction.DeprioritizeCaseAction(jobWrapper.getJob()); + deprioritizeCaseAction.setEnabled(jobWrapper.getPriority() > 0); actions.add(deprioritizeCaseAction); break; case RUNNING_JOB: - actions.add(new AutoIngestAdminActions.ProgressDialogAction(autoIngestJob)); - actions.add(new AutoIngestAdminActions.CancelJobAction(autoIngestJob)); + actions.add(new AutoIngestAdminActions.ProgressDialogAction(jobWrapper.getJob())); + actions.add(new AutoIngestAdminActions.CancelJobAction(jobWrapper.getJob())); // actions.add(new AutoIngestAdminActions.CancelModuleAction()); break; case COMPLETED_JOB: - actions.add(new AutoIngestAdminActions.ReprocessJobAction(autoIngestJob)); - actions.add(new AutoIngestAdminActions.DeleteCaseAction(autoIngestJob)); - actions.add(new AutoIngestAdminActions.ShowCaseLogAction(autoIngestJob)); + actions.add(new AutoIngestAdminActions.ReprocessJobAction(jobWrapper.getJob())); + actions.add(new AutoIngestAdminActions.DeleteCaseAction(jobWrapper.getJob())); + actions.add(new AutoIngestAdminActions.ShowCaseLogAction(jobWrapper.getJob())); break; default: } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java index a3cdc1c51d..fb6f79ab0c 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java @@ -171,7 +171,7 @@ final class AutoIngestJobsPanel extends javax.swing.JPanel implements ExplorerMa ((AutoIngestJobsNode) explorerManager.getRootContext()).refresh(refreshEvent); } else { //Make a new AutoIngestJobsNode with it's own EventBus and set it as the root context - explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getJobsSnapshot(), status, new EventBus("AutoIngestJobsNodeEventBus"))); + explorerManager.setRootContext(new AutoIngestJobsNode(refreshEvent.getMonitor(), status, new EventBus("AutoIngestJobsNodeEventBus"))); } outline.setRowSelectionAllowed(true); outline.setFocusable(true); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 77ddcf8d63..8dddd2a3e6 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -490,6 +490,14 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen break; case RESUME: resume(); + + /** + * Kick off an immediate scan so that the next pending job + * will be picked up sooner than having to wait for the + * InputDirScannerTask to run again. + */ + scanInputDirsNow(); + break; case SHUTDOWN: shutDown(); @@ -1840,13 +1848,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen */ setChanged(); notifyObservers(Event.RESUMED); - - /** - * Publish an event to let remote listeners know that the - * node has been resumed. - */ - eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); } + /** + * Publish an event to let remote listeners know that the node + * has been resumed. + */ + eventPublisher.publishRemotely(lastPublishedStateEvent = new AutoIngestNodeStateEvent(Event.RESUMED, AutoIngestManager.LOCAL_HOST_NAME)); + pauseLock.notifyAll(); } } @@ -2790,6 +2798,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen sysLogger.log(Level.INFO, "Finished ingest modules analysis for {0} ", manifestPath); IngestJob.ProgressSnapshot jobSnapshot = ingestJob.getSnapshot(); for (IngestJob.ProgressSnapshot.DataSourceProcessingSnapshot snapshot : jobSnapshot.getDataSourceSnapshots()) { + AutoIngestJobLogger nestedJobLogger = new AutoIngestJobLogger(manifestPath, snapshot.getDataSource(), caseDirectoryPath); if (!snapshot.isCancelled()) { List cancelledModules = snapshot.getCancelledDataSourceIngestModules(); if (!cancelledModules.isEmpty()) { @@ -2798,15 +2807,15 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen setCaseNodeDataErrorsOccurred(caseDirectoryPath); for (String module : snapshot.getCancelledDataSourceIngestModules()) { sysLogger.log(Level.WARNING, String.format("%s ingest module cancelled for %s", module, manifestPath)); - jobLogger.logIngestModuleCancelled(module); + nestedJobLogger.logIngestModuleCancelled(module); } } - jobLogger.logAnalysisCompleted(); + nestedJobLogger.logAnalysisCompleted(); } else { currentJob.setProcessingStage(AutoIngestJob.Stage.CANCELLING, Date.from(Instant.now())); currentJob.setErrorsOccurred(true); setCaseNodeDataErrorsOccurred(caseDirectoryPath); - jobLogger.logAnalysisCancelled(); + nestedJobLogger.logAnalysisCancelled(); CancellationReason cancellationReason = snapshot.getCancellationReason(); if (CancellationReason.NOT_CANCELLED != cancellationReason && CancellationReason.USER_CANCELLED != cancellationReason) { throw new AnalysisStartupException(String.format("Analysis cancelled due to %s for %s", cancellationReason.getDisplayName(), manifestPath)); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index d9fb71f56c..cfdf6cf973 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -116,11 +117,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } catch (AutopsyEventException ex) { throw new AutoIngestMonitorException("Failed to open auto ingest event channel", ex); //NON-NLS } - coordSvcQueryExecutor.scheduleWithFixedDelay(new CoordinationServiceQueryTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); + coordSvcQueryExecutor.scheduleWithFixedDelay(new StateRefreshTask(), 0, CORRD_SVC_QUERY_INERVAL_MINS, TimeUnit.MINUTES); eventPublisher.addSubscriber(EVENT_LIST, this); - // Publish an event that asks running nodes to send their state. - eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + refreshNodeState(); } /** @@ -172,7 +172,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(event.getJob()); jobsSnapshot.addOrReplaceRunningJob(event.getJob()); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -190,7 +190,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removePendingJob(job); // Update the state of the existing job in the running jobs table - for (AutoIngestJob runningJob : jobsSnapshot.getRunningJobs()) { + for (AutoIngestJob runningJob : getRunningJobs()) { if (runningJob.equals(job)) { runningJob.setIngestJobsSnapshot(job.getIngestJobSnapshots()); runningJob.setIngestThreadSnapshot(job.getIngestThreadActivitySnapshots()); @@ -200,7 +200,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } } setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -216,7 +216,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen jobsSnapshot.removeRunningJob(job); jobsSnapshot.addOrReplaceCompletedJob(job); setChanged(); - notifyObservers(jobsSnapshot); + notifyObservers(); } } @@ -226,7 +226,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case prioritization event. */ private void handleCasePrioritizationEvent(AutoIngestCasePrioritizedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -235,7 +235,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @param event A job/case deletion event. */ private void handleCaseDeletedEvent(AutoIngestCaseDeletedEvent event) { - coordSvcQueryExecutor.submit(new CoordinationServiceQueryTask()); + coordSvcQueryExecutor.submit(new StateRefreshTask()); } /** @@ -259,15 +259,35 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * Gets the auto ingest monitor's current snapshot of the pending jobs - * queue, running jobs list, and completed jobs list for an auto ingest - * cluster. + * Gets the snapshot of the pending jobs queue for an auto ingest cluster. * - * @return The snapshot. + * @return The pending jobs queue. */ - JobsSnapshot getJobsSnapshot() { + List getPendingJobs() { synchronized (jobsLock) { - return jobsSnapshot; + return new ArrayList<>(jobsSnapshot.pendingJobs); + } + } + + /** + * Gets the snapshot of the running jobs list for an auto ingest cluster. + * + * @return The running jobs list. + */ + List getRunningJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.runningJobs); + } + } + + /** + * Gets the snapshot of the completed jobs list for an auto ingest cluster. + * + * @return The completed jobs list. + */ + List getCompletedJobs() { + synchronized (jobsLock) { + return new ArrayList<>(jobsSnapshot.completedJobs); } } @@ -277,7 +297,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @return */ List getNodeStates() { - return nodeStates.values().stream().collect(Collectors.toList()); + // We only report the state for nodes for which we have received + // a 'state' event in the last 15 minutes. + return nodeStates.values() + .stream() + .filter(s -> s.getLastSeenTime().isAfter(Instant.now().minus(Duration.ofMinutes(15)))) + .collect(Collectors.toList()); } /** @@ -287,13 +312,20 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * * @return The refreshed snapshot. */ - JobsSnapshot refreshJobsSnapshot() { + void refreshJobsSnapshot() { synchronized (jobsLock) { jobsSnapshot = queryCoordinationService(); - return jobsSnapshot; } } + /** + * Ask running auto ingest nodes to report their state. + */ + private void refreshNodeState() { + // Publish an event that asks running nodes to send their state. + eventPublisher.publishRemotely(new AutoIngestRequestNodeStateEvent(AutoIngestManager.Event.REPORT_STATE)); + } + /** * Gets a new snapshot of the pending jobs queue, running jobs list, and * completed jobs list for an auto ingest cluster. @@ -358,13 +390,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeCase(final String caseName) throws AutoIngestMonitorException { + void deprioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToDeprioritize = new ArrayList<>(); synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getManifest().getCaseName().equals(caseName)) { jobsToDeprioritize.add(pendingJob); } @@ -392,10 +423,9 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_DEPRIORITIZED, "")); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_DEPRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -407,13 +437,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the jobs for the case. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeCase(final String caseName) throws AutoIngestMonitorException { + void prioritizeCase(final String caseName) throws AutoIngestMonitorException { List jobsToPrioritize = new ArrayList<>(); int highestPriority = 0; synchronized (jobsLock) { - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -445,10 +474,9 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_PRIORITIZED, "")); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.CASE_PRIORITIZED, "")); }).start(); } - return jobsSnapshot; } } @@ -460,15 +488,14 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error removing the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void deprioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { AutoIngestJob jobToDeprioritize = null; /* * Make sure the job is still in the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.equals(job)) { jobToDeprioritize = job; break; @@ -502,11 +529,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen final String dataSourceName = jobToDeprioritize.getManifest().getDataSourceFileName(); new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_DEPRIORITIZED, dataSourceName)); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_DEPRIORITIZED, dataSourceName)); }).start(); } - return jobsSnapshot; } } @@ -518,9 +544,8 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @throws AutoIngestMonitorException If there is an error bumping the * priority of the job. * - * @return The latest jobs snapshot. */ - JobsSnapshot prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { + void prioritizeJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { int highestPriority = 0; AutoIngestJob jobToPrioritize = null; @@ -528,7 +553,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * Get the highest known priority and make sure the job is still in * the pending jobs queue. */ - for (AutoIngestJob pendingJob : jobsSnapshot.getPendingJobs()) { + for (AutoIngestJob pendingJob : getPendingJobs()) { if (pendingJob.getPriority() > highestPriority) { highestPriority = pendingJob.getPriority(); } @@ -565,11 +590,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen final String dataSourceName = jobToPrioritize.getManifest().getDataSourceFileName(); new Thread(() -> { eventPublisher.publishRemotely(new AutoIngestCasePrioritizedEvent(LOCAL_HOST_NAME, caseName, - AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_PRIORITIZED, dataSourceName)); + AutoIngestManager.getSystemUserNameProperty(), AutoIngestCasePrioritizedEvent.EventType.JOB_PRIORITIZED, dataSourceName)); }).start(); } - return jobsSnapshot; } } @@ -591,7 +615,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen */ void reprocessJob(AutoIngestJob job) throws AutoIngestMonitorException { synchronized (jobsLock) { - if (!jobsSnapshot.getCompletedJobs().contains(job)) { + if (!getCompletedJobs().contains(job)) { return; } @@ -660,7 +684,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen // Update the state of completed jobs associated with this case to indicate // that the case has been deleted - for (AutoIngestJob completedJob : jobsSnapshot.getCompletedJobs()) { + for (AutoIngestJob completedJob : getCompletedJobs()) { if (caseName.equals(completedJob.getManifest().getCaseName())) { try { completedJob.setProcessingStatus(DELETED); @@ -674,8 +698,8 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } // Remove jobs associated with this case from the completed jobs collection. - jobsSnapshot.completedJobs.removeIf((AutoIngestJob completedJob) -> - completedJob.getManifest().getCaseName().equals(caseName)); + jobsSnapshot.completedJobs.removeIf((AutoIngestJob completedJob) + -> completedJob.getManifest().getCaseName().equals(caseName)); // Publish a message to update auto ingest nodes. eventPublisher.publishRemotely(new AutoIngestCaseDeletedEvent(caseName, LOCAL_HOST_NAME, AutoIngestManager.getSystemUserNameProperty())); @@ -724,25 +748,25 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen } /** - * A task that queries the coordination service for auto ingest manifest - * node data and converts it to auto ingest jobs for publication top its - * observers. + * A task that updates the state maintained by the monitor. At present this + * includes auto ingest job and auto ingest node data. The job data is + * refreshed by querying the coordination service for auto ingest manifest + * nodes. The auto ingest node data is refreshed by publishing a message + * asking all nodes to report their state. */ - private final class CoordinationServiceQueryTask implements Runnable { + private final class StateRefreshTask implements Runnable { - /** - * Queries the coordination service for auto ingest manifest node data - * and converts it to auto ingest jobs for publication top its - * observers. - */ @Override public void run() { if (!Thread.currentThread().isInterrupted()) { - synchronized (jobsLock) { - jobsSnapshot = queryCoordinationService(); - setChanged(); - notifyObservers(jobsSnapshot); - } + // Query coordination service for jobs data. + refreshJobsSnapshot(); + + // Ask running auto ingest nodes to report their status. + refreshNodeState(); + + setChanged(); + notifyObservers(); } } @@ -752,42 +776,12 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * A snapshot of the pending jobs queue, running jobs list and completed * jobs list for an auto ingest cluster. */ - static final class JobsSnapshot { + private static final class JobsSnapshot { private final Set pendingJobs = new HashSet<>(); private final Set runningJobs = new HashSet<>(); private final Set completedJobs = new HashSet<>(); - /** - * Gets the snapshot of the pending jobs queue for an auto ingest - * cluster. - * - * @return The pending jobs queue. - */ - List getPendingJobs() { - return new ArrayList<>(this.pendingJobs); - } - - /** - * Gets the snapshot of the running jobs list for an auto ingest - * cluster. - * - * @return The running jobs list. - */ - List getRunningJobs() { - return new ArrayList<>(this.runningJobs); - } - - /** - * Gets the snapshot of the completed jobs list for an auto ingest - * cluster. - * - * @return The completed jobs list. - */ - List getCompletedJobs() { - return new ArrayList<>(this.completedJobs); - } - /** * Adds an auto job to the snapshot of the pending jobs queue for an * auto ingest cluster. If an equivalent job already exists, it is @@ -886,6 +880,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen private final String nodeName; private final State nodeState; + private final Instant lastSeenTime; AutoIngestNodeState(String name, Event event) { nodeName = name; @@ -915,6 +910,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen nodeState = State.UNKNOWN; break; } + lastSeenTime = Instant.now(); } String getName() { @@ -924,6 +920,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen State getState() { return nodeState; } + + Instant getLastSeenTime() { + return lastSeenTime; + } } /** diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java index 2c554bf00f..338bce31c7 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.experimental.autoingest; -import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot; - /** * Class which contains events to identify what should be refreshed in the * AutoIngestJobsNode @@ -31,19 +29,20 @@ class AutoIngestNodeRefreshEvents { */ static class AutoIngestRefreshEvent { - private final JobsSnapshot jobsSnapshot; + private final AutoIngestMonitor monitor; - AutoIngestRefreshEvent(JobsSnapshot jobs) { - this.jobsSnapshot = jobs; + AutoIngestRefreshEvent(AutoIngestMonitor monitor) { + this.monitor = monitor; } /** - * Get the state of the jobs lists when the event was fired. + * Get the monitor which will provide access to the state of + * the jobs. * * @return */ - JobsSnapshot getJobsSnapshot() { - return this.jobsSnapshot; + AutoIngestMonitor getMonitor() { + return this.monitor; } } @@ -56,8 +55,8 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshChildrenEvent. */ - RefreshChildrenEvent(JobsSnapshot jobs) { - super(jobs); + RefreshChildrenEvent(AutoIngestMonitor monitor) { + super(monitor); } } @@ -72,11 +71,11 @@ class AutoIngestNodeRefreshEvents { /** * Contructs a RefreshCaseEvent * - * @param jobs The current state of the jobs lists. + * @param monitor The monitor that will provide access to the current state of the jobs lists. * @param name The name of the case whose nodes should be refreshed. */ - RefreshCaseEvent(JobsSnapshot jobs, String name) { - super(jobs); + RefreshCaseEvent(AutoIngestMonitor monitor, String name) { + super(monitor); caseName = name; } @@ -103,11 +102,11 @@ class AutoIngestNodeRefreshEvents { /** * Constructs a RefreshJobEvent. * - * @param jobs The curent state of the jobs lists. + * @param monitor The monitor which will provide access to the current state of the jobs lists. * @param job The job which should be refreshed. */ - RefreshJobEvent(JobsSnapshot jobs, AutoIngestJob job) { - super(jobs); + RefreshJobEvent(AutoIngestMonitor monitor, AutoIngestJob job) { + super(monitor); autoIngestJob = job; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java index 3d57057bff..006dc3e580 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java @@ -145,7 +145,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -184,7 +184,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor.getJobsSnapshot(), getJob()); + return new AutoIngestNodeRefreshEvents.RefreshJobEvent(monitor, getJob()); } } @@ -225,7 +225,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } @@ -266,7 +266,7 @@ abstract class PrioritizationAction extends AbstractAction { @Override AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent(AutoIngestMonitor monitor) { - return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor.getJobsSnapshot(), getJob().getManifest().getCaseName()); + return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(monitor, getJob().getManifest().getCaseName()); } } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java index fe86fb9f0b..d6bef92ab8 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/objectdetection/ObjectDetectectionFileIngestModule.java @@ -106,10 +106,22 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter try { imageInMemory = IOUtils.toByteArray(inputStream); } catch (IOException ex) { - logger.log(Level.WARNING, "Unable to perform object detection on " + file.getName(), ex); + logger.log(Level.WARNING, "Unable to read image to byte array for performing object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); return IngestModule.ProcessResult.ERROR; } - Mat originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + Mat originalImage; + try { + originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_GRAYSCALE); + } catch (CvException ex) { + //The image was something which could not be decoded by OpenCv, our isImageThumbnailSupported(file) check above failed us + logger.log(Level.WARNING, "Unable to decode image from byte array to perform object detection on " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), ex); //NON-NLS + return IngestModule.ProcessResult.ERROR; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered attempting to use OpenCV to decode picture: " + file.getParentPath() + file.getName() + " with object id of " + file.getId(), unexpectedException); + return IngestModule.ProcessResult.ERROR; + } + MatOfRect detectionRectangles = new MatOfRect(); //the rectangles which reprent the coordinates on the image for where objects were detected for (String classifierKey : classifiers.keySet()) { //apply each classifier to the file @@ -117,7 +129,10 @@ public class ObjectDetectectionFileIngestModule extends FileIngestModuleAdapter classifiers.get(classifierKey).detectMultiScale(originalImage, detectionRectangles); } catch (CvException ignored) { //The image was likely an image which we are unable to generate a thumbnail for, and the classifier was likely one where that is not acceptable - logger.log(Level.INFO, String.format("Classifier '%s' could not be applied to file '%s'.", classifierKey, file.getParentPath() + file.getName())); //NON-NLS + continue; + } catch (Exception unexpectedException) { + //hopefully an unnecessary generic exception catch but currently present to catch any exceptions OpenCv throws which may not be documented + logger.log(Level.SEVERE, "Unexpected Exception encountered for image " + file.getParentPath() + file.getName() + " with object id of " + file.getId() +" while trying to apply classifier " + classifierKey, unexpectedException); continue; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java index dc05cb8376..5dbdca94ce 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java @@ -56,7 +56,7 @@ public class MemoryDSProcessor implements DataSourceProcessor { * * @return A data source type display string for this data source processor. */ - @Messages({"MemoryDSProcessor.dataSourceType=Memory Image File"}) + @Messages({"MemoryDSProcessor.dataSourceType=Memory Image File (Volatility)"}) public static String getType() { return Bundle.MemoryDSProcessor_dataSourceType(); } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java index 2b3fc1692f..7fbcaac1f0 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryOptionsPanel.java @@ -203,17 +203,20 @@ final class ImageGalleryOptionsPanel extends javax.swing.JPanel { } void store() { - Case openCase; - try { - openCase = Case.getCurrentCaseThrows(); - } catch (NoCurrentCaseException ex) { - Logger.getLogger(ImageGalleryOptionsPanel.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return; - } ImageGalleryPreferences.setEnabledByDefault(enabledByDefaultBox.isSelected()); ImageGalleryController.getDefault().setListeningEnabled(enabledForCaseBox.isSelected()); - new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); ImageGalleryPreferences.setGroupCategorizationWarningDisabled(groupCategorizationWarningBox.isSelected()); + + // If a case is open, save the per case setting + try { + Case openCase = Case.getCurrentCaseThrows(); + new PerCaseProperties(openCase).setConfigSetting(ImageGalleryModule.getModuleName(), PerCaseProperties.ENABLED, Boolean.toString(enabledForCaseBox.isSelected())); + } catch (NoCurrentCaseException ex) { + // It's not an error if there's no case open + } + + + } /** diff --git a/InternalPythonModules/android/browserlocation.py b/InternalPythonModules/android/browserlocation.py index 6db230c2e6..a0d73fdd28 100644 --- a/InternalPythonModules/android/browserlocation.py +++ b/InternalPythonModules/android/browserlocation.py @@ -105,7 +105,7 @@ class BrowserLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactTypeName(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactTypeName()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/cachelocation.py b/InternalPythonModules/android/cachelocation.py index 019c03c654..53e60b51ee 100644 --- a/InternalPythonModules/android/cachelocation.py +++ b/InternalPythonModules/android/cachelocation.py @@ -138,7 +138,7 @@ class CacheLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS trackpoint artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/calllog.py b/InternalPythonModules/android/calllog.py index 79d0e82479..b20c0d0512 100644 --- a/InternalPythonModules/android/calllog.py +++ b/InternalPythonModules/android/calllog.py @@ -157,7 +157,7 @@ class CallLogAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index call log artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/contact.py b/InternalPythonModules/android/contact.py index 1d556607e0..22a1f797ec 100644 --- a/InternalPythonModules/android/contact.py +++ b/InternalPythonModules/android/contact.py @@ -164,7 +164,7 @@ class ContactAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index contact artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/googlemaplocation.py b/InternalPythonModules/android/googlemaplocation.py index 444c0477ee..712aacc2be 100644 --- a/InternalPythonModules/android/googlemaplocation.py +++ b/InternalPythonModules/android/googlemaplocation.py @@ -115,7 +115,7 @@ class GoogleMapLocationAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index GPS route artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/tangomessage.py b/InternalPythonModules/android/tangomessage.py index bd65738a6e..514bd48ecb 100644 --- a/InternalPythonModules/android/tangomessage.py +++ b/InternalPythonModules/android/tangomessage.py @@ -116,7 +116,7 @@ class TangoMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index Tango message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/textmessage.py b/InternalPythonModules/android/textmessage.py index a6b36ad549..5854977e8b 100644 --- a/InternalPythonModules/android/textmessage.py +++ b/InternalPythonModules/android/textmessage.py @@ -130,7 +130,7 @@ class TextMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index text message artifact for keyword search.", artifact.getDisplayName()) diff --git a/InternalPythonModules/android/wwfmessage.py b/InternalPythonModules/android/wwfmessage.py index 446ab78b67..6b41ab2677 100644 --- a/InternalPythonModules/android/wwfmessage.py +++ b/InternalPythonModules/android/wwfmessage.py @@ -125,7 +125,7 @@ class WWFMessageAnalyzer(general.AndroidComponentAnalyzer): blackboard = Case.getCurrentCase().getServices().getBlackboard() blackboard.indexArtifact(artifact) except Blackboard.BlackboardException as ex: - self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex) + self._logger.log(Level.SEVERE, "Unable to index blackboard artifact " + str(artifact.getArtifactID()), ex) self._logger.log(Level.SEVERE, traceback.format_exc()) MessageNotifyUtil.Notify.error("Failed to index WWF message artifact for keyword search.", artifact.getDisplayName()) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index c06394bf23..a8dbfb6268 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -43,7 +43,6 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { private final String keywordSearchErrorDialogHeader = org.openide.util.NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.dialogErrorHeader"); protected int filesIndexed; private final Map dataSourceMap = new HashMap<>(); - private final List toolTipList = new ArrayList<>(); private List dataSources = new ArrayList<>(); private final DefaultListModel dataSourceListModel = new DefaultListModel<>(); @@ -153,14 +152,12 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { */ synchronized List getDataSourceArray() { List dsList = new ArrayList<>(); - toolTipList.clear(); Collections.sort(this.dataSources, (DataSource ds1, DataSource ds2) -> ds1.getName().compareTo(ds2.getName())); for (DataSource ds : dataSources) { String dsName = ds.getName(); File dataSourceFullName = new File(dsName); String displayName = dataSourceFullName.getName(); dataSourceMap.put(ds.getId(), displayName); - toolTipList.add(dsName); dsList.add(displayName); } return dsList; @@ -175,22 +172,13 @@ abstract class AdHocSearchPanel extends javax.swing.JPanel { } /** - * Get dataSourceMap with object id and data source display name. Add the - * data source full name to toolTipList + * Get dataSourceMap with object id and data source display name. * * @return The list of data source name */ Map getDataSourceMap() { return dataSourceMap; } - - /** - * Get a list of tooltip text for data source - * @return A list of tool tips - */ - List getDataSourceToolTipList() { - return toolTipList; - } /** * Get a list of DataSourceListModel diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index 72a1cbab5a..0c7566f52e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -18,11 +18,11 @@ */ package org.sleuthkit.autopsy.keywordsearch; -import java.awt.*; +import java.awt.Component; +import java.awt.Cursor; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -32,7 +32,6 @@ import java.util.List; import java.util.Set; import java.util.logging.Level; import javax.swing.JCheckBox; -import javax.swing.JList; import javax.swing.JTable; import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; @@ -73,22 +72,6 @@ class DropdownListSearchPanel extends AdHocSearchPanel { dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList dsList = (JList) evt.getSource(); - int index = dsList.locationToIndex(evt.getPoint()); - if (index > -1) { - dsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } static synchronized DropdownListSearchPanel getDefault() { @@ -683,11 +666,21 @@ class DropdownListSearchPanel extends AdHocSearchPanel { * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java index b859df85ef..bdc80084c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownSingleTermSearchPanel.java @@ -23,8 +23,6 @@ import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.FocusEvent; import java.awt.event.FocusListener; -import java.awt.event.MouseEvent; -import java.awt.event.MouseMotionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; @@ -32,7 +30,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.logging.Level; -import javax.swing.JList; import javax.swing.JMenuItem; import javax.swing.event.ListSelectionEvent; import org.openide.util.NbBundle; @@ -88,22 +85,6 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { this.dataSourceList.addListSelectionListener((ListSelectionEvent evt) -> { firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); }); - this.dataSourceList.addMouseMotionListener(new MouseMotionListener() { - - @Override - public void mouseDragged(MouseEvent evt) { - //Unused by now - } - - @Override - public void mouseMoved(MouseEvent evt) { - JList DsList = (JList) evt.getSource(); - int index = DsList.locationToIndex(evt.getPoint()); - if (index > -1) { - DsList.setToolTipText(getDataSourceToolTipList().get(index)); - } - } - }); } /** @@ -390,18 +371,26 @@ public class DropdownSingleTermSearchPanel extends AdHocSearchPanel { } setComponentsEnabled(); firePropertyChange(Bundle.DropdownSingleTermSearchPanel_selected(), null, null); - } /** * Set the dataSourceList enabled if the dataSourceCheckBox is selected */ private void setComponentsEnabled() { - boolean enabled = this.dataSourceCheckBox.isSelected(); - this.dataSourceList.setEnabled(enabled); - if (enabled) { - this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + if (getDataSourceListModel().size() > 1) { + this.dataSourceCheckBox.setEnabled(true); + + boolean enabled = this.dataSourceCheckBox.isSelected(); + this.dataSourceList.setEnabled(enabled); + if (enabled) { + this.dataSourceList.setSelectionInterval(0, this.dataSourceList.getModel().getSize()-1); + } else { + this.dataSourceList.setSelectedIndices(new int[0]); + } } else { + this.dataSourceCheckBox.setEnabled(false); + this.dataSourceCheckBox.setSelected(false); + this.dataSourceList.setEnabled(false); this.dataSourceList.setSelectedIndices(new int[0]); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 83b770bc58..a8abc03b83 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -28,7 +28,6 @@ import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -92,6 +91,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { private boolean startedSearching = false; private List textExtractors; private StringsTextExtractor stringExtractor; + private TextFileExtractor txtFileExtractor; private final KeywordSearchJobSettings settings; private boolean initialized = false; private long jobId; @@ -244,6 +244,8 @@ public final class KeywordSearchIngestModule implements FileIngestModule { stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); stringExtractor.setOptions(KeywordSearchSettings.getStringExtractOptions()); + txtFileExtractor = new TextFileExtractor(); + textExtractors = new ArrayList<>(); //order matters, more specific extractors first textExtractors.add(new HtmlTextExtractor()); @@ -343,7 +345,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { textExtractors.clear(); textExtractors = null; stringExtractor = null; - + txtFileExtractor = null; initialized = false; } @@ -568,6 +570,17 @@ public final class KeywordSearchIngestModule implements FileIngestModule { putIngestStatus(jobId, aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); } + if ((wasTextAdded == false) && (aFile.getNameExtension().equalsIgnoreCase("txt"))) { + try { + if (Ingester.getDefault().indexText(txtFileExtractor, aFile, context)) { + putIngestStatus(jobId, aFile.getId(), IngestStatus.TEXT_INGESTED); + wasTextAdded = true; + } + } catch (IngesterException ex) { + logger.log(Level.WARNING, "Unable to index as unicode", ex); + } + } + // if it wasn't supported or had an error, default to strings if (wasTextAdded == false) { extractStringsAndIndex(aFile); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java new file mode 100644 index 0000000000..bc11515e96 --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TextFileExtractor.java @@ -0,0 +1,80 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import java.io.IOException; +import java.io.Reader; +import java.util.logging.Level; +import org.apache.tika.parser.txt.CharsetDetector; +import org.apache.tika.parser.txt.CharsetMatch; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.ReadContentInputStream; + +/** + * Extract text from .txt files + */ +final class TextFileExtractor extends ContentTextExtractor { + + //Set a Minimum confidence value to reject matches that may not have a valid text encoding + //Values of valid text encodings were generally 100, xml code sometimes had a value around 50, + //and pictures and other files with a .txt extention were showing up with a value of 5 or less in limited testing. + //This limited information was used to select the current value as one that would filter out clearly non-text + //files while hopefully working on all files with a valid text encoding + static final private int MIN_MATCH_CONFIDENCE = 20; + static final private Logger logger = Logger.getLogger(TextFileExtractor.class.getName()); + + @Override + boolean isContentTypeSpecific() { + return true; + } + + @Override + boolean isSupported(Content file, String detectedFormat) { + return true; + } + + @Override + public Reader getReader(Content source) throws TextExtractorException { + CharsetDetector detector = new CharsetDetector(); + ReadContentInputStream stream = new ReadContentInputStream(source); + try { + detector.setText(stream); + } catch (IOException ex) { + throw new TextExtractorException("Unable to get string from detected text in UnicodeTextExtractor", ex); + } + CharsetMatch match = detector.detect(); + if (match.getConfidence() < MIN_MATCH_CONFIDENCE) { + throw new TextExtractorException("Text does not match any character set with a high enough confidence for UnicodeTextExtractor"); + } + + return match.getReader(); + } + + @Override + public boolean isDisabled() { + return false; + } + + @Override + public void logWarning(String msg, Exception ex) { + logger.log(Level.WARNING, msg, ex); + } + +} diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index e5d93a91d4..4c8999ec47 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2012-2014 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -237,8 +237,19 @@ class SearchEngineURLQueryAnalyzer extends Extract { try { //try to decode the url String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS return decoded; - } catch (UnsupportedEncodingException uee) { //if it fails, return the encoded string - logger.log(Level.FINE, "Error during URL decoding ", uee); //NON-NLS + } catch (UnsupportedEncodingException exception) { //if it fails, return the encoded string + logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:" + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS + return x; + } catch (IllegalArgumentException exception) { //if it fails, return the encoded string + logger.log(Level.SEVERE, "Illegal argument passed to URL decoding, returning undecoded value:" + + "\n\tURL: " + url + + "\n\tUndecoded value: " + x + + "\n\tEngine name: " + eng.getEngineName() + + "\n\tEngine domain: " + eng.getDomainSubstring(), exception); //NON-NLS) return x; } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index a47d33f22a..8b246b05aa 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -86,11 +86,12 @@ class Util { public static String getBaseDomain(String url) { String host = null; + //strip protocol - String cleanUrl = url.replaceFirst("/.*:\\/\\//", ""); + String cleanUrl = url.replaceFirst(".*:\\/\\/", ""); //strip after slashes - String dirToks[] = cleanUrl.split("/\\//"); + String dirToks[] = cleanUrl.split("\\/"); if (dirToks.length > 0) { host = dirToks[0]; } else { @@ -141,7 +142,6 @@ class Util { if (result == null || result.trim().isEmpty()) { return getBaseDomain(value); } - return result; }